# import gevent
# from gevent.pool import Pool
import random
import traceback
import ujson
import asyncio
from quart.ctx import copy_current_websocket_context

from users import UserManager
from manager import ChessManager, GameOverException
from tournaments import TournamentManager

class ControllerExcetions(Exception):
    pass

class InvalidActionFormatException(ControllerExcetions):
    pass

class InvalidNoActionException(ControllerExcetions):
    pass

class InvalidActionNameException(ControllerExcetions):
    pass

class InvalidNoDataException(ControllerExcetions):
    pass

class InvalidLoginException(ControllerExcetions):
    pass

class InvalidTokenException(ControllerExcetions):
    pass

class InvalidRegisterException(ControllerExcetions):
    pass

class TimeoutException(ControllerExcetions):
    pass

class InvalidSaveTurnException(object):
    pass

class Controller:

    def __init__(self, redis_pool, app, connected_websockets):
        self.chess_manager = ChessManager(redis_pool)
        self.user_manager = UserManager(redis_pool, app)
        self.tournament_manager = TournamentManager(redis_pool, self.chess_manager)
        self.board_subscribers = {}
        self.redis_pool = redis_pool
        self.app = app
        self.connected_websockets = connected_websockets

    async def execute_message(self, client, message):
        self.app.logger.info(
            'process_message: message: {}'.format(message)
        )
        await self.process_message(client, message)

    async def get_current_username(self, client):
        auth_token = client.args.get('authtoken')
        if not auth_token:
            raise NoTokenException()
        return await self.user_manager.get_username_by_auth_token(auth_token)

    async def process_message(self, client, message):
        method_name, data = await self.parse_message(message)
        current_username = await self.get_current_username(client)
        self.app.logger.info('process_message from {}: {} {}'.format(current_username, method_name, data))
        method = getattr(self, method_name)
        try:
            await method(current_username, client, data)
            # await self.send(client, 'response_ok', data)
        except Exception as e:
            tb = traceback.format_exc()
            self.app.logger.error('exception {} {}'.format(e, tb))

            data = {
                'exception': str(type(e))
            }
            # gevent.spawn(
            await self.send(client, 'response_error', data)
            raise e

    async def parse_message(self, message):
        try:
            job = ujson.loads(message)
        except ValueError:
            raise InvalidActionFormatException()

        if 'action' not in job:
            raise InvalidNoActionException()
        action_name = job['action']
        method_name = 'action_' + str(action_name)
        if not hasattr(self, method_name):
            raise InvalidActionNameException()

        if 'data' not in job:
            raise InvalidNoDataException()

        data = job['data']
        return method_name, data

    def valid_auth(self, data):
        return 'username' in data and 'password' in data

    async def action_register(self, current_username, client, data):
        if not self.valid_auth(data):
            raise InvalidRegisterException()
        return self.user_manager.register(data['username'], data['password'])

    async def action_get_connected_users(self, current_username, client, data):
        self.app.logger.info('action_get_connected {} {}'.format(client, current_username))
        data = {
            'users_list': await self.get_active_users()
        }
        await self.send(client, 'update_user_list', data)

    async def action_login(self, current_username, client, data=None):
        self.app.logger.info('action_login {} {}'.format(client, current_username))
        client.username = current_username
        client.queue.username = current_username
        data = {
            'users_list': await self.get_active_users()
        }
        self.app.logger.info('connected users: {}'.format(data))
        await self.broadcast('update_user_list', data)

        return True

    async def get_active_users(self):
        return {
            queue.username for queue in self.connected_websockets
            if hasattr(queue, 'username')
        }

    def get_username_by_client(self, client):
        for queue in self.connected_websockets:
            if(
                hasattr(queue, 'webservice') and
                queue.webservice == client and
                hasattr(queue, 'username')
            ):
                return queue.username

    async def action_challenge(self, current_username, client, data):
        challenged_username = data['username']
        challenger_username = current_username
        await self._challenge(challenger_username, challenged_username)

    async def challenge_with_auth_token(self, auth_token, username, message):
        challenger_username = await self.user_manager.get_username_by_auth_token(auth_token)
        return await self._challenge(challenger_username, username)

    async def _challenge(self, challenger_username, challenged_username):
        self.app.logger.info('action_challenge {} from {}'.format(challenged_username, challenger_username))
        if random.choice([True, False]):
            white_username = challenger_username
            black_username = challenged_username
        else:
            white_username = challenged_username
            black_username = challenger_username
        move_left = 200
        board_id = self.chess_manager.challenge(
            white_username=white_username,
            black_username=black_username,
            move_left=move_left,
        )
        data = {
            'username': challenger_username,
            'board_id': board_id,
        }
        await self.broadcast('ask_challenge', data, challenged_username)
        return True

    async def broadcast(self, event, data, username=None):
        for queue in self.connected_websockets:
            if(
                not username or
                (
                    hasattr(queue, 'username') and
                    username == queue.username
                )
            ):
                message = {
                    'event': event,
                    'data': data,
                }
                await queue.put(ujson.dumps(message))

    async def action_accept_challenge(self, current_username, client, data):
        board_id = data['board_id']
        await self._start_board(board_id)
        return True

    async def _start_board(self, board_id):
        turn_token, username, actual_turn, board, move_left, opponent_username = self.chess_manager.challenge_accepted(board_id)
        next_turn_data = {
            'board_id': board_id,
            'turn_token': turn_token,
            'username': username,
            'actual_turn': actual_turn,
            'board': board,
            'move_left': move_left,
            'opponent_username': opponent_username,
        }
        self.app.logger.info('action_accept_challenge ok'.format(board_id, next_turn_data))
        await self.set_next_turn(board_id, next_turn_data)

    async def action_abort(self, current_username, client, data):
        board_id = data['board_id']
        self.chess_manager.abort(board_id, current_username)
        await self.send_gameover(board_id)

    async def action_move(self, current_username, client, data):
        board_id = data['board_id']
        turn_token = data['turn_token']
        key = self.get_next_turn_key(board_id, turn_token)
        self.app.logger.info('action_move control timeout {}'.format(key))
        if self.redis_pool.exists(key):
            self.app.logger.info('action_move control timeout OK {}'.format(key))
            self.redis_pool.delete(key)
        else:
            # timeout...
            self.app.logger.info('action_move control timeout ERROR {}'.format(key))
            raise TimeoutException()
        processed = False
        try:
            turn_token, username, actual_turn, board, move_left, opponent_username = self.chess_manager.move_with_turn_token(
                turn_token=data['turn_token'],
                from_row=data['from_row'],
                from_col=data['from_col'],
                to_row=data['to_row'],
                to_col=data['to_col'],
            )
            processed = True
        except GameOverException:
            await self.send_gameover(board_id)
            return
        except Exception as e:
            tb = traceback.format_exc()
            try:
                self.app.logger.error('action_move {} exception  {} {}'.format(board_id, e, tb))
                await self.force_change_turn(data['board_id'], data['turn_token'])
                return
                # turn_token, username, actual_turn, board, move_left = self.chess_manager._next_turn_token(board_id)
            except GameOverException:
                await self.send_gameover(board_id)
                return
        next_turn_data = {
            'board_id': board_id,
            'turn_token': turn_token,
            'username': username,
            'actual_turn': actual_turn,
            'board': board,
            'move_left': move_left,
            'opponent_username': opponent_username,
        }
        await self.set_next_turn(board_id, next_turn_data)

    def get_next_turn_key(self, board_id, turn_token):
        return "next_turn:{}:{}".format(board_id, turn_token)

    async def set_next_turn(self, board_id, next_turn_data):
        self.app.logger.info('set_next_turn {} {}'.format(board_id, next_turn_data))
        key = self.get_next_turn_key(board_id, next_turn_data['turn_token'])
        self.redis_pool.set(key, ujson.dumps(next_turn_data))
        await self.enqueue_next_turn(key)
        # if not self._save_turn(next_turn_data):
        #     raise InvalidSaveTurnException()

    async def enqueue_next_turn(self, key):
        self.app.logger.info('enqueue_next_turn {}'.format(key))
        # self.redis_pool.rpush("next_turn_queue", key)
        # self.pool.wait_available()
        # self.pool.spawn(self.process_next_turn, key)
        await self.process_next_turn(key)

    def _save_turn(self, data):
        try:
            data_json = ujson.dumps(data)
            self.redis_pool.set(
                "{0}:{1}".format('turn', data['turn_token']),
                data_json)
            return True
        except Exception:
            return False

    async def send_gameover(self, board_id):
        board = self.chess_manager.get_board_by_id(board_id)
        if self.tournament_manager.get_tournament_key('') in board_id:
            self.tournament_manager.board_finish(board_id)
        data = {
            'board': board.board.get_simple(),
            'white_username': str(board.white_username),
            'black_username': str(board.black_username),
            'white_score': str(board.white_score),
            'black_score': str(board.black_score),
            'board_id': board_id,
        }
        await self.broadcast('gameover', data, board.white_username)
        await self.broadcast('gameover', data, board.black_username)

    async def force_change_turn(self, board_id, turn_token):
        self.app.logger.info('force_change_turn {} {}'.format(board_id, turn_token))
        try:
            turn_token, username, actual_turn, board, move_left, opponent_username = self.chess_manager.force_change_turn(board_id, turn_token)
        except GameOverException:
            await self.send_gameover(board_id)
            return
        next_turn_data = {
                'board_id': board_id,
                'turn_token': turn_token,
                'username': username,
                'actual_turn': actual_turn,
                'board': board,
                'move_left': move_left,
                'opponent_username': opponent_username,
        }
        self.app.logger.info('force_change_turn set_next_turn {} {}'.format(board_id, turn_token))
        await self.set_next_turn(board_id, next_turn_data)

    async def process_next_turn(self, key):
        self.app.logger.info('process_next_turn {}'.format(key))
        try:
            # key = self.redis_pool.blpop('next_turn_queue')
            if not key:
                self.app.logger.info('Nothing pending to process')
                return
            data = ujson.loads(self.redis_pool.get(key))
            self.app.logger.info('next_turn key: {} data: {}'.format(key, data))
            await self.broadcast('your_turn', data, data['username'])
            # self.notify_to_board_subscribers(data['board_id'])
            # control timeout
            await asyncio.sleep(30)
            self.app.logger.info('Checking timeout {} {}'.format(data['board_id'], data['turn_token']))
            if self.redis_pool.exists(key):
                self.app.logger.info('Forcing timeout {} {}'.format(data['board_id'], data['turn_token']))
                self.redis_pool.delete(key)
                await self.force_change_turn(data['board_id'], data['turn_token'])
        except Exception as e:
            tb = traceback.format_exc()
            self.app.logger.error('process_next_turn {} exception  {} {}'.format(key, e, tb))
        self.app.logger.info('end process_next_turn {}'.format(key))

    def notify_to_board_subscribers(self, board_id):
        board = self.chess_manager.get_board_by_id(board_id)
        for board_subscriber_client in self.board_subscribers.get(board_id, []):
            self.notify_board_update(board_subscriber_client, board)

    async def notify_board_update(self, board_subscriber_client, board):
        data = {
            'board': board.board.get_simple(),
            'white_username': board.white_username,
            'black_username': board.black_username,
            'white_score': board.white_score,
            'black_score': board.black_score,

        }
        await self.send(board_subscriber_client, 'update_board', data)

    async def action_subscribe(self, current_username, client, data):
        board_id = data['board_id']
        board = self.chess_manager.get_board_by_id(board_id)
        if board_id not in self.board_subscribers:
            self.board_subscribers[board_id] = []
        self.board_subscribers[board_id].append(client)
        self.notify_board_update(client, board)
        return True

    async def send(self, client, event, data):
        """
        Send given data to the registered client.
        Automatically discards invalid connections.
        """
        try:
            self.app.logger.info(u'send to client: {}, event: {}, data: {}'.format(client, event, data))
            message = {
                'event': event,
                'data': data,
            }
            # print 'sent to {0}: {1}'.format(client, message)
            await client.send(ujson.dumps(message))
        except Exception:
            pass
            #  app.logger.info(u'Exception on sending to client: {}'.format(client))
            #  self.clients.remove(client)

    async def action_create_tournament(self, current_username, client, data):
        tournament = self.tournament_manager.create_tournament()
        await self.send(client, 'tournament_created', tournament)
        return True

    async def action_add_user_to_tournament(self, current_username, client, data):
        tournament_id = data['tournament_id']
        username = data['username']
        if username == '*':
            active_usernames = await self.get_active_users()
            for username in active_usernames:
                self.tournament_manager.add_user(tournament_id, username)
        else:
            self.tournament_manager.add_user(tournament_id, username)
        users = self.tournament_manager.get_users(tournament_id)
        await self.send(client, 'user_added_to_tournament', users)
        return True

    async def action_start_tournament(self, current_username, client, data):
        tournament_id = data['tournament_id']
        tournament = self.tournament_manager.get_tournament(tournament_id)
        # TODO: control and change state...
        boards = self.tournament_manager.start(tournament_id)
        for board_id in boards:
            asyncio.create_task(self._start_board(board_id))
        users = self.tournament_manager.get_users(tournament_id)

        await self.send(client, 'tournament_started', users)
        return True


import json

import pytest
from bs4 import BeautifulSoup
from flask import url_for
from freezegun import freeze_time
from tests.conftest import (
    SERVICE_ONE_ID,
    mock_get_notifications,
    normalize_spaces,
)

from app.main.views.jobs import get_time_left

def test_get_jobs_should_return_list_of_all_real_jobs(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_jobs,
    mocker,
):
    response = logged_in_client.get(url_for('main.view_jobs', service_id=service_one['id']))

    assert response.status_code == 200
    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
    assert page.h1.string == 'Uploaded files'
    jobs = [x.text for x in page.tbody.find_all('a', {'class': 'file-list-filename'})]
    assert len(jobs) == 4

def test_get_jobs_shows_page_links(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_jobs,
    mocker,
):
    response = logged_in_client.get(url_for('main.view_jobs', service_id=service_one['id']))

    assert response.status_code == 200
    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
    assert 'Next page' in page.find('li', {'class': 'next-page'}).text
    assert 'Previous page' in page.find('li', {'class': 'previous-page'}).text

@pytest.mark.parametrize(
    "status_argument, expected_api_call", [
        (
            '',
            [
                'created', 'pending', 'sending', 'pending-virus-check',
                'delivered', 'sent',
                'failed', 'temporary-failure', 'permanent-failure', 'technical-failure', 'virus-scan-failed',
            ]
        ),
        (
            'sending',
            ['sending', 'created', 'pending', 'pending-virus-check']
        ),
        (
            'delivered',
            ['delivered', 'sent']
        ),
        (
            'failed',
            ['failed', 'temporary-failure', 'permanent-failure', 'technical-failure', 'virus-scan-failed']
        )
    ]
)
@freeze_time("2016-01-01 11:09:00.061258")
def test_should_show_page_for_one_job(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_service_template,
    mock_get_job,
    mocker,
    mock_get_notifications,
    fake_uuid,
    status_argument,
    expected_api_call,
):

    response = logged_in_client.get(url_for(
        'main.view_job',
        service_id=service_one['id'],
        job_id=fake_uuid,
        status=status_argument
    ))

    assert response.status_code == 200
    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
    assert page.h1.text.strip() == 'thisisatest.csv'
    assert ' '.join(page.find('tbody').find('tr').text.split()) == (
        '07123456789 template content Delivered 1 January at 11:10am'
    )
    assert page.find('div', {'data-key': 'notifications'})['data-resource'] == url_for(
        'main.view_job_updates',
        service_id=service_one['id'],
        job_id=fake_uuid,
        status=status_argument,
    )
    csv_link = page.select_one('a[download]')
    assert csv_link['href'] == url_for(
        'main.view_job_csv',
        service_id=service_one['id'],
        job_id=fake_uuid,
        status=status_argument
    )
    assert csv_link.text == 'Download this report'
    assert page.find('span', {'id': 'time-left'}).text == 'Data available for 7 days'
    mock_get_notifications.assert_called_with(
        service_one['id'],
        fake_uuid,
        status=expected_api_call
    )

def test_get_jobs_should_tell_user_if_more_than_one_page(
    logged_in_client,
    fake_uuid,
    service_one,
    mock_get_job,
    mock_get_service_template,
    mock_get_notifications_with_previous_next,
):
    response = logged_in_client.get(url_for(
        'main.view_job',
        service_id=service_one['id'],
        job_id=fake_uuid,
        status=''
    ))

    assert response.status_code == 200
    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
    assert page.find('p', {'class': 'table-show-more-link'}).text.strip() == 'Only showing the first 50 rows'

def test_should_show_job_in_progress(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_service_template,
    mock_get_job_in_progress,
    mocker,
    mock_get_notifications,
    fake_uuid,
):

    response = logged_in_client.get(url_for(
        'main.view_job',
        service_id=service_one['id'],
        job_id=fake_uuid
    ))

    assert response.status_code == 200
    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
    assert page.find('p', {'class': 'hint'}).text.strip() == 'Report is 50% complete…'

@freeze_time("2016-01-01 11:09:00.061258")
def test_should_show_letter_job(
    client_request,
    mock_get_service_letter_template,
    mock_get_job,
    fake_uuid,
    active_user_with_permissions,
    mocker,
):

    get_notifications = mock_get_notifications(mocker, active_user_with_permissions, diff_template_type='letter')

    page = client_request.get(
        'main.view_job',
        service_id=SERVICE_ONE_ID,
        job_id=fake_uuid,
    )
    assert normalize_spaces(page.h1.text) == 'thisisatest.csv'
    assert normalize_spaces(page.select('p.bottom-gutter')[0].text) == (
        'Sent by Test User on 1 January at 11:09am'
    )
    assert page.select('.banner-default-with-tick') == []
    assert normalize_spaces(page.select('tbody tr')[0].text) == (
        '1 Example Street template content 1 January at 11:09am'
    )
    assert normalize_spaces(page.select('.keyline-block')[0].text) == (
        '1 Letter'
    )
    assert normalize_spaces(page.select('.keyline-block')[1].text) == (
        '6 January Estimated delivery date'
    )
    assert page.select('[download=download]') == []
    assert page.select('.hint') == []

    get_notifications.assert_called_with(
        SERVICE_ONE_ID,
        fake_uuid,
        status=[
            'created',
            'pending',
            'sending',
            'pending-virus-check',
            'delivered',
            'sent',
            'failed',
            'temporary-failure',
            'permanent-failure',
            'technical-failure',
            'virus-scan-failed',
        ],
    )

@freeze_time("2016-01-01 11:09:00.061258")
def test_should_show_letter_job_with_banner_after_sending(
    client_request,
    mock_get_service_letter_template,
    mock_get_job,
    mock_get_notifications,
    fake_uuid,
):

    page = client_request.get(
        'main.view_job',
        service_id=SERVICE_ONE_ID,
        job_id=fake_uuid,
        just_sent='yes',
    )

    assert page.select('p.bottom-gutter') == []
    assert normalize_spaces(page.select('.banner-default-with-tick')[0].text) == (
        'We’ve started printing your letters'
    )

@freeze_time("2016-01-01T00:00:00.061258")
def test_should_show_scheduled_job(
    logged_in_client,
    active_user_with_permissions,
    mock_get_service_template,
    mock_get_scheduled_job,
    mocker,
    mock_get_notifications,
    fake_uuid,
):
    response = logged_in_client.get(url_for(
        'main.view_job',
        service_id=SERVICE_ONE_ID,
        job_id=fake_uuid
    ))

    assert response.status_code == 200
    page = BeautifulSoup(response.data.decode('utf-8'), 'html.parser')
    assert normalize_spaces(page.select('main p')[1].text) == (
        'Sending Two week reminder today at midnight'
    )
    assert page.select('main p a')[0]['href'] == url_for(
        'main.view_template_version',
        service_id=SERVICE_ONE_ID,
        template_id='5d729fbd-239c-44ab-b498-75a985f3198f',
        version=1,
    )
    assert page.select_one('button[type=submit]').text.strip() == 'Cancel sending'

def test_should_cancel_job(
    logged_in_client,
    service_one,
    fake_uuid,
    mocker,
):
    mock_cancel = mocker.patch('app.main.jobs.job_api_client.cancel_job')
    response = logged_in_client.post(url_for(
        'main.cancel_job',
        service_id=service_one['id'],
        job_id=fake_uuid
    ))

    mock_cancel.assert_called_once_with(service_one['id'], fake_uuid)
    assert response.status_code == 302
    assert response.location == url_for('main.service_dashboard', service_id=service_one['id'], _external=True)

def test_should_not_show_cancelled_job(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_cancelled_job,
    mocker,
    fake_uuid,
):
    response = logged_in_client.get(url_for(
        'main.view_job',
        service_id=service_one['id'],
        job_id=fake_uuid
    ))

    assert response.status_code == 404

@freeze_time("2016-01-01 00:00:00.000001")
def test_should_show_updates_for_one_job_as_json(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_notifications,
    mock_get_service_template,
    mock_get_job,
    mocker,
    fake_uuid,
):
    response = logged_in_client.get(url_for('main.view_job_updates', service_id=service_one['id'], job_id=fake_uuid))

    assert response.status_code == 200
    content = json.loads(response.get_data(as_text=True))
    assert 'sending' in content['counts']
    assert 'delivered' in content['counts']
    assert 'failed' in content['counts']
    assert 'Recipient' in content['notifications']
    assert '07123456789' in content['notifications']
    assert 'Status' in content['notifications']
    assert 'Delivered' in content['notifications']
    assert '12:01am' in content['notifications']
    assert 'Sent by Test User on 1 January at midnight' in content['status']

@pytest.mark.parametrize(
    "job_created_at, expected_message", [
        ("2016-01-10 11:09:00.000000+00:00", "Data available for 7 days"),
        ("2016-01-04 11:09:00.000000+00:00", "Data available for 1 day"),
        ("2016-01-03 11:09:00.000000+00:00", "Data available for 11 hours"),
        ("2016-01-02 23:59:59.000000+00:00", "Data no longer available")
    ]
)
@freeze_time("2016-01-10 12:00:00.000000")
def test_time_left(job_created_at, expected_message):
    assert get_time_left(job_created_at) == expected_message

@pytest.mark.parametrize('original_file_contents', [
    """
        phone_number
        07800900123
    """,
    """
        phone_number, a, b, c
        07800900123,  🐜,🐝,🦀
    """,
    """
        "phone_number", "a", "b", "c"
        "07800900123","🐜,🐜","🐝,🐝","🦀"
    """,
])
@freeze_time("2016-01-01 11:09:00.061258")
def test_should_download_csv_even_with_unicode_in_template_name(
    logged_in_client,
    service_one,
    active_user_with_permissions,
    mock_get_unicode_service_template,
    mock_get_job,
    mocker,
    mock_get_notifications,
    fake_uuid,
    original_file_contents,
):
    mocker.patch(
        'app.main.s3_client.s3download',
        return_value=original_file_contents,
    )

    response = logged_in_client.get(url_for(
        'main.view_job_csv',
        service_id=service_one['id'],
        job_id=fake_uuid
    ))

    assert response.status_code == 200

    for header, header_value in response.headers:
        header_value.encode('ascii') # this throws exception if not ascii


from rdflib import BNode, Literal, Namespace, Graph
from rdflib.namespace import FOAF, XSD, RDF
from rdflib.extras.external_graph_libs import rdflib_to_networkx_multidigraph, rdflib_to_networkx_digraph
# import networkx as nx
from random import shuffle
from ast import literal_eval
import os
import pandas as pd
import json

class PatientKG (object):
    def __init__(self) -> None:
        super().__init__()
        self.diseaselist = ['abnormal', 'normal', 'lung opacity', 'atelectasis', 'pleural effusion', 'vascular congestion', 'pulmonary edema/hazy opacity',
                    'low lung volumes', 'pneumonia', 'endotracheal tube', 'enlarged cardiac silhouette', 'aspiration', 'linear/patchy atelectasis',
                    'vascular calcification', 'tortuous aorta', 'pleural/parenchymal scarring', 'consolidation', 'fluid overload/heart failure',
                    'mass/nodule (not otherwise specified)', 'copd/emphysema', 'hyperaeration', 'airspace opacity', 'lobar/segmental collapse',
                    'enlarged hilum', 'calcified nodule']

    def newGraph(self, filename):
        AIDAS = Namespace('http://aidas.org/')

        mido = Namespace('https://bioportal.bioontology.org/ontologies/MIDO/')
        ddo = Namespace('http://purl.obolibrary.org/obo/DDO.owl#')
        radlex = Namespace('radlex.org/RID/RadLex.owl/')
        ncit = Namespace('http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#')
        symp = Namespace('http://purl.obolibrary.org/obo/symp.owl/')
        ogms = Namespace('http://purl.obolibrary.org/obo/OGMS_0000031/')
        prov = Namespace('https://www.w3.org/TR/prov-o/')
        doid = Namespace('http://purl.obolibrary.org/obo/doid#')

        filename = filename

        KGs = {}
        try:
            patient = pd.read_csv('patient.csv', sep='\t')
            patient[['output', 'target']].to_numpy()
        except FileNotFoundError:
            print('{} not in directory'.format(filename))
        else:
            g = Graph()
            # data = json.load(f)

            patientProfile = ddo['demographic']#BNode()
            # age = Literal(data['age_decile'], datatype=XSD['string'])
            imageID = Literal(patient['image_id'][0], datatype=XSD['string'])
            patientID = Literal(patient['image_id'][0], datatype=XSD['string'])
            # gender = Literal(data['gender'], datatype=XSD['string'])
            # StudyDateTime = Literal(data['StudyDateTime'], datatype=XSD['datetime'])

            g.add((patientID, RDF.type, ncit['Patient']))
            g.add((patientID, ddo['hasDemographics'], patientProfile))
            # g.add((patientProfile, CXRO['hasAge'], age))
            g.add((patientProfile, mido['hasID'], patientID))
            # g.add((patientProfile, CXRO['hasGender'], gender))
            g.add((patientID, ddo['hasLaboratoryTest'], ogms['LaboratoryTest']))
            g.add((ogms['LaboratoryTest'], ddo['hasReport'], radlex['Report']))
            g.add((radlex['Report'], mido['hasID'], imageID))
            # g.add((CXRO['cxrReport'], PROV['wasGeneratedAtTime'], StudyDateTime))

            # for attribute in data['attributes']:
            for index, row in patient.iterrows():
                att = row['objects'].replace(' ', '_')
                name = row['objects']
                g.add((patientID, mido['hasAttribute'], mido[att]))
                g.add((mido[att], RDF.type, mido['AnatomicalRegion']))

                patient = PatientKG()
                #create subgraph
                subG = patient.subGraph(row)
                #reason over sub graph
                patient.reasoner(row)
                KGs[name] = subG
                g += subG
                # print(subG.serialize())

            g.bind('ddo', ddo)
            g.bind('RadLex', radlex)
            g.bind('mido', mido)
            g.bind('ogms', ogms)
            g.bind('ncit', ncit)
            g.bind('prov', prov)

            KGs['full KG'] = g
        # g = g.serialize(format='turtle')
        # g = g.serialize()
        return g

    def subGraph(self, attribute):
        mido = Namespace('https://bioportal.bioontology.org/ontologies/MIDO/')
        ddo = Namespace('http://purl.obolibrary.org/obo/DDO.owl#')
        radlex = Namespace('radlex.org/RID/RadLex.owl/')
        ncit = Namespace('http://ncicb.nci.nih.gov/xml/owl/EVS/Thesaurus.owl#')
        symp = Namespace('http://purl.obolibrary.org/obo/symp.owl/')
        ogms = Namespace('http://purl.obolibrary.org/obo/OGMS_0000031/')
        prov = Namespace('https://www.w3.org/TR/prov-o/')
        doid = Namespace('http://purl.obolibrary.org/obo/doid#')

        g = Graph()
        att = attribute['objects'].replace(' ', '_')
        positive = Literal('Positive', datatype=XSD['string'])
        negative = Literal('Negative', datatype=XSD['string'])
        # print(attribute['output'])
        att = literal_eval(attribute['output'])
        for index, disease in enumerate(att):
            # print(disease)
            if disease == 1:
                dis = self.diseaselist[index]
                dis = dis.replace(' ', '_')
                # find = disease.split('|')[0]
                g.add((radlex['Report'], mido['hasFinding'], mido[dis]))
                # g.add((CXRO[dis], RDF.type, CXRO[find]))
                g.add((mido[dis], prov['value'], positive))
                g.add((mido[dis], prov['wasAssociatedWith'], mido[att]))
            # elif disease == 0:
            #     dis = self.diseaselist[index]
            #     dis = dis.replace(' ', '_')
            #     # find = disease.split('|')[0]
            #     g.add((radlex['Report'], mido['hasFinding'], mido[dis]))
            #     # g.add((CXRO[dis], RDF.type, CXRO[find]))
            #     g.add((mido[dis], prov['value'], negative))
            #     g.add((mido[dis], prov['wasAssociatedWith'], mido[att]))
        return g

    def reasoner(self, row):
        #abnormal and any other condition
        att = row['objects']
        diseases = literal_eval(row['output'])
        normal = diseases.pop(1)
        if (normal == 1) and (1 in diseases):
            index_list = [i for i, x in enumerate(diseases) if x == 1]
            res_list = [self.diseaselist[i] for i in index_list]
            print ("Patient's {} cannot be normal and also have {} attributes: ".format(att, str(res_list)))

    def generateContext(self, rdfgraph):
        context = []
        for s,p,o in rdfgraph:
            sen = s.split('/')[-1].replace('_', ' ')
            sen += ' ' + p.split('/')[-1].replace('_', ' ').split('#')[-1]
            sen += ' ' + o.split('/')[-1].replace('_', ' ')
            context.append(sen)
        shuffle(context)
        return context

    def randomwalk(self, rdfgraph):
        G = rdflib_to_networkx_digraph(rdfgraph)
        all_paths = nx.all_pairs_shortest_path(G)
        # print(len(G))
        return all_paths

if __name__=='__main__':
    filename = '/Users/nneka/Downloads/3rd year phd/ISWC/Code/data/patient.csv'
    patient = PatientKG()
    g = patient.newGraph(filename)
    g.serialize(destination="patient.ttl")


import cppyy
import numpy as np
from array import array

# Load the library
cppyy.include('../include/pixel.h')
cppyy.load_library('../bin/pixel.so')
# The library lives in cppyy.gbl

###########################    FROM PYTHON TO C    ############################
##########    1D ARRAY    ##########
## python list - must use vector
print("\n1D    python list")
cppyy.cppdef("""
    void test_1D(const std::vector<int>& array) {
        for (int i=0; i<array.size(); i++) {
                std::cout << array[i] << " ";
        }
        std::cout << std::endl;
    }
""")

var = np.array([1, 2, 3, 4]).tolist()
cppyy.gbl.test_1D(var)

## array.array()
print("\n1D    array.array()")
R = array('f',[1., 3., 5., 8.])
z = array('f',[2., 3., 5., 7.])
o1 = cppyy.gbl.pixel(R,z)
o1.Print()

## np.array()
## dtype must be explicitly defined to match C types
print("\n1D    np.array()")
R = np.array([1., 3., 5., 8.], dtype=np.float32)    # float64 not suported
z = np.array([2., 3., 5., 7.], dtype=np.float32)
o2 = cppyy.gbl.pixel(R,z)
o2.Print()

##########    2D ARRAY    ##########

## np.array()                                                   [DOES NOT WORK]
#print("2D    np.array()")
### The C module access only first sub-array correctly
#R = np.array([[1.,2.], [3.,3.], [5.,5.], [8.,7.]], dtype=np.float32, order='C')
#o4 = cppyy.gbl.pixel(R)
#o4.Print()

## python list - must use vector
print("\n2D   python list with vector on C side")
cppyy.cppdef("""
    void test_2D(const std::vector<std::vector<double>>& array) {
        for (int i=0; i<array.size(); i++) {
            for (int j=0; j<array[i].size(); j++) {
                std::cout << array[i][j] << " ";
            }
            std::cout << std::endl;
        }
    }
""")

#var = np.array([[1, 2], [3, 4]]).tolist()          # np.array not accepted
#cppyy.gbl.test_2D(var)

# it works only if C expects a vector of double and python uses np.float32
var = np.array([[1., 2.], [3., 4.]], dtype=np.float32).tolist()
cppyy.gbl.test_2D(var)
#cppyy.gbl.test(var)

###########################    FROM C TO PYTHON    ############################
##########    1D ARRAY    ##########
# it works with dynamically declared arrays only!!!
print("\n\nHow to return variables to Python")
cppyy.cppdef("""
float* create_float_array(int sz) {
    //float* pf = (float*)malloc(sizeof(float)*sz);
    float* pf = new float[sz];
    for (int i = 0; i < sz; ++i) pf[i] = 2*i;
    return pf;
}""")
NDATA = 8
arr = cppyy.gbl.create_float_array(NDATA)
print(arr)
arr.reshape((NDATA,))   # adjust the llv's size
v = np.frombuffer(arr, dtype=np.float32, count=NDATA)  # cast to float
# frombuffer supports only 1D array
print(len(v))
print(v)


"""
https://leetcode.com/problems/heaters/
"""

from typing import List
from bisect import bisect_left

class Solution:
    def findRadius(self, houses: List[int], heaters: List[int]) -> int:
        heaters.sort()
        r, n = 0, len(heaters)
        for h in houses:
            # Try to find which heater pairs the current house is between.
            i = bisect_left(heaters, h)
            if i == 0:  # On the left side of the first heater.
                r = max(r, heaters[0] - h)
            elif i == n:  # On the right side of the last heater.
                r = max(r, h - heaters[-1])
            else:  # Between two heaters, try to find the mininum distance.
                r = max(r, min(h - heaters[i - 1], heaters[i] - h))

        return r


import asyncio
from typing import List

from . import utils
from .abc import IRaftServer
from .errors import *
from .rpc import protocol as prot
from .rpc import rpc
from .state_machine import RaftStateMachine, State, Command

ELECTION_TIMEOUT = 0.5
FLEXIBLE_PAXOS_QUORUM = 2 / 6
RPC_TIMEOUT = 1

class ClusterMember:
    def __init__(self, ip: str, port: int):
        self.ip = ip
        self.port = port
        self.id = utils.get_id(self, ip, port)

class Server(ClusterMember):
    def __init__(self, ip: str, port: int, cluster):
        super().__init__(ip, port)
        self._cluster: List[rpc.RemoteRaftServer] = cluster or []
        self._leader = None
        self._leader_hbeat = asyncio.Event()
        self._leader_volatile_state_data = None
        self._listener_task = None

    async def _start_listening(self):
        self._listener_task = await asyncio.start_server(
            self._handle_request, self.ip, self.port
        )

    async def _handle_request(self, reader, writer):
        message = await prot.read_decode_msg(reader)
        if isinstance(self, RaftServer):
            await rpc.handle_request(self, (reader, writer), message)
        else:
            raise TypeError("Invalid Server instance")
        writer.close()

class RaftServer(IRaftServer, Server, RaftStateMachine):
    def __init__(
        self, ip: str, port: int, cluster, state: State = State.FOLLOWER, log=None
    ):
        Server.__init__(self, ip, port, cluster)
        RaftStateMachine.__init__(self, state, log)

        self._voted_for = None  # candidateId that received vote in currentterm
        self._next_indexes = (
            {}
        )  # for each server, index of the next log entryto send to that server
        self._match_indexes = (
            {}
        )  # for each server, index of highest log entryknown to be replicated on server
        self._commands_queue = (
            asyncio.Queue()
        )  # Queue where commands waiting for commit process to start  are stored

        self._cluster_locks = (
            {}
        )  # Lock for mantaining order when several AppendEntries RPC calls are sent for same server

        self._election_task = None
        self._timeout_task = None

        self._entries_task = None
        self._hbeat_task = None

        self._append_tasks = (
            {}
        )  # TODO: every cluster member has a list of tasks running
        # TODO: init tasks

    async def update_state(self, key, value):
        command = Command(key, value)
        if self._leader is self:
            await self._queue_command(command)
        else:
            while True:
                try:
                    await self._leader.command_request(command)
                    break
                except TermConsistencyError as error:
                    self._current_term = error.term
                    pass  # TODO: do something else?
                except LeaderConsistencyError as error:
                    self.leader = list(
                        filter(lambda s: s.id == error.leader_id, self.cluster)
                    )[0]

    async def join_cluster(self, random_server: ClusterMember):
        if random_server:
            remote_server = rpc.RemoteRaftServer(random_server.ip, random_server.port)
            self.cluster, leader_id = await remote_server.get_cluster_configuration()
            self.leader = list(filter(lambda s: s.id == leader_id, self.cluster))[0]

            if self not in self.cluster:
                self.cluster.append(self)
                # TODO: init configuration change
            else:
                pass  # TODO: means it already was in the cluster, but it had crushed
        else:
            pass  # TODO: first cluster member

    async def leave_cluster(self):
        self.cluster.remove(self)
        # TODO: init configuration change

    async def remove_cluster_member(self, id):
        self.cluster = list(filter(lambda s: s.id != id, self.cluster))
        # TODO: init configuration change

    async def _run_timeout_task(self):
        while True:
            try:
                await asyncio.wait_for(
                    self._leader_hbeat.wait(), timeout=ELECTION_TIMEOUT
                )  # TODO: random timeout
            except asyncio.TimeoutError:
                await self._change_state(State.CANDIDATE)
            finally:
                self._leader_hbeat.clear()

    async def _run_entries_task_(self):
        command = await self._commands_queue.get()
        self._append_command(command)
        rpc_calls = []
        for server in filter(lambda s: s is not self, self.cluster):
            append_task = asyncio.create_task(
                self._append_entry_task(s, len(self._log) - 1)
            )
            rpc_calls.append(append_task)
            self._append_tasks[server.id].append(append_task)

        committed_amount = 1  # Starts on '1' because of itself
        for rpc_call in asyncio.as_completed(rpc_calls):
            await rpc_call.result()
            committed_amount += 1
            if committed_amount >= int(len(self._cluster) * FLEXIBLE_PAXOS_QUORUM):
                self._commit_command(command)
                break

    async def _run_hbeat_task(self):
        while True:
            await self._send_hbeat()
            await asyncio.sleep(
                ELECTION_TIMEOUT * 0.9
            )  # Just in case there is high latency

    async def _run_election_task(self):
        self._current_term += 1
        self._leader_hbeat.set()
        last_log_index = self._last_applied
        last_log_term = (
            0 if not len(self._log) > last_log_index else self._log[last_log_index].term
        )
        voting_rpcs = list(
            map(
                lambda s: asyncio.create_task(
                    s.request_vote(
                        self._current_term, self.id, last_log_index, last_log_term
                    )
                ),
                filter(lambda s: s is not self, self._cluster),
            )
        )
        granted_votes = 1  # 1 -> its own vote
        votes = 1
        election_win = False
        for next_vote in asyncio.as_completed(voting_rpcs, timeout=RPC_TIMEOUT):
            try:
                vote = (await next_vote).result()
                granted_votes += int(vote)
            except asyncio.TimeoutError:
                pass
            votes += 1
            if granted_votes >= int(
                len(self._cluster) * (1 - FLEXIBLE_PAXOS_QUORUM) + 1
            ):  # Equal because itself is not considered
                election_win = True
        if election_win:
            self._change_state(State.LEADER)

    async def _queue_command(self, command: Command):
        await self._commands_queue.put(command)

    async def _send_hbeat(self):
        for server in filter(lambda s: s != self, self.cluster):
            task = asyncio.create_task(
                server.append_entries(
                    self._current_term,
                    self.id,
                    self._last_applied,
                    self._log[self._last_applied].term,
                    None,
                    self._commit_index,
                )
            )
            self._append_tasks[server.id].append(task)

    def _change_state(self, new_state: State):
        if new_state is State.FOLLOWER:
            self._cancel_leader_tasks()
            self._cancel_candidate_tasks()
            self._timeout_task = asyncio.create_task(self._run_timeout_task())
            self._state = State.FOLLOWER
        elif new_state is State.LEADER:
            if self._timeout_task and not self._timeout_task.cancelled():
                self._timeout_task.cancel()
            self._hbeat_task = asyncio.create_task(self._run_hbeat_task())
            self._entries_task = asyncio.create_task(self._run_entries_task_())
            self._state = State.LEADER
        elif new_state is State.CANDIDATE:
            self._cancel_leader_tasks()
            self._cancel_candidate_tasks()
            self._election_task = asyncio.create_task(self._run_election_task())
            self._state = State.CANDIDATE

    def _cancel_leader_tasks(self):
        if self._hbeat_task and not self._hbeat_task.cancelled():
            self._hbeat_task.cancel()
        if self._entries_task and not self._entries_task.cancelled():
            self._entries_task.cancel()
        self._cancel_append_tasks()
        self._cluster_locks.clear()

    def _cancel_append_tasks(self):
        for server_tasks in self._append_tasks.values():
            for task in server_tasks:
                if not task.cancelled():
                    task.cancel()
            server_tasks.clear()

    def _cancel_candidate_tasks(self):
        if self._election_task and not self._election_task.cancelled():
            self._election_task.cancel()

    def _im_leader(self):
        return self._state is State.LEADER

    async def _append_entry_task(
        self, server: rpc.RemoteRaftServer, entries_index: int
    ):
        async with self._cluster_locks[server.id]:
            while True:
                try:
                    await server.append_entries(
                        self._current_term,
                        self.id,
                        max(entries_index - 1, 0),
                        self._log[max(entries_index - 1, 0)].term,
                        self._log[entries_index:],
                        self._commit_index,
                    )
                    break
                except TermConsistencyError as error:
                    self._current_term = error.term
                    self._change_state(State.FOLLOWER)
                    break
                except EntriesConsistencyError:
                    entries_index = max(entries_index - 1, 0)
                except:
                    pass  # Network error, so retry until it answers


import torch
import torch.nn as nn
import torch.nn.functional as F

class ConvEncoder(nn.Module):
    def __init__(self, num_input_channels: int=1, num_filters: int=32,
                 z_dim: int=8, act_fn: nn.Module=nn.ReLU):
        """
        Convolutional Encoder network with Convolution and Linear layers, ReLU activations. The output layer
        uses a Fully connected layer to embed the representation to a latent code with z_dim dimension.
        Inputs:
            z_dim - Dimensionality of the latent code space.
        """
        super().__init__()
        # For an intial architecture, you can use the encoder of Tutorial 9.
        # Feel free to experiment with the architecture yourself, but the one specified here is
        # sufficient for the assignment.
        c_hid = num_filters
        self.net = nn.Sequential(
            nn.Conv2d(num_input_channels, c_hid, kernel_size=3, padding=1, stride=2), # 32x32 => 16x16
            act_fn(),
            nn.Conv2d(c_hid, c_hid, kernel_size=3, padding=1),
            act_fn(),
            nn.Conv2d(c_hid, 2*c_hid, kernel_size=3, padding=1, stride=2), # 16x16 => 8x8
            act_fn(),
            nn.Conv2d(2*c_hid, 2*c_hid, kernel_size=3, padding=1),
            act_fn(),
            nn.Conv2d(2*c_hid, 2*c_hid, kernel_size=3, padding=1, stride=2), # 8x8 => 4x4
            act_fn(),
            nn.Flatten(), # Image grid to single feature vector
            nn.Linear(2*16*c_hid, z_dim)
        )
        # self.mean = nn.Linear(2 * 16 * num_filters, z_dim)
        # self.log_std = nn.Linear(2 * 16 * num_filters, z_dim)

    def forward(self, x):
        """
        Inputs:
            x - Input batch of Images. Shape: [B,C,H,W]
        Outputs:
            z - Output of latent codes [B, z_dim]
        """
        x = x.float() / 15 * 2.0 - 1.0  # Move images between -1 and 1
        z = self.net(x)

        return z

    @property
    def device(self):
        """
        Property function to get the device on which the generator is
        """
        return next(self.parameters()).device

class ConvDecoder(nn.Module):
    def __init__(self, num_input_channels: int=1, num_filters: int=32,
                 z_dim: int=8, act_fn: nn.Module=nn.ReLU):
        """
        Convolutional Decoder network with linear and deconvolution layers and ReLU activations. The output layer
        uses a Tanh activation function to scale the output between -1 and 1.
        Inputs:
              z_dim - Dimensionality of the latent code space.
        """
        super().__init__()
        c_hid = num_filters
        self.linear = nn.Sequential(
            nn.Linear(z_dim, 2*16*c_hid),
            act_fn()
        )
        self.net = nn.Sequential(
            nn.ConvTranspose2d(2*c_hid, 2*c_hid, kernel_size=3, output_padding=0, padding=1, stride=2), # 4x4 => 7x7
            act_fn(),
            nn.Conv2d(2*c_hid, 2*c_hid, kernel_size=3, padding=1),
            act_fn(),
            nn.ConvTranspose2d(2*c_hid, c_hid, kernel_size=3, output_padding=1, padding=1, stride=2), # 7x7 => 14x14
            act_fn(),
            nn.Conv2d(c_hid, c_hid, kernel_size=3, padding=1),
            act_fn(),
            nn.ConvTranspose2d(c_hid, num_input_channels, kernel_size=3, output_padding=1, padding=1, stride=2), # 14x14 => 28x28
            nn.Tanh() # The input images is scaled between -1 and 1, hence the output has to be bounded as well
        )

    def forward(self, z):
        """
        Inputs:
            z - Batch of latent codes. Shape: [B,z_dim]
        Outputs:
            recon_x - Reconstructed image of shape [B,C,H,W]
        """
        recon_x = self.linear(z)
        recon_x = recon_x.reshape(recon_x.shape[0], -1, 4, 4)
        recon_x = self.net(recon_x)

        return recon_x

class Discriminator(nn.Module):
    def __init__(self, n_hidden: int=512, z_dim: int=8, act_fn: nn.Module=nn.LeakyReLU(negative_slope=0.2)):
        """
        Discriminator network with linear layers and LeakyReLU activations.
        Inputs:
              z_dim - Dimensionality of the latent code space.
        """
        super().__init__()
        # You are allowed to experiment with the architecture and change the activation function, normalization, etc.
        # However, the default setup is sufficient to generate fine images and gain full points in the assignment.
        # As a default setup, we recommend 3 linear layers (512 for hidden units) with LeakyReLU activation functions (negative slope 0.2).
        self.net = nn.Sequential(
            nn.Linear(z_dim, n_hidden),
            act_fn,
            nn.Linear(n_hidden, n_hidden),
            act_fn,
            nn.Linear(n_hidden, 1),
            act_fn
        )

    def forward(self, z):
        """
        Inputs:
            z - Batch of latent codes. Shape: [B,z_dim]
        Outputs:
            preds - Predictions whether a specific latent code is fake (<0) or real (>0).
                    No sigmoid should be applied on the output. Shape: [B,1]
        """
        preds = self.net(z)

        return preds

    @property
    def device(self):
        """
        Property function to get the device on which the generator is
        """
        return next(self.parameters()).device

class AdversarialAE(nn.Module):
    def __init__(self, z_dim=8):
        """
        Adversarial Autoencoder network with a Encoder, Decoder and Discriminator.
        Inputs:
              z_dim - Dimensionality of the latent code space. This is the number of neurons of the code layer
        """
        super(AdversarialAE, self).__init__()
        self.z_dim = z_dim
        self.encoder = ConvEncoder(z_dim=z_dim)
        self.decoder = ConvDecoder(z_dim=z_dim)
        self.discriminator = Discriminator(z_dim=z_dim)

    def forward(self, x):
        """
        Inputs:
            x - Batch of input images. Shape: [B,C,H,W]
        Outputs:
            recon_x - Reconstructed image of shape [B,C,H,W]
            z - Batch of latent codes. Shape: [B,z_dim]
        """
        z = self.encoder(x)
        recon_x = self.decoder(z)

        return recon_x, z

    def get_loss_autoencoder(self, x, recon_x, z_fake, lambda_=1):
        """
        Inputs:
            x - Batch of input images. Shape: [B,C,H,W]
            recon_x - Reconstructed image of shape [B,C,H,W]
            z_fake - Batch of latent codes for fake samples. Shape: [B,z_dim]
            lambda_ - The reconstruction coefficient (between 0 and 1).

        Outputs:
            recon_loss - The MSE reconstruction loss between actual input and its reconstructed version.
            gen_loss - The Generator loss for fake latent codes extracted from input.
            ae_loss - The combined adversarial and reconstruction loss for AAE
                lambda_ * reconstruction loss + (1 - lambda_) * adversarial loss
        """
        recon_loss = nn.MSELoss()(recon_x, x)
        _ , disc_logging_dict = self.get_loss_discriminator(z_fake)
        gen_loss = disc_logging_dict['loss_fake']
        ae_loss = lambda_ * recon_loss + (1 - lambda_) * gen_loss
        # logging
        ae_logging_dict = {'recon_loss': recon_loss.item(),
                           'gen_loss': gen_loss.item(),
                           'ae_loss': ae_loss.item()}

        return ae_loss, ae_logging_dict

    def get_loss_discriminator(self,  z_fake):
        """
        Inputs:
            z_fake - Batch of latent codes for fake samples. Shape: [B,z_dim]

        Outputs:
            disc_loss - The discriminator loss for real and fake latent codes.
            logging_dict - A dictionary for logging the model performance by following keys:
                disc_loss - The discriminator loss for real and fake latent codes.
                loss_real - The discriminator loss for latent codes sampled from the standard Gaussian prior.
                loss_fake - The discriminator loss for latent codes extracted by encoder from input
                accuracy - The accuracy of the discriminator for both real and fake samples.
        """
        z_real = torch.randn_like(z_fake)
        preds_disc_real = self.discriminator(z_real)
        preds_disc_fake = self.discriminator(z_fake)
        loss_real = nn.BCEWithLogitsLoss()(preds_disc_real, torch.ones_like(preds_disc_real))
        loss_fake = nn.BCEWithLogitsLoss()(preds_disc_fake, torch.zeros_like(preds_disc_fake))
        disc_loss = (loss_real + loss_fake) / 2
        acc = (torch.sum(self.discriminator(z_real) > 0) + torch.sum(self.discriminator(z_fake) < 0)) / (z_real.shape[0] * 2)
        # logging
        logging_dict = {'disc_loss': disc_loss,
                        'loss_real': loss_real,
                        'loss_fake': loss_fake,
                        'accuracy': acc}

        return disc_loss, logging_dict

    @torch.no_grad()
    def sample(self, batch_size):
        """
        Function for sampling a new batch of random or conditioned images from the generator.
        Inputs:
            batch_size - Number of images to generate
        Outputs:
            x - Generated images of shape [B,C,H,W]
        """
        z = torch.randn(batch_size, self.z_dim, device=self.device)
        x = self.decoder(z)

        return x

    @property
    def device(self):
        """
        Property function to get the device on which the generator is
        """
        return self.encoder.device


#!/usr/bin/env python
# -*- coding:  utf-8 -*-
"""a collection tools for hd5 stock database

    HD5 file structure (data_xx.h5)
    /(RootGroup) 'xx A share'
    /SPLITS (Group) 'splits & dividends'
    /SPLITS /code (Table) ' '
        -> (time,sd, ss, ssp, cd)
        here, sd: stock dividends, ss: stock splits
              ssp: stock splits price, cd: cash dividends
    /DAILY (Group) 'daily quote'
    /DAILY /code (Table) 'name'
        -> (time, open, high, low, close, volume, sum)
    /MIN5 (Group) '5-minute quote'
    /MIN5 /code (Table) 'name'
        -> (time, open, high, low, close, volume, sum)
"""
# ============================================================================
# imports
# ============================================================================

import sys
import datetime as dt
import os
import logging
import cStringIO

import numpy as np
import tables as tb
import plac

from tools import *
import fxj_parser as fpar

__status__ = "prototype"

# ============================================================================
# defination
# ============================================================================
class DescQuote(tb.IsDescription):
    time  = tb.Time32Col(pos=0)      # Seconds since 1970.1.1): integer
    open  = tb.Float32Col(pos=1)     # open price: float
    high  = tb.Float32Col(pos=2)     # highest price: float
    low   = tb.Float32Col(pos=3)     # lowest price: float
    close = tb.Float32Col(pos=4)     # close price: float
    vol   = tb.UInt32Col(pos=5)      # volumn(100 share):
    sum   = tb.Float32Col(pos=6)     # sum: float

class DescSPLITS(tb.IsDescription):
    """ struct: date, sending ratio(free), sending ratio (charged),
        sending price, dividend
    """
    time  = tb.Time32Col(pos=0)      # seconds ince 1970.1.1): integer
    sd    = tb.Float32Col(pos=1)     # songgu ratio: float
    ss    = tb.Float32Col(pos=2)     # peigu ratio: float
    ssp   = tb.Float32Col(pos=3)     # peigu ratio: float
    cd    = tb.Float32Col(pos=4)     # dividend: float

class SymbolTable(tb.IsDescription):
    code   = tb.StringCol(8,pos=0)    # code('SH600000')
    name   = tb.StringCol(8,pos=1)    # name('PuFaYinHang')

MARKETS=['SH','SZ']
TYPE_DAILY='DAILY'
TYPE_MIN5 ='MIN5'
TYPE_SPLITS ='SPLITS'
TYPE_HD5 = [TYPE_DAILY,TYPE_MIN5,TYPE_SPLITS]
GRP_DAILY='/DAILY'
GRP_MIN5='/MIN5'
GRP_SPLITS='/SPLITS'
TYPE2GRP = {TYPE_DAILY:'/DAILY',
           TYPE_MIN5:'/MIN5',
           TYPE_SPLITS:'/SPLITS'}
TYPE2DESC = {TYPE_DAILY:DescQuote,
           TYPE_MIN5:DescQuote,
           TYPE_SPLITS:DescSPLITS}
TYPE2EXPECTEDROW = {TYPE_DAILY: 10000,
           TYPE_MIN5:10000,
           TYPE_SPLITS:200}
MK_SH = 'SH'
MK_SZ = 'SZ'
MK_INDEX={'SH':'SH000001','SZ':'SZ399001'}

DAD_URL = 'http://www.000562.com/fxjdata/'
# ============================================================================
# functions
# ============================================================================

# ============================================================================
# Logging
# ============================================================================
# init logging
log = logging.getLogger('quote_hd5')
log.setLevel(logging.INFO)
# define a handler which writes INFO messages or higher to sys.stderr
console = logging.StreamHandler()
#console.setLevel(logging.DEBUG)

# set a format which is simpler for console use
# tell the handler to use this format
console.setFormatter(
        logging.Formatter('%(levelname)-8s %(message)s') )
# add the handler to the log
log.addHandler(console)

# ============================================================================
# HD5 Class
# ============================================================================

class QuoteHD5():
    """ class of quote hd5 DB
    TODO: use external link to combin two HD5s together for convenience
    """
    def __init__(self, hd5_path):
        self._hd5fp = {}
        self._last_update={}
        if os.path.isdir(hd5_path):
            hd5_fn= [ os.path.join(hd5_path,'data_sh.h5'),
                    os.path.join(hd5_path,'data_sz.h5')]
        else:
            hd5_fn=[hd5_path]

        for  fn in hd5_fn:
            if os.path.exists(fn):
                fp = tb.openFile(fn,
                                mode='r+',
                                rootuep="/",
                                nodecachesize =1024)
                mk = fp.title[:2]
                if mk in MARKETS:
                    self._hd5fp[mk] = fp
                    self._last_update[mk] = fp.root._v_attrs.LAST_UPDATE
                    log.info('open hd5 file %s' %(fn))
                else:
                    log.error('Hd5 file title incorrect: %s' %fp.title)
                    fp.close()
            else:
                log.error('file not exists: %s' %(fn))

    def __repr__(self):
        tmp=''
        for mk in MARKETS:
            fp = self._hd5fp.get(mk)
            if fp is not None:
                tmp += fp.__str__()
        return tmp

    def close(self):
        for fp in self._hd5fp.values():
            log.info('close hd5 file %s' %(fp.filename))
            fp.close()

    def get_lastupdate(self):
        rst =[]
        for mk in MARKETS:
            fp = self._hd5fp.get(mk)
            if fp is None: continue
            rst.append('last update date of %s:' %fp.filename)
            for grp in fp.walkGroups():
                tmp = '{0:7}: {1}'.format(grp._v_name, grp._v_attrs.LAST_UPDATE)
                rst.append(tmp)
        print('\n'.join(rst))

    def _createHD5(self,name,title='',cache=1024):
        """ create new hd5
        """
        filters = tb.Filters(complevel=1,
                         complib='lzo',
                         shuffle=True,
                         fletcher32=False)

        hd5_fp = tb.openFile(name,
                          mode="w",
                          title= title,
                          rootUEP="/",
                          filters =filters,
                          nodeCacheSize = cache)
        hd5_fp.root._v_attrs.LAST_UPDATE = ''

        dst_daily = hd5_fp.createGroup(hd5_fp.root,
                            'DAILY', title='daily quote')
        dst_daily._v_attrs.LAST_UPDATE = ''
        dst_5min = hd5_fp.createGroup(hd5_fp.root,
                            'MIN5', title='5-minute quote')
        dst_5min._v_attrs.LAST_UPDATE = ''
        dst_splits = hd5_fp.createGroup(hd5_fp.root,
                            'SPLITS', title='splits & dividends')
        dst_splits._v_attrs.LAST_UPDATE = ''

        return hd5_fp

    def get_lostdate(self, code=None):
        """check lost date of 5min DB in given hd5 database
        Note: 5min DB starts from 20051103
        SH index: SH000001, SZ index: SZ399001

        @hd5: hd5 file name
        @code: stock to be checked. if is '', use index of SH or SZ market

        return a list of missing date
        """

        if code is None:
            code = MK_INDEX[MK_SH]
        else:
            code = code.upper()
            #code = 'SZ399001'

        #tbl = fp_hd5.root.gMinute._f_getChild(code)
        mk = code[:2]
        try:
            tbl = self._hd5fp[mk].getNode(GRP_MIN5, name=code)
        except (NoSuchNodeError, NameError):
            log.debug('code % not available in DB' %code)
            return None

        rec_days = set(map(dtnum2str,tbl[:]['time']))
        work_days = get_workingdays(
                        dtnum2str(tbl[0][0]),dtnum2str(tbl[-1][0]))
        lost_date = [x for x in work_days if x not in rec_days]
        self.lost_date = lost_date
        return lost_date

    def _append_quote(self,
                        data_type,
                        data_source,
                        checkorder=True):
        """append data into hd5 file

        @data_type: data type of data source, 'DAILY', 'MIN5', 'SPLITS'
        @dad_source: data to be appended, iteratable, structure likes
            [code,numarray]
        @checkorder: if True, only allow to append data according to
                   time order. if False, can append w/o order, means
                   later appended data can be earlier than existed data.
                   useed to
        """

        count=[0,0]

        for code, data in data_source:
            count[0] += 1
            code = code.upper()
            mk=code[:2]
            if mk not in MARKETS:
                log.error('undefined code prefix %s' %code)
                continue
            fp = self._hd5fp[mk]

            grp = fp.getNode(TYPE2GRP[data_type])
            lupdate = grp._v_attrs.LAST_UPDATE
            start = dtstr2num(lupdate) if lupdate else 0

            if checkorder:
                rows = data[np.where(data['time'] >= start)]
                if len(rows) < 1:
                    log.debug(
                            '%s has no data later than %s' %(code, lupdate))
                    continue
            else:
                rows = data
                #TODO: process for data sanity check

            try:
                tbl = fp.getNode(TYPE2GRP[data_type], name=code)
            except tb.NoSuchNodeError:
                log.debug('create new table for %s' %code)
                tbl = fp.createTable(
                           TYPE2GRP[data_type],
                           code,
                           TYPE2DESC[data_type],
                           expectedrows = TYPE2EXPECTEDROW[data_type])
            tbl.append(rows)
            tbl.flush()
            count[1] += 1

            # update last_update
            if code in MK_INDEX.values():
                tmp = dtnum2str(rows[-1][0])
                tbl._v_parent._v_attrs.LAST_UPDATE = tmp
                fp.root._v_attrs.LAST_UPDATE = tmp

        if data_type == TYPE_SPLITS:
            tmp=dt.datetime.strftime(dt.date.today(),'%Y%m%d')
            for mk in MARKETS:
                self._hd5fp[mk].root.SPLITS._v_attrs.LAST_UPDATE = tmp

        log.info('%s of total %s stocks appended into DB' %(
                                                    count[1],count[0]))

    def update_hd5(self, url_path=DAD_URL):
        """update hd5 to date with dads from web

        @url_path: url of dad files located, should end with a '/'
        """
        w_days = get_workingdays(self._last_update[MK_SH])
        for day in w_days[1:]:
            for url_type, data_type in [('.dad',TYPE_DAILY),
                                        ('m.dad',TYPE_MIN5)]:
                url = url_path + day + url_type
                msg = '-'.join([day,data_type.lower()])
                d_tmp = download(url,rep=3)
                if d_tmp is not None:
                    #d = src.read()
                    fp_tmp = cStringIO.StringIO(d_tmp)
                    data_src = fpar.parse_dad(fp_tmp,out_dtfmt=None)
                    if data_src:
                        log.info('update %s' %msg)
                        err = self._append_quote(data_type,data_src)
                        data_src.close()
                    else:
                        log.error('parse %s failed' %msg)
                    fp_tmp.close()
                else:
                    log.error('download %s failed' %msg)

        # update splits data
        url = url_path + 'SPLIT.PWR'
        d_tmp = download(url,rep=3)
        if d_tmp is not None:
            fp_tmp = cStringIO.StringIO(d_tmp)
            data_src = fpar.parse_pwr(fp_tmp, out_dtfmt=None)
            if data_src:
                log.info('update splits')
                err = self._append_quote(TYPE_SPLITS,data_src)
                data_src.close()
            fp_tmp.close()
        else:
            log.error('down splits error')

    def update_hd5_local(self,dad_path):
        """update hd5 DB with local dads

        @dad_path: local path of dad files
        """
        w_days = get_workingdays(self._last_update[MK_SH])
        for day in w_days[1:]:
            dad_daily = os.path.join(dad_path,day+'.dad')
            dad_min = os.path.join(dad_path,day+'m.dad')
            for fn, data_type in [(dad_daily,TYPE_DAILY),
                            (dad_min,TYPE_MIN5)]:

                data_source = fpar.iter_parser(fn, out_dtfmt=None)
                if data_source:
                    log.info('update %s' %fn)
                    err = self._append_quote(data_type,data_source)
                    data_source.close()
                else:
                    log.error('source file error %s' %fn)

        # update splits data
        splits = os.path.join(dad_path,'split.pwr')
        data_source = fpar.iter_parser(splits, out_dtfmt=None)
        if data_source:
            log.info('update %s' %splits)
            err = self._append_quote(TYPE_SPLITS,data_source)
            data_source.close()
        else:
            log.error('Source file error %s' %fn)

    def sort_hd5(self):
        """sort hd5 files
        """
        for fp in self._hd5fp.values():
            log.debug('Start sort %s' %fp.name )
            for node in fp.walkNodes():
                if isinstance(node,tb.table.Table):
                    rows = node.read()
                    rows.sort(order='time')
                    node.modifyRows(start=0,stop=-1,rows=rows)
            log.debug('Sort finished')

    def _dump(self,code,db_type):
        """dump all quote from HD5 and return numpy recarray

        @code: stock code, 'SH510051'
        @db_type: dialy or 5min data

        return: a numpy recarray if success, otherwise, None
        """
        code = code.upper()
        mk= code[:2]
        if mk not in MARKETS:
            log.error('incorrect code name: %s' %code)
            return None
        fp = self._hd5fp[mk]
        try:
            tbl = fp.getNode(TYPE2GRP[db_type], name=code)
            rows = tbl.read()
            return rows
        except tb.NoSuchNodeError:
            log.error('code not exist: %s' %code)
            return None

    def get_daily(self,code):
        """get daily quote of given code
        @code: stock code, link 'sh510050'
        return: a numpy recarry if success
        """
        return self._dump(code,TYPE_DAILY)

    def get_min5(self,code):
        """get 5min quote of given code
        @code: stock code, link 'sh510050'
        return: a numpy recarry if success
        """
        return self._dump(code,TYPE_MIN5)

    def extract(self,code_lst, fn,title=''):
        """extract some codes to a new hd5 file

        @fn: new hd5 file name
        @title: title of new hd5, should startswith 'SH' or 'SZ'
        @code_lst: code to be copied, if [], copy all
        """
        if len(code_lst)<1:
            log.warning('no input code')
            return
        if title=='':
            title=','.join(code_lst).upper()
        log.debug('create new hd5 db: %s' %fn)
        dst_fp = self._createHD5(name=fn,title=title)
        count=[0,0]
        for code in code_lst:
            code = code.upper()
            if (code[:2] not in MARKETS) or len(code) != 8:
                log.error('invalide code %s' %code)
                continue
            count[0] += 1
            src_fp = self._hd5fp[code[:2]]
            for grp in TYPE2GRP.values():
                try:
                    dst_grp = dst_fp.getNode(grp)
                    src_tbl = src_fp.getNode(grp,name=code)
                    src_tbl.copy(dst_grp,src_tbl.name,overwrite=True)
                    ldp = src_grp._v_attrs.LAST_UPDATE
                    dst_grp._v_attrs.LAST_UPDATE = ldp
                    dst_grp._v_parent._v_attrs.LAST_UPDATE = ldp
                    log.debug('copy %s: %s -> %s' %(code,grp,grp))
                    count[1] += 1
                except tb.NoSuchNodeError:
                    log.warning('%s not exist in src %s ' %(code,grp))
                    continue
                except:
                    (type, value, traceback) = sys.exc_info()
                    log.error('unexpected %s: %s' %(type,value))
                    continue
        dst_fp.close()
        log.info('%s tables of %s code extracted to %s' %(
                                                count[1],count[0],fn))
        return
# ============================================================================
# main
# ============================================================================
@plac.annotations(
    hd5path=("hd5 DB path", 'positional', None),
    update_l=("update DB from local dad path",'option','U',str,None,"DATAPATH"),
    update_r=("update DB from remote",'flag','u'),
    extract=("extract codes to a new hd5 DB, split code by ','",
                                    'option','e',str,None,"CODES"),
    outfn=("output DB name",'option','o',str,None,"OUTPUT"),
    debug=("debug mode",'flag','d'),
    lastupdate=("print last update date",'flag','l'),
    sort=("sort DB by time order",'flag','s'),
    show=("show DB information",'flag','w'),
        )
def main(hd5path,update_l,update_r,extract,
         outfn,debug,lastupdate,sort,show):

    # Get the log
    log = logging.getLogger('quote_hd5')

    if debug:
        log.setLevel(logging.DEBUG)

    # create file handler which logs even debug messages
    if os.path.isdir(hd5path):
        log_fn=os.path.join(hd5path,'data_h5.log')
    elif os.path.isfile(hd5path):
        base,ext = os.path.splitext(hd5path)
        log_fn=base+'.log'
    else:
        print('wrong hd5 file name')
        return

    fh = logging.FileHandler(log_fn,'a')
    fmt = '%(asctime)s %(name)-12s: %(levelname)-8s %(message)s'
    fh.setFormatter(logging.Formatter(fmt))
    log.addHandler(fh)

    if update_r:
        db=QuoteHD5(hd5path)
        db.update_hd5()
        db.close()
        return
    if update_l:
        db=QuoteHD5(hd5path)
        db.update_hd5_local(update_l)
        db.close()
        return
    if lastupdate:
        db=QuoteHD5(hd5path)
        db.get_lastupdate()
        db.close()
        return
    if extract:
        codes =extract.split(',')
        fn = outfn if outfn else codes[0]+'.h5'
        db=QuoteHD5(hd5path)
        db.extract(codes,fn)
        db.close()
        return
    if show:
        db=QuoteHD5(hd5path)
        print(db)
        db.close()
        return

if __name__ == "__main__":
    plac.call(main)


import pickle
from pprint import pprint
from typing import Dict, List

from src.utils.helper import Helper

def main():
    commands_dict: Dict[str, Dict[str, List[str]]] = dict()
    command_types = ['addressed', 'if', 'io', 'unaddressed']

    dict_types = ['code', 'description', 'mnemonic', 'nzvc']

    for command_type in command_types:
        if command_type not in commands_dict.keys():
            commands_dict[command_type] = dict()

        for dict_type in dict_types:
            with open(f'{Helper.get_project_root()}/commands/{command_type}/{dict_type}.txt') as file:
                for line_raw in file.read().splitlines():
                    line = line_raw.strip().replace('  ', ' ')

                    if dict_type not in commands_dict[command_type].keys():
                        commands_dict[command_type][dict_type] = []
                    commands_dict[command_type][dict_type].append(line)

    with open(f'{Helper.get_project_root()}/commands_dict.pickle', 'wb+') as file:
        pickle.dump(commands_dict, file)
    pprint(commands_dict)

if __name__ == '__main__':
    main()


import os
from google.cloud import storage
from termcolor import colored
from bulber.params import MODEL_VERSION

def storage_upload(model_version=MODEL_VERSION, bucket=BUCKET_NAME, rm=False):
    client = storage.Client().bucket(bucket)
    storage_location = 'models/{}/versions/{}/{}'.format(
        MODEL_NAME,
        model_version,
        'model.bulber')
    blob = client.blob(storage_location)
    blob.upload_from_filename('bulber_v2')
    print("=> model.bumbulb uploaded to bucket {} inside {}".format(BUCKET_NAME, storage_location))
    if rm:
        os.remove('model.bulber')


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tree_tensor_network

if __name__ == "__main__":
  backend = "jax"  # "numpy" and "tensorflow" are also supported!

  if backend == "tensorflow":
    import tensorflow as tf
    dtype = tf.float64
  elif backend == "jax":
    from jax.config import config
    config.update("jax_enable_x64", True)
    import jax.numpy as np
    dtype = np.float64
  elif backend == "numpy":
    import numpy as np
    dtype = np.float64

  tree_tensor_network.set_backend(backend)

  num_layers = 6
  max_bond_dim = 16
  build_graphs = True

  num_sweeps = 1000

  Ds = [min(2**i, max_bond_dim) for i in range(1, num_layers + 1)]

  print("----------------------------------------------------")
  print("Variational ground state optimization.")
  print("----------------------------------------------------")
  print("System size:", 2**num_layers)
  print("Bond dimensions:", Ds)

  H = tree_tensor_network.get_ham_ising(dtype)
  isos_012 = tree_tensor_network.random_tree_tn_uniform(Ds, dtype, top_rank=1)

  isos_012 = tree_tensor_network.opt_tree_energy(
      isos_012,
      H,
      num_sweeps,
      1,
      verbose=1,
      graphed=build_graphs,
      ham_shift=0.2)


from django.urls import path,include
from rest_framework_jwt.views import obtain_jwt_token, refresh_jwt_token
from .views import *
from drf_yasg.views import get_schema_view
from drf_yasg import openapi
from rest_framework import permissions
from .views import *
schema_view = get_schema_view(
   openapi.Info(
      title="WESCO LOTTOHUB",
      default_version='v1',
      description="description",
      terms_of_service="https://www.google.com/policies/terms/",
      contact=openapi.Contact(email="gitech@gmail.com"),
      license=openapi.License(name="BSD License"),
   ),
   public=True,
   permission_classes=(permissions.AllowAny,),
)

urlpatterns = [
    path('api-token-auth/', obtain_jwt_token),
    path('api-refresh-token/', refresh_jwt_token),
    path('getgame/', getgameAPIview.as_view()),
    path('cancelticket/', cancelticketviewset.as_view()),
    path('checkwinner/', checkwinnerviewset.as_view()),
    path('ticketstatus/', ticketstatusviewset.as_view()),
    path('updatewinner/', Updatewinnerviewset.as_view()),
    path('sellticket/', sellticketviewset.as_view()),
    path('winnerlist/', winnerlistviewset.as_view()),
    path('swagger/', schema_view.with_ui('swagger')),
    path('swaggerdocs/', schema_view.with_ui('redoc')),
]


from Settings import *
from pix_mapper_pics import *
import re
import glob
import os
import pyautogui as gui
import time
from time import sleep

screen_center = (gui.size()[0] / 2, (gui.size()[1] / 2) - 50)

#job = ['J8804', 'CLT', 'landrover', 'Pix4D']

def start():
    os.startfile(pix_mapper_path)
    time.sleep(15)

def new_project(job):
    gui.hotkey('ctrl','n')
    gui.write(job[0] + '_' + job[2])
    gui.press('tab', presses=3)
    gui.press('enter')
    time.sleep(1)
    gui.press('tab')
    gui.press('enter')
    gui.press('tab', presses=5)
    gui.press('enter')

def load_pics(job):
    gui.write(new_job_folder + '\\' + job[0] + scan_folder + r'\filtered')
    gui.press('enter')
    time.sleep(1)
    gui.press('tab')
    time.sleep(1)
    gui.press('tab')
    time.sleep(1)
    gui.press('tab')
    time.sleep(1)
    gui.press('tab')
    gui.hotkey('ctrl','a')
    gui.press('enter')
    time.sleep(1)
    gui.press('enter')
    time.sleep(5)
    gui.press('tab',presses=5)
    time.sleep(5)
    gui.press('enter')
    time.sleep(1)
    gui.press('enter')
    time.sleep(7)
    if job[2] != 'site':
        gui.press('down')
    else:
        pass
    gui.press('enter')

def start_processing():
    check_for_image(pixDSMOrthoIndex)
    gui.click()
    check_for_image(pixMapperStart)
    gui.click()

def close_pix():
    check_for_image(pixDone)
    gui.hotkey('alt','f4')

def check_for_image(image):
    checking = True
    while checking:
        image_location = gui.locateCenterOnScreen(image)
        gui.moveTo(image_location)
        if gui.position() == image_location:
            checking = False


from osgeo import ogr
import shapefile
from uafgi import shputil

# Read a shapefile of polygons, compute area of each.  This can be
# used to find the most-retreated terminus polygon, which should have
# the largest area.

fname_closed = 'data/calfin/domain-termini-closed/termini_1972-2019_Jakobshavn-Isbrae_closed_v1.0.shp'
fname_term = 'data/calfin/domain-termini/termini_1972-2019_Jakobshavn-Isbrae_v1.0.shp'

# with shapefile.Reader(fname) as sf:
#     for i in range(0, len(sf)):
#         shape = sf.polygon(i)
#         print(type(shape))
#         if shape.shapeType != shapefile.POLYGON:
#             raise ValueError('shapefile.POLYGON shapeType expected in file {}'.format(self.fname))
#         print(shape.__dict__)
#         print('{},{}'.format(i,shape.area))

dataSource = ogr.GetDriverByName("ESRI Shapefile").Open(fname_closed)
layer = dataSource.GetLayer()

schema = []
ldefn = layer.GetLayerDefn()
for n in range(ldefn.GetFieldCount()):
    fdefn = ldefn.GetFieldDefn(n)
    schema.append(fdefn.name)
print(schema)

polygons = list()
for id,feature in enumerate(layer):
    geom = feature.GetGeometryRef()
    area = geom.GetArea()
    attrs = [-area, id+1]
    attrs += [feature.GetFieldAsString(fd) for fd in ('Date', 'ImageID')]
    polygons.append(attrs)

polygons.sort()
id = polygons[0][1]
image_id0 = polygons[0][3]
print(polygons[0])

print(id)
shputil.select_feature(fname_closed, id, 'poly.shp')

#print(polygons[:5])
# ---------------------------------------------------

dataSource = ogr.GetDriverByName("ESRI Shapefile").Open(fname_term)
layer = dataSource.GetLayer()
for id,feature in enumerate(layer):
    image_id = feature.GetFieldAsString('ImageID')
    if image_id == image_id0:
        print(id, image_id)
        break


from django.db import models
from django.conf import settings
import datetime
from django.core.validators import MaxValueValidator, MinValueValidator
from django.utils.translation import gettext as _
from django.db.models.signals import post_save, pre_save

User = settings.AUTH_USER_MODEL

def current_year():
    return datetime.date.today().year

def max_value_current_year(value):
    return MaxValueValidator(current_year() + 11)(value)

ENROLLMENT_STATUS = (
    ('applicant','Applicant'),
    ('current student', 'Current student'),
    ('alumni','Alumni'),
)

class MyUniversity(models.Model):
    user                = models.ForeignKey(User, null=True, blank=True, on_delete=models.CASCADE, related_name="university")
    country             = models.CharField(max_length=256, null=True, blank=True)
    uni_name            = models.CharField(max_length=256, null=True, blank=True)
    degree              = models.CharField(max_length=256, null=True, blank=True)
    field_of_study      = models.CharField(max_length=256, null=True, blank=True)
    st_year             = models.IntegerField(_('year'), null=True, blank=True, validators=[MinValueValidator(1970), max_value_current_year],default=current_year)
    en_year             = models.IntegerField(_('en_year'), null=True, blank=True, validators=[MinValueValidator(1970), max_value_current_year],default=current_year)
    grade               = models.CharField(max_length=256, null=True, blank=True)
    # grade_rate          = models.IntegerField(default=60, null=True, blank=True)
    # foundation_body     = models.CharField(max_length=256, null=True, blank=True)
    # foundation_payback  = models.BooleanField(default=True)
    # foundation_amount   = models.CharField(max_length=256, null=True, blank=True)
    # description         = models.CharField(max_length=1000, null=True, blank=True)
    enr_status          = models.CharField(max_length=64, choices=ENROLLMENT_STATUS)

    def __str__(self):
        return f"{self.id} {str(self.user)} {self.uni_name}"

def pre_save_userprofile_receiver(sender, instance, *args, **kwargs):
    en_year = instance.en_year
    st_year  = instance.st_year
    if en_year < current_year():
        instance.enr_status = "alumni"
    elif st_year > current_year():
        instance.enr_status = "applicant"
    else:
        instance.enr_status = "current student"

pre_save.connect(pre_save_userprofile_receiver, sender=MyUniversity)


from __future__ import with_statement  # (Py2.5 needs this)

from functools import wraps
from inspect import getmro, isfunction

__all__ = (
    'ClassNameConflictError',
    'aux', 'primary',
    'AutoAuxBase', 'AutoAuxMeta',
)

#
# exceptions

class ClassNameConflictError(Exception):
    """
    Conflict: class names are identical after stripping leading underscores.
    """
    def __str__(self):
        cls1, cls2 = self.args
        return (
            'Class names: %r and %r -- are identical after stripping leading '
            'underscores, which is forbidden when using aux/primary methods.'
            % (cls1.__name__, cls2.__name__))

#
# non-public stuff

_SUFFIXES = '_primary', '_before', '_after', '_around'

class _WrappedMethodPlaceholder(object):

    def __init__(self, func):
        self.func = func

    def __call__(self, *args, **kwargs):
        raise TypeError('method placeholder is not callable '
                        '(forgot to apply aux() class decorator?)')

def _next_around(obj_around, self, basename, *args, **kwargs):
    # try to get and call next `around` aux method
    meth_around = getattr(obj_around, basename + '_around', None)
    if meth_around is not None:
        return meth_around(*args, **kwargs)
    else:
        # if there is no more `around` methods, get and call:
        # `before` aux method (it can call superclasses' `before` methods)
        meth_before = getattr(self, basename + '_before', None)
        if meth_before is not None:
            meth_before(*args, **kwargs)
        # primary method (it can call superclasses' primary methods)
        meth_primary = getattr(self, basename + '_primary')
        pri_result = meth_primary(*args, **kwargs)
        # `after` aux method (it can call superclasses' `after` methods)
        meth_after = getattr(self, basename + '_after', None)
        if meth_after is not None:
            meth_after(*args, **kwargs)
        return pri_result

def _provide_wrapper(cls, func, basename):
    @wraps(func)
    def wrapper(self, *args, **kwargs):
        return _next_around(self, self, basename, *args, **kwargs)
    added_doc = '(See: %s%s() signature).' % (basename, '_primary')
    existing_doc = (getattr(wrapper, '__doc__', None) or '').rstrip()
    if existing_doc:
        wrapper.__doc__ = '%s\n\n%s' % (existing_doc, added_doc)
    else:
        wrapper.__doc__ = added_doc
    setattr(cls, basename, wrapper)

def _provide_primary(cls, func, basename):
    suffixed_name = basename + '_primary'
    func.__name__ = suffixed_name
    func.__doc__ = (
        'The actual method implementation '
        '(%s() is only a wrapper).' % basename)
    setattr(cls, suffixed_name, func)

def _provide_wrapped_primary(cls, func):
    basename = func.__name__
    _provide_wrapper(cls, func, basename)
    _provide_primary(cls, func, basename)

def _strip_and_check_cls_name(cls):
    cls_stripped_name = cls.__name__.lstrip('_')
    for supercls in getmro(cls):
        if (supercls is not cls and
              cls_stripped_name == supercls.__name__.lstrip('_')):
            raise ClassNameConflictError(supercls, cls)
    return cls_stripped_name

def _provide_call_next(cls, suffixed_name):
    cls_stripped_name = _strip_and_check_cls_name(cls)
    basename, qualifier = suffixed_name.rsplit('_', 1)
    cn_name = '_%s__%s' % (
        cls_stripped_name,
        (basename if qualifier == 'primary' else suffixed_name))
    if cn_name in vars(cls):
        return
    if qualifier == 'around':
        def call_next(self, *args, **kwargs):
            return _next_around(
                super(cls, self), self, basename, *args, **kwargs)
    else:
        def call_next(self, *args, **kwargs):
            super_meth = getattr(super(cls, self), suffixed_name, None)
            if super_meth is not None:
                return super_meth(*args, **kwargs)
    call_next.__name__ = cn_name
    setattr(cls, cn_name, call_next)

#
# actual decorators

def aux(cls):
    """Class decorator (for classes containing primary and/or aux methods)."""
    if not isinstance(cls, type):
        raise TypeError('%r is not a type' % cls)
    # wrap/rename primary methods
    for name, obj in tuple(vars(cls).items()):  # (Py2.x/3.x-compatibile way)
        if isinstance(obj, _WrappedMethodPlaceholder):
            _provide_wrapped_primary(cls, obj.func)
    # provide `call-next-method`-like methods
    for name, obj in tuple(vars(cls).items()):
        if isfunction(obj) and obj.__name__.endswith(_SUFFIXES):
            _provide_call_next(cls, obj.__name__)
    return cls

def primary(func):
    """Method decorator (for primary methods only)."""
    if not isfunction(func):
        raise TypeError('%r is not a function' % func)
    return _WrappedMethodPlaceholder(func)

#
# convenience classes (any of them can be used *optionally*...)

class AutoAuxMeta(type):
    """Convenience metaclass: `aux()`-decorates classes created by it."""
    def __new__(mcs, name, bases, attr_dict):
        return aux(type.__new__(mcs, name, bases, attr_dict))

# (here: Py2.x/3.x-compatibile way to create a class with a custom metaclass)
AutoAuxBase = AutoAuxMeta('AutoAuxBase', (object,), {'__doc__':
    """`AutoAuxMeta`-created base class: `aux()`-decorates its subclasses."""})

#
# basic example

if __name__ == '__main__':

    import sys
    import time

    class TimedAction(AutoAuxBase):
        # note: AutoAuxBase automatically decorates your classes with aux()

        def action_before(self, *args, **kwargs):
            """Start action timer."""
            print('starting action timer...')
            self.start_time = time.time()

        def action_after(self, *args, **kwargs):
            """Stop action timer and report measured duration."""
            self.action_duration = time.time() - self.start_time
            print('action duration: %f' % self.action_duration)

    class FileContentAction(AutoAuxBase):

        def action_around(self, path):
            """Read file and pass its content on; report success or error."""
            print('opening file %r...' % path)
            try:
                with open(path) as f:
                    content = f.read()
            except EnvironmentError:
                print(sys.exc_info()[1])
            else:
                result = self.__action_around(path, content)
                print('file %r processed successfully' % path)
                return result

    class NewlinesCounter(FileContentAction, TimedAction):

        item_descr = 'newlines'

        @primary
        def action(self, path, content):
            """Get number of newlines in a given string."""
            return content.count('\n')

        def action_before(self, path, *args):
            """Print a message and go on..."""
            print('counting %s in file %r will start...' % (
                self.item_descr, path))
            self.__action_before(path, *args)

        def action_around(self, path):
            """Start operation with given file path. Finally, show summary."""
            result = self.__action_around(path)
            if result is not None:
                print('%s in file %r: %s\n' % (
                    self.item_descr, path, result))
            else:
                print('could not count %s in file %r\n' % (
                    self.item_descr, path))
            return result

    class SpacesAndNewlinesCounter(NewlinesCounter):

        item_descr = 'spaces and newlines'

        @primary
        def action(self, path, content):
            """Get number of spaces and newlines in a given string."""
            spaces = content.count(' ')
            newlines = self.__action(path, content)
            return spaces + newlines

    example_file_paths = __file__, 'spam/spam/spam/non-existent'

    nl_counter = NewlinesCounter()
    spc_nl_counter = SpacesAndNewlinesCounter()

    for path in example_file_paths:
        nl_counter.action(path)
        spc_nl_counter.action(path)


from selenium import webdriver
import time

#다운받은 웹드라이버를 통해 Chrome을 켜겠다. driver는 켜진 웹드라이버를 가르킴
#webdriver.Chrome("드라이버 위치") : 드라이버 변수 생성, 크롬을 켠다.
from selenium.common.exceptions import NoSuchElementException

driver = webdriver.Chrome("./chromedriver.exe")

#url에 접속
driver.get("https://v4.map.naver.com/")

#네이버 지도 업데이트 안내끄기
driver.find_elements_by_css_selector("button.btn_close")[1].click()

#검색창에 검색어 입력하기
search_box = driver.find_element_by_css_selector("input#search-input")
search_box.send_keys("피자")

# 검색버튼 누르기 // 검색버튼: button.spm
search_button = driver.find_element_by_css_selector("button.spm")
search_button.click()

time.sleep(1)

stores = driver.find_elements_by_css_selector("div.lsnx")

for n in range(1, 20):
    time.sleep(1)

    stores = driver.find_elements_by_css_selector("div.lsnx")

    for store in stores:
        try:
            phone = store.find_element_by_css_selector("dd.tel").text
        except NoSuchElementException as e:
            phone = "전화번호 없음"
        name = store.find_element_by_css_selector("dt > a").text
        addr = store.find_element_by_css_selector("dd.addr").text

        print("="*50)
        print("가게명:", name)
        print("주소:", addr)
        print("전화번호:", phone)

    page_bar = driver.find_elements_by_css_selector("div.paginate > *")

    try:
        if n%5 != 0:
            page_bar[n%5+1].click()
        else:
            page_bar[6].click()
    except:
        print("데이터 수집 완료")
        break

#driver.close()


#!/usr/bin/env python
# encoding: utf-8
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import mpl_toolkits.mplot3d.art3d as art3d
from random import randrange

def foo():
    x = randrange(-50, 50)
    y = randrange(-50, 50)
    return (x**2, y**2, x * y)
    return (x**2, y**2, np.sqrt(2) * x * y)

def bar():
    x = randrange(-50, 50)
    y = randrange(-50, 50)
    return (x, y)

def highdim():
    fig = plt.figure()
    data = tuple(
            bar() for i in range(int(1e3/2))
           )
    data = np.array(data)
    idx = np.sum(data[:, :2]**2, 1) > 28**2
    idx = idx & (np.sum(data[:, :2]**2, 1) < 32**2)
    data = data[~idx]
    idx = np.sum(data[:, :2]**2, 1) < 30**2
    data1 = data[idx]
    data2 = data[~idx]
    plt.plot(data2[:, 0], data2[:, 1], 'bo')
    plt.plot(data1[:, 0], data1[:, 1], 'g+')
    c = plt.Circle((0, 0), 30, fill=False)
    plt.gca().add_artist(c)
    plt.savefig('svm_2d.png')

    #data1[:, 2] = np.random.randint(0, 8, data1.shape[0])
    #data2[:, 2] = np.random.randint(12, 20, data2.shape[0])
    data = tuple(
            foo() for i in range(int(1e3/2))
           )
    data = np.array(data)
    idx = np.sum(data[:, :2], 1) > 28**2
    idx = idx & (np.sum(data[:, :2], 1) < 32**2)
    data = data[~idx]
    idx = np.sum(data[:, :2], 1) < 30**2
    data1 = data[idx]
    data2 = data[~idx]
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.scatter(data1[:, 0], data1[:, 1], data1[:, 2], marker='+', c='g')
    ax.scatter(data2[:, 0], data2[:, 1], data2[:, 2], marker='o', c='b')
    #p = plt.Rectangle((0, 0), 100**2, 100**2, fill=False)
    #ax.add_patch(p)
    #art3d.pathpatch_2d_to_3d(p, z=15**2)
    ax.view_init(200, 135)
    plt.savefig('svm_3d.png')

def maxmarg():
    fig = plt.figure()
    data = tuple((randrange(0, 100), randrange(0, 100)) for i in range(int(1e3/2)))
    data = np.array(data)

    idx = data[:, 0] < 30
    idx |= data[:, 0] > 40
    data = data[idx, :]
    idx = data[:, 0] < 30
    data1 = data[idx, :]
    data2 = data[~idx, :]

    plt.plot(data1[:, 0], data1[:, 1], 'g+')
    plt.plot(data2[:, 0], data2[:, 1], 'bo')
    plt.plot((32, 32), (0, 100), 'k-')
    plt.plot((30, 40), (0, 100), 'k-')
    plt.plot((40, 30), (0, 100), 'k-')
    plt.plot((38, 38), (0, 100), 'k-')
    plt.savefig('svm_lin.png')

    fig = plt.figure()
    plt.plot(data1[:, 0], data1[:, 1], 'g+')
    plt.plot(data2[:, 0], data2[:, 1], 'bo')
    plt.plot((35, 35), (0, 100), 'k-')
    plt.savefig('svm_lin_maxmarg.png')

maxmarg()
highdim()
plt.show()