from django.http import HttpResponse, HttpResponseServerError
from django.views.generic import TemplateView, View

import json

from rdflib import Graph, RDFS, URIRef, Literal

from utils import JSONResponseMixin, get_env
from services import FASTService, VIVOService

#Setup a triple store
# from backend import SQLiteBackend
# tstore =SQLiteBackend()
from backend import VivoBackend
ep = get_env('ENDPOINT')
tstore = VivoBackend(ep)

from backend import D, VIVO, RDF
from display import organization, person

class ResourceView(View):
    """
    Handle edits posted by the rich text and tagging widgets..
    """
    def post(self, *args, **kwargs):
        posted = self.request.POST
        edit = json.loads(posted.get('edit'))
        if edit.get('type') == 'ck':
            add_stmts = edit['add']
            add_g = tstore.make_edit_graph(add_stmts)
            subtract_g = tstore.get_subtract_graph(add_stmts)
        elif edit.get('type') in [u'multi-tag']:
            add_stmts = edit.get('add')
            if (add_stmts is not None) and (add_stmts != {}):
                add_g = Graph()
                if add_stmts['object'] == u'new':
                    uri, g = tstore.create_resource(
                        add_stmts['range'],
                        add_stmts['text']
                    )
                    add_g += g
                    add_stmts['object'] = uri
                else:
                    #Make sure we have the text for the added KW
                    obj_uri = URIRef(add_stmts['object'])
                    add_g.add((obj_uri, RDFS.label, Literal(add_stmts['text'])))
                    #Add type.
                    atype = tstore.get_prop_from_abbrv(add_stmts['range'])
                    add_g.add((obj_uri, RDF.type, atype))

                add_g += tstore.make_edit_graph(add_stmts)
            else:
                add_g = Graph()
            remove_stmts = edit.get('subtract')
            if remove_stmts is not None:
                subtract_g = tstore.make_edit_graph(remove_stmts)
            else:
                subtract_g = Graph()
        else:
            return HttpResponseServerError("Edit failed.  Edit type not recognized.")

        done = tstore.add_remove(add_g, subtract_g)
        if done is True:
            return HttpResponse('', 200 )
        else:
            return HttpResponseServerError("Edit failed.")

        return HttpResponseServerError("Edit failed.")

class BasePropertyView(View):

    def get_place_research_areas(self, uri):
        rq = """
        select ?ra ?label
        where {
            ?uri vivo:hasResearchArea ?ra .
            ?ra a schema:Place .
            ?ra rdfs:label ?label .
        }
        """
        out = []
        for row in tstore.graph.query(rq, initBindings={'uri': uri}):
            d = {}
            d['uri'] = row.ra.toPython()
            d['id'] = d['uri']
            d['text'] = row.label.toPython()
            out.append(d)
        return out

class IndexView(TemplateView, ResourceView):
    template_name = 'index.html'

    def get_groups(self):
        rq = """
        select ?org ?label
        where {
            ?org a foaf:Group .
            ?org rdfs:label ?label .
        }
        ORDER BY ?label
        """
        out = []
        for row in tstore.graph.query(rq):
            d = {}
            d['uri'] = row.org.toPython().split('/')[-1]
            d['id'] = d['uri']
            d['text'] = row.label.toPython()
            out.append(d)
        return out

    def get_organizations(self):
        rq = """
        select ?org ?label
        where {
            ?org a foaf:Organization .
            ?org rdfs:label ?label .
        }
        ORDER BY ?label
        LIMIT 10
        """
        out = []
        for row in tstore.graph.query(rq):
            d = {}
            d['uri'] = row.org.toPython().split('/')[-1]
            d['id'] = d['uri']
            d['text'] = row.label.toPython()
            out.append(d)
        return out

    def get_faculty(self):
        rq = """
        select ?fac ?label
        where {
            ?fac a vivo:FacultyMember .
            ?fac rdfs:label ?label .
        }
        ORDER BY ?label
        LIMIT 10
        """
        out = []
        for row in tstore.graph.query(rq):
            d = {}
            d['uri'] = row.fac.toPython().split('/')[-1]
            d['id'] = d['uri']
            d['text'] = row.label.toPython()
            out.append(d)
        return out

    def get_context_data(self, **kwargs):
        context = super(IndexView, self).get_context_data(**kwargs)
        url = self.request.resolver_match.url_name
        if url == u'index':
            context['name'] = 'Index'
            orgs = self.get_organizations()
            faculty = self.get_faculty()
            context['orgs']  = orgs
            context['people'] = faculty
        elif url == u'people':
            context['name'] = 'People'
            faculty = self.get_faculty()
            context['people'] = faculty
        elif url == u'organizations':
            context['name'] = 'Organizations'
            orgs = self.get_organizations()
            groups = self.get_groups()
            orgs += groups
            context['orgs'] = orgs
        return context

class PersonView(TemplateView, ResourceView, BasePropertyView):
    template_name = 'person.html'

    def get_research_areas(self, uri):
        rq = """
        select ?ra ?label
        where {
            ?uri vivo:hasResearchArea ?ra .
            ?ra rdfs:label ?label .
            FILTER NOT EXISTS {?ra a schema:Place }
        }
        """
        out = []
        for row in tstore.graph.query(rq, initBindings={'uri': uri}):
            d = {}
            d['uri'] = row.ra.toPython()
            d['id'] = d['uri']
            d['text'] = row.label.toPython()
            out.append(d)
        return out

    def get_affiliations(self, uri):
        rq = """
        select ?org ?label
        where {
            ?uri schema:affiliation ?org .
            ?org rdfs:label ?label .
        }
        """
        out = []
        for row in tstore.graph.query(rq, initBindings={'uri': uri}):
            d = {}
            d['uri'] = row.org.toPython()
            d['id'] = d['uri']
            d['text'] = row.label.toPython()
            out.append(d)
        return out

    def get_collaborators(self, uri):
        rq = """
        select ?collab ?name
        where {
            ?uri vivo:hasCollaborator ?collab .
            ?collab rdfs:label ?name .
        }"""
        out = []
        for row in tstore.graph.query(rq, initBindings={'uri': uri}):
            d = {}
            d['uri'] = row.collab.toPython()
            d['id'] = d['uri']
            d['text'] = row.name.toPython()
            out.append(d)
        return out

    def get_details(self, uri):
        #Get the name, title and other non-editable details.
        rq = """
        CONSTRUCT {
            ?fac d:name ?label .
            ?fac d:title ?title .
            ?org d:org ?orgName .
            ?collab d:collabName ?collabName .
        }
        where {
            ?fac rdfs:label ?label
            OPTIONAL {
                #Property paths are real slow.
                ?fac obo:ARG_2000028 ?vc .
                ?vc vcard:hasTitle ?t .
                ?t vcard:title ?title .
            }
            OPTIONAL {
                ?position a vivo:Position ;
                          vivo:relates ?fac ;
                          vivo:relates ?org .
                ?org a foaf:Organization.
                ?org rdfs:label ?orgName .
            }
        }
        """
        data = tstore.graph.query(rq, initBindings={'fac': uri})
        g = data.graph
        return {
            'name': g.value(subject=uri, predicate=D['name']),
            'title': g.value(subject=uri, predicate=D['title']),
            'orgs': [{'id' : o.org.toPython().replace(D, ''), 'name': o.name} for o in g.query('select ?org ?name where { ?org d:org ?name}')],
        }

    def get_context_data(self, local_name=None, **kwargs):
        context = super(PersonView, self).get_context_data(**kwargs)
        uri = D[local_name]
        details = self.get_details(uri)
        context['uri'] = uri
        context['local_name'] = uri
        #In production, this will need to be refactored because each call
        #to graph generates a SPARQL query.
        profile = {
            'overview': tstore.graph.value(subject=uri, predicate=VIVO.overview),
            'researchOverview': tstore.graph.value(subject=uri, predicate=VIVO.researchOverview),
            'teachingOverview': tstore.graph.value(subject=uri, predicate=VIVO.teachingOverview),
        }
        profile.update(details)
        prepared_sections = []
        for section in person:
            if section['id'] == 'researchArea':
                section['data'] = json.dumps(self.get_research_areas(uri))
            elif section['id'] == 'placeResearchArea':
                section['data'] = json.dumps(self.get_place_research_areas(uri))
            elif section['id'] == 'affiliations':
                section['data'] = json.dumps(self.get_affiliations(uri))
            elif section['id'] == 'collaborators':
                section['data'] = json.dumps(self.get_collaborators(uri))
            else:
                section['data'] = profile.get(section['id'])
            prepared_sections.append(section)
        context['sections']  = prepared_sections
        context['profile'] = profile
        return context

class OrganizationView(TemplateView, ResourceView, BasePropertyView):
    template_name = 'organization.html'

    def get_context_data(self, local_name=None, **kwargs):
        context = {}
        uri = D[local_name]
        context['uri'] = uri
        context['name'] = tstore.graph.value(subject=uri, predicate=RDFS.label)
        profile = {
            'overview': tstore.graph.value(subject=uri, predicate=VIVO.overview),
        }
        prepared_sections = []
        for section in organization:
            if section['id'] == 'researchArea':
                section['data'] = json.dumps(self.get_research_areas(uri))
            elif section['id'] == 'placeResearchArea':
                section['data'] = json.dumps(self.get_place_research_areas(uri))
            else:
                section['data'] = profile.get(section['id'])
            prepared_sections.append(section)
        context['sections']  = prepared_sections
        context['profile'] = profile
        return context

#
# - Services for autcomplete widgets
#

class JSONServiceView(View, JSONResponseMixin):
    def render_to_response(self, context):
        return JSONResponseMixin.render_to_response(self, context)

class FASTTopicAutocompleteView(JSONServiceView):

    def get(self, request, *args, **kwargs):
        fs = FASTService()
        #topics
        index = 'suggest50'
        context = {}
        query = self.request.GET.get('query')
        out = fs.get(query, index)
        context['results'] = out
        return self.render_to_response(context)

class FASTPlaceAutocompleteView(JSONServiceView):
    def get(self, request, *args, **kwargs):
        fs = FASTService()
        #locations/geograph
        index = 'suggest51'
        context = {}
        query = self.request.GET.get('query')
        out = fs.get(query, index)
        context['results'] = out
        return self.render_to_response(context)

class FASTOrganizationAutocompleteView(JSONServiceView):
    def get(self, request, *args, **kwargs):
        fs = FASTService()
        context = {}
        #organizations
        index = 'suggest10'
        query = self.request.GET.get('query')
        out = fs.get(query, index)
        context['results'] = out
        return self.render_to_response(context)

class VIVOCollaboratorsAutocompleteView(JSONServiceView):
    def get(self, request, *args, **kwargs):
        #fellow faculty members.
        context = {}
        vs = VIVOService()
        query = self.request.GET.get('query')
        out = vs.get(query, VIVO['FacultyMember'].toPython())
        context['results'] = out
        return self.render_to_response(context)


"""
机器人的运动范围
地上有一个m行n列的方格，从坐标 [0,0] 到坐标 [m-1,n-1] 。一个机器人从坐标 [0, 0] 的格子开始移动，它每次可以向左、右、上、下
移动一格（不能移动到方格外），也不能进入行坐标和列坐标的数位之和大于k的格子。例如，当k为18时，机器人能够进入方格 [35, 37]
因为3+5+3+7=18。但它不能进入方格 [35, 38]，因为3+5+3+8=19。请问该机器人能够到达多少个格子？
示例 1：
输入：m = 2, n = 3, k = 1
输出：3

示例 2：
输入：m = 3, n = 1, k = 0
输出：1

提示：
1 <= n,m <= 100
0 <= k <= 20
来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/ji-qi-ren-de-yun-dong-fan-wei-lcof
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。
"""
# 这个题目，我先想到的是，注意，比如坐标为（9，9)时，如果m=n=10,且 k = 8这种的话，那么(9,9),(9,10),(10,9)都不能访问，但是
# （10，10）是可以访问的，但是访问不到，被挡住了哦。
# 初步看了第一感觉就是深度优先或者广度优先

import numpy as np
from typing import List
def visit(x:int, y:int, k:int, jvzhen:List[List[int]], m:int, n:int) -> int:
	ans = 0
	step_dict = {0:[-1,0],1:[1,0],2:[0,-1],3:[0,1]}  # 对应上下左右
	for i in range(4):
		loc_x = x
		loc_y = y
		if (loc_x+step_dict[i][0])>=0 and (loc_x+step_dict[i][0])<m and (loc_y+step_dict[i][1])>=0 and (loc_y+step_dict[i][1])<n:
			num = sum([int(a) for a in str(loc_x + step_dict[i][0])])+sum([int(b) for b in str(loc_y+step_dict[i][1])])
			if num<=k and jvzhen[loc_x + step_dict[i][0]][loc_y+step_dict[i][1]] == 0:
				ans +=1
				loc_x = loc_x + step_dict[i][0]
				loc_y = loc_y + step_dict[i][1]
				jvzhen[loc_x][loc_y] = 1
				ans += visit(loc_x,loc_y,k,jvzhen,m,n)
	return ans

class Solution:
	def movingCount(self, m: int, n: int, k: int) -> int:
		jvzhen = np.zeros((m,n)).astype(int)
		jvzhen[0][0] = 1
		ans = visit(0,0,k,jvzhen,m,n)
		return ans+1

solu = Solution()
ans = solu.movingCount(3,1,0)
print(ans)

names=['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light',
        'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',
        'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
        'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',
        'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',
        'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
        'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',
        'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',
        'hair drier', 'toothbrush']
print(len(names))


#! /usr/bin/env python3

import sys
import copy
import time
import json
import hashlib
import logging
import websocket
import threading
import asyncio

from . import formatter

from . import websocket_api

## sets up the socket BASE for binances socket API.
SOCKET_BASE = 'wss://stream.binance.com:9443'
FUTURE_BASE = 'wss://fstream.binance.com'

class Binance_SOCK:

    def __init__(self, streams, isFuture=False):
        '''
        Setup the connection and setup data containers and management variables for the socket.
        '''
        self.MAX_REQUEST_ITEMS      = 10
        self.requested_items        = {}

        self.last_data_recv_time    = 0

        self.socketRunning          = False
        self.socketBuffer           = {}
        self.ws                     = None
        self.stream_names           = streams
        self.query                  = ''
        self.id_counter             = 0

        self.BASE_CANDLE_LIMIT      = 200
        self.BASE_DEPTH_LIMIT       = 10
        self.isFuture               = isFuture

        ## For locally managed data.
        self.live_and_historic_data = False
        self.candle_data            = {}
        self.book_data              = {}
        self.ticker_data            = {}
        self.mark_price             = {}
        self.best_ask               = {}
        self.best_bid               = {}
        self.reading_books          = False

        self.userDataStream_added   = False
        self.listen_key             = None

    def build_query(self):
        self.query = ''
        url = SOCKET_BASE
        if (self.isFuture):
            url = FUTURE_BASE

        if len(self.stream_names) == 0 or self.stream_names == []:
            return('NO_STREAMS_SET')

        elif len(self.stream_names) == 1:
            query = '{0}/ws/{1}'.format(url, self.stream_names[0])

        else:
            query = '{0}/stream?streams='.format(url)

            for i, stream_name in enumerate(self.stream_names):
                query += stream_name
                if i != len(self.stream_names) - 1:
                    query+='/'

        self.query = query
        return('QUERY_BUILT_SUCCESSFULLY')

    def clear_query(self):
        self.query              = ''
        self.stream_names       = []
        self.candles_markets    = []
        self.book_markets       = []

        return('QUERY_CLEARED')

    ## ------------------ [DATA_ACCESS_ENDPOINT] ------------------ ##
    def get_live_depths(self, symbol=None):
        got_books = False
        return_books = self.book_data

        # while not(got_books):
        #     got_books = True
        #     for key in self.book_data:
        #         try:
        #             ask_Price_List = self._orderbook_sorter_algo(copy.deepcopy(self.book_data[key]['a']), 'ask')
        #             bid_Price_List = self._orderbook_sorter_algo(copy.deepcopy(self.book_data[key]['b']), 'bid')
        #             return_books.update({key:{'a':ask_Price_List, 'b':bid_Price_List}})
        #         except RuntimeError as error:
        #             if error == 'dictionary changed size during iteration':
        #                 print('dodged book error')
        #             got_books = False
        #             break

        if symbol:
            if not symbol in return_books:
                #print(return_books)
                pass
            return(return_books[symbol])

        return(return_books)

    def get_live_candles(self, symbol=None):
        if symbol:
            return(self.candle_data[symbol])
        return(self.candle_data)

    def get_live_ticker(self, symbol=None):
        if (symbol):
            return(self.ticker_data[symbol])
        return(self.ticker_data)

    def get_live_markPrice(self, symbol=None):
        if symbol:
            return (self.mark_price[symbol])
        return (self.mark_price)
    def get_best_ask(self,symbol):
        return self.best_ask[symbol]
    def get_best_bid(self,symbol):
        return self.best_bid[symbol]

    ## ------------------ [MANUAL_CALLS_EXCLUSIVE] ------------------ ##
    def subscribe_streams(self, **kwargs):
        return(self._send_message('SUBSCRIBE', **kwargs))

    def unsubscribe_streams(self, **kwargs):
        return(self._send_message('UNSUBSCRIBE', **kwargs))

    def get_current_streams(self):
        return(self._send_message('LIST_SUBSCRIPTIONS'))

    def set_property(self, **kwargs):
        ''' combined = true'''
        return(self._send_message('SET_PROPERTY', **kwargs))

    def get_property(self):
        return(self._send_message('GET_PROPERTY'))

    ## ------------------ [WEBSOCKET_EXCLUSIVE] ------------------ ##
    def set_aggTrade_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_aggTrade_stream, kwargs))

    def set_trade_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_trade_stream, kwargs))

    def set_candle_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_candle_stream, kwargs))

    def set_miniTicker_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_miniTicker_stream, kwargs))

    def set_global_miniTicker_stream(self):
        return(self._param_check(websocket_api.set_global_miniTicker_stream))

    def set_ticker_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_ticker_stream, kwargs))

    def set_gloal_ticker_stream(self):
        return(self._param_check(websocket_api.set_gloal_ticker_stream))

    def set_bookTicker_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_bookTicker_stream, kwargs))

    def set_global_bookTicker_stream(self):
        return(self._param_check(websocket_api.set_global_bookTicker_stream))

    def set_partialBookDepth_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_partialBookDepth_stream, kwargs))

    def set_manual_depth_stream(self, **kwargs):
        return(self._param_check(websocket_api.set_manual_depth_stream, kwargs))

    ## ------------------ [FULL_DATA_EXCLUSIVE] ------------------ ##
    def set_live_and_historic_combo(self, rest_api):
        if not(self.live_and_historic_data):
            tasks = []
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            for stream in self.stream_names:
                symbol = stream.split('@')[0].upper()
                if 'kline' in stream:
                    tasks.append(loop.create_task(self._set_initial_candles(symbol, stream.split('_')[1], rest_api)))
                if 'depth' in stream:
                    tasks.append(loop.create_task(self._set_initial_depth(symbol, rest_api)))
                time.sleep(0.2)
            loop.run_until_complete(asyncio.gather(*tasks))
            RETURN_MESSAGE = 'STARTED_HISTORIC_DATA'
        else:
            if self.candle_data != {}:
                self.candle_data = {}

            RETURN_MESSAGE = 'STOPPED_HISTORIC_DATA'

        self.live_and_historic_data = not(self.live_and_historic_data)

        return(RETURN_MESSAGE)

    def set_live_and_historic_depth(self, rest_api):
        if not(self.live_and_historic_data):
            tasks = []
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            for stream in self.stream_names:
                symbol = stream.split('@')[0].upper()
                if 'depth' in stream:
                    tasks.append(loop.create_task(self._set_initial_depth(symbol, rest_api)))
                time.sleep(0.1)
            loop.run_until_complete(asyncio.gather(*tasks))
            RETURN_MESSAGE = 'STARTED_HISTORIC_DATA'
        else:
            if self.book_data != {}:
                self.book_data = {}

            RETURN_MESSAGE = 'STOPPED_HISTORIC_DATA'

        self.live_and_historic_data = not(self.live_and_historic_data)

        return(RETURN_MESSAGE)

    def set_live_and_historic_ticker(self,rest_api):
        if not(self.live_and_historic_data):
            if (len(self.stream_names) > 40):
                res = rest_api.get_future_24h_ticker()
                for sym in res:
                    hist = formatter.format_ticker(sym, "REST")
                    self.ticker_data.update({sym["symbol"]:hist})
                RETURN_MESSAGE = 'STARTED_HISTORIC_DATA'
            else:
                tasks = []
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
                for stream in self.stream_names:
                    symbol = stream.split('@')[0].upper()
                    if 'ticker' in stream:
                        tasks.append(loop.create_task(self._set_initial_ticker(symbol, rest_api)(symbol, stream.split('_')[1], rest_api)))
                    time.sleep(0.1)
                loop.run_until_complete(asyncio.gather(*tasks))
            RETURN_MESSAGE = 'STARTED_HISTORIC_DATA'
        else:
            if self.ticker_data != {}:
                self.ticker_data = {}

            RETURN_MESSAGE = 'STOPPED_HISTORIC_DATA'

        self.live_and_historic_data = not(self.live_and_historic_data)

        return(RETURN_MESSAGE)

    def set_live_and_historic_markPrice(self, rest_api):
        if not(self.live_and_historic_data):
            if (len(self.stream_names) > 40):
                res = rest_api.get_mark_price()
                for sym in res:
                    data = formatter.format_markPrice(sym,"REST")
                    self.ticker_data.update({sym["symbol"]:data})
                RETURN_MESSAGE = 'STARTED_HISTORIC_DATA'
            else:
                tasks = []
                loop = asyncio.new_event_loop()
                asyncio.set_event_loop(loop)
                for stream in self.stream_names:
                    symbol = stream.split('@')[0].upper()
                    if 'ticker' in stream:
                        tasks.append(loop.create_task(self._set_initial_ticker(symbol, rest_api)(symbol, stream.split('_')[1], rest_api)))
                    time.sleep(0.1)
                loop.run_until_complete(asyncio.gather(*tasks))
            RETURN_MESSAGE = 'STARTED_HISTORIC_DATA'
        else:
            if self.candle_data != {}:
                self.candle_data = {}

            RETURN_MESSAGE = 'STOPPED_HISTORIC_DATA'

        self.live_and_historic_data = not(self.live_and_historic_data)

        return(RETURN_MESSAGE)

    ## ------------------ [USER_DATA_STREAM_EXCLUSIVE] ------------------ ##
    def set_userDataStream(self, AUTHENTICATED_REST, user_data_stream_type, remove_stream=False):
        if remove_stream:
            message = self._param_check(None, {'listenKey':self.listen_key, 'remove_stream':True})
            self.listen_key = None
        else:
            if self.listen_key == None:

                key_auth = AUTHENTICATED_REST.get_listenKey(user_data_stream_type)
                listen_key = key_auth['listenKey']

                message = self._param_check(None, {'listenKey':listen_key})
                self.listen_key = listen_key

                logging.info('[SOCKET_MASTER] Starting local managing')
                lkkaT = threading.Thread(target=self.listenKey_keepAlive, args=(AUTHENTICATED_REST,user_data_stream_type))
                lkkaT.start()

        return(message)

    def listenKey_keepAlive(self, AUTHENTICATED_REST, user_data_stream_type):
        lastUpdate = time.time()

        while self.listen_key != None:
            if (lastUpdate + 1800) < time.time():
                AUTHENTICATED_REST.send_listenKey_keepAlive(user_data_stream_type, listenKey=self.listen_key)
                lastUpdate = time.time()

            time.sleep(1)

    def _param_check(self, api_info, users_passed_parameters=None):
        if 'listenKey' not in users_passed_parameters:
            if api_info.params != None:
                missingParameters = []

                if 'symbol' in users_passed_parameters:
                    if '-' in users_passed_parameters['symbol']:
                        base, quote = users_passed_parameters['symbol'].split('-')
                        users_passed_parameters.update({'symbol':(quote+base).lower()})

                if 'R' in api_info.params:
                    for param in api_info.params['R']:
                        if not(param in users_passed_parameters):
                            missingParameters.append(param)

                if len(missingParameters) >= 1:
                    return('MISSING_REQUIRED_PARAMETERS', missingParameters)

                if 'O' in api_info.params:
                    allParams = api_info.params['R'] + api_info.params['O']
                else:
                    allParams = api_info.params['R']
                unknownParams = []

                for param in users_passed_parameters:
                    if not(param in allParams):
                        unknownParams.append(param)

                if len(unknownParams) >= 1:
                    return('UNEXPECTED_PARAMETERS', unknownParams)

                stream_name = api_info.endpoint

                for param in allParams:
                    if param == 'local_manager':
                        continue

                    if param in users_passed_parameters:
                        stream_name = stream_name.replace('<{0}>'.format(param), users_passed_parameters[param])
                    else:
                        stream_name = stream_name.replace('<{0}>'.format(param), '')

                if stream_name[-1] == '@':
                    stream_name = stream_name[:-1]

            else:
                if users_passed_parameters != None and users_passed_parameters != {}:
                    return('ENDPOINT_TAKES_NO_PARAMETERS_BUT_SOME_WHERE_GIVEN', users_passed_parameters)

                stream_name = api_info.endpoint
        else:
            stream_name = users_passed_parameters['listenKey']

        if 'remove_stream' in users_passed_parameters:
            if users_passed_parameters['remove_stream']:
                if stream_name in self.stream_names:
                    self.stream_names.remove(stream_name)
                return({'result':'REMOVED_STREAM_NAME', 'stream':stream_name})

                if self.ws != None and self.socketRunning:
                    self.unsubscribe_streams(params=[stream_name])

        else:
            if stream_name in self.stream_names:
                return('STREAM_[{0}]_ALREADY_EXISTS'.format(stream_name))
            self.stream_names.append(stream_name)

            if self.ws != None and self.socketRunning:
                self.subscribe_streams(params=[stream_name])

                if self.query.split('/')[3] == 'ws':
                    self.set_property(params=['combined',True])

            return({'result':'CREATED_STREAM_NAME', 'stream':stream_name})

    def start(self):
        '''
        This is used to start the socket.
        '''
        if self.socketRunning:
            return('SOCKET_STILL_RUNNING_PLEASE_RESTART')

        ## -------------------------------------------------------------- ##
        ## Here the sockets URL is set so it can be connected to.NO_STREAMS_SET
        logging.debug('[SOCKET_MASTER] Setting up socket stream URL.')
        if self.query == '':
            if self.build_query() == 'NO_STREAMS_SET':
                return('UNABLE_TO_START_NO_STREAMS_SET')
        self.destURL = self.query

        ## -------------------------------------------------------------- ##
        ## Here the 'create_socket' function is called to attempt a connection to the socket.
        logging.debug('[SOCKET_MASTER] Setting up socket connection.')
        self._create_socket()

        ## -------------------------------------------------------------- ##
        # This block is used to test connectivity to the socket.
        conn_timeout = 5
        while not self.ws.sock or not self.ws.sock.connected and conn_timeout:
            time.sleep(5)
            conn_timeout -= 1

            if not conn_timeout:
                ## If the timeout limit is reached then the websocket is force closed.
                self.ws.close()
                raise websocket.WebSocketTimeoutException('Couldn\'t connect to WS! Exiting.')

        self.socketRunning = True
        logging.info('[SOCKET_MASTER] Sucessfully established the socket.')

    def stop(self):
        self.ws.close()

        while self.socketRunning:
            time.sleep(0.2)

        self.socketRunning = False
        self.socketBuffer = {}
        self.ws = None

    def _create_socket(self):
        '''
        This is used to initilise connection and set it up to the exchange.
        '''
        self.ws = websocket.WebSocketApp(self.destURL,
            on_open = self._on_Open,
            on_message = self._on_Message,
            on_error = self._on_Error,
            on_close = self._on_Close,
            on_ping = self._on_Ping,
            on_pong = self._on_Pong)

        wsthread = threading.Thread(target=lambda: self.ws.run_forever())
        wsthread.start()

    def _send_message(self, method, params=None):

        message = {'method':method,
                'id':self.id_counter}

        if params != None and params != []:
            message.update({'params':params})

        message = json.dumps(message)

        response_data = self.ws.sock.send(message)

        keys = self.requested_items.keys()

        if len(keys) > self.MAX_REQUEST_ITEMS:
            del self.requested_items[min(keys)]

        self.requested_items.update({self.id_counter:None})
        self.id_counter += 1
        return(self.id_counter)

    def _on_Open(self, wsapp):
        '''
        This is called to manually open the websocket connection.
        '''
        logging.debug('[SOCKET_MASTER] Websocket Opened.')

    def _on_Message(self, wsapp, message):
        '''
        This is used to handle any messages recived via the websocket.
        '''
        try:
            raw_data = json.loads(message)
        except Exception as e:
            print('section 2')
            print(e)
            raw_data = None

        self.last_data_recv_time = time.time()

        if raw_data != None:
            if 'data' in raw_data:
                data = raw_data['data']
            else:
                data = raw_data

            if 'id' in data:
                if int(data['id']) in self.requested_items:
                    if data['result'] == None:
                        self.requested_items[int(data['id'])] = True
                    else:
                        self.requested_items[int(data['id'])] = data['result']

            if 'e' in data:
                if self.live_and_historic_data:
                    if data['e'] == 'kline':
                        self._update_candles(data)
                    elif data['e'] == 'depthUpdate':
                        self._update_depth(data)
                    elif data['e'] == '24hrMiniTicker':
                        self._update_ticker(data)
                    elif data['e'] == 'markPriceUpdate':
                        formatted = formatter.format_markPrice(data, 'SOCK')
                        self.mark_price[data['s']] = formatted

                    else:
                        if 'outboundAccountInfo' == data['e']:
                            self.socketBuffer.update({data['e']:data})
                        elif 'outboundAccountPosition' == data['e']:
                            self.socketBuffer.update({data['e']:data})
                        else:
                            if data['e'] == 'balanceUpdate':
                                pass
                            else:
                                try:
                                    self.socketBuffer.update({data['s']:{data['e']:data}})
                                except Exception as e:
                                    print(e)
                                    print('section 1')
                                    print(raw_data)
                else:
                    if data['e'] == 'bookTicker':
                        self.best_ask[data['s']] = float(data['a'])
                        self.best_bid[data['s']] = float(data['b'])
                    else:
                        self.socketBuffer.update({data['e']:data})

    def _on_Ping(self, wsapp, message):
        '''
        This is called to manually open the websocket connection.
        '''
        logging.debug('[SOCKET_MASTER] Websocket ping.')

    def _on_Pong(self, wsapp, message):
        '''
        This is called to manually open the websocket connection.
        '''
        logging.debug('[SOCKET_MASTER] Websocket pong.')

    def _on_Error(self, wsapp, error):
        '''
        This is called when the socket recives an connection based error.
        '''
        logging.warning('[SOCKET_MASTER] Socket error: {0}'.format(error))

    def _on_Close(self, wsapp):
        '''
        This is called for manually closing the websocket.
        '''
        self.socketRunning = False
        logging.info('[SOCKET_MASTER]: Socket closed.')

    async def _set_initial_candles(self, symbol, interval, rest_api):
        try:
            hist_candles = rest_api.get_custom_candles(symbol=symbol, interval=interval, limit=self.BASE_CANDLE_LIMIT, isFuture=self.isFuture)
        except Exception as error:
            logging.critical('[SOCKET_MASTER] _initial_candles error {0}'.format(error))
            logging.warning('[SOCKET_MASTER] _initial_candles {0}'.format(hist_candles))
        self.candle_data.update({symbol:hist_candles})

    async def _set_initial_depth(self, symbol, rest_api):
        try:
            if (self.isFuture):
                rest_data = rest_api.get_future_orderBook(symbol=symbol, limit=self.BASE_DEPTH_LIMIT)
            else:
                rest_data = rest_api.get_orderBook(symbol=symbol, limit=self.BASE_DEPTH_LIMIT)
            hist_books = formatter.format_depth(rest_data, 'REST')
        except Exception as error:
            logging.critical('[SOCKET_MASTER] _set_initial_depth error {0}'.format(error))
            logging.warning('[SOCKET_MASTER] _set_initial_depth {0}'.format(rest_data))
        self.book_data.update({symbol:hist_books})

    async def _set_initial_ticker(self, symbol, rest_api):
        try:
            rest_data = rest_api.get_future_24h_ticker(symbol=symbol)
            hist_ticker = formatter.format_ticker(rest_data, 'REST')
        except Exception as error:
            logging.critical('[SOCKET_MASTER] _initial_candles error {0}'.format(error))
            logging.warning('[SOCKET_MASTER] _initial_candles {0}'.format(hist_ticker))
        self.ticker_data.update({symbol:hist_ticker})

    async def _set_initial_premium(self, symbol, rest_api):
        try:
            rest_data = rest_api.get_mark_price(symbol=symbol)
        except Exception as error:
            logging.critical('[SOCKET_MASTER] _initial_candles error {0}'.format(error))
        self.ticker_data.update({symbol:rest_data["markPrice"]})

    def _update_ticker(self, data):
        ticker = formatter.format_ticker(data, 'SOCK')

        self.ticker_data[data['s']] = ticker

    def _update_candles(self, data):
        rC = data['k']

        live_candle_data = formatter.format_candles(rC, 'SOCK')

        if live_candle_data[0] == self.candle_data[rC['s']][0][0]:
            self.candle_data[rC['s']][0] = live_candle_data

        else:
            if live_candle_data[0] > self.candle_data[rC['s']][0][0]:
                self.candle_data[rC['s']].insert(0, live_candle_data)
                self.candle_data[rC['s']] = self.candle_data[rC['s']][:self.BASE_CANDLE_LIMIT]

    def _update_depth(self, data):
        live_depth_data = formatter.format_depth(data, 'SOCK')

        lastUpdate = live_depth_data['a'][0][0]

        symbol = data['s']

        self.book_data[symbol] = live_depth_data
        # for lask in live_depth_data['a']:
        #     if lask[1] in self.book_data[symbol]['a']:
        #         if lask[2] == 0.0:
        #             del self.book_data[symbol]['a'][lask[1]]
        #             continue

        #         if self.book_data[symbol]['a'][lask[1]][0] >= lask[0]:
        #             continue

        #     self.book_data[symbol]['a'].update({lask[1]:[lask[0],lask[2]]})

        # for lbid in live_depth_data['b']:
        #     if lbid[1] in self.book_data[symbol]['b']:
        #         if lbid[2] == 0.0:
        #             del self.book_data[symbol]['b'][lbid[1]]
        #             continue

        #         if self.book_data[symbol]['b'][lbid[1]][0] >= lbid[0]:
        #             continue

        #     self.book_data[symbol]['b'].update({lbid[1]:[lbid[0],lbid[2]]})

        # all_ask_prices = list(self.book_data[data['s']]['a'].keys())
        # if (len(all_ask_prices) > self.BASE_CANDLE_LIMIT):
        #     all_ask_prices_to_cut = []
        #     for aPrice in all_ask_prices:
        #         if (self.book_data[symbol]['a'][aPrice][0] < lastUpdate):
        #             all_ask_prices_to_cut.append(aPrice)
        #     for aPrice in all_ask_prices_to_cut:
        #         if aPrice in self.book_data[symbol]['a']:
        #             del self.book_data[symbol]['a'][aPrice]

        # all_bid_prices = list(self.book_data[symbol]['b'].keys())
        # if (len(all_bid_prices) > self.BASE_CANDLE_LIMIT):
        #     all_bid_prices_to_cut = []
        #     for bPrice in all_bid_prices:
        #         if (self.book_data[symbol]['b'][bPrice][0] < lastUpdate):
        #             all_bid_prices_to_cut.append(bPrice)
        #     for bPrice in all_bid_prices_to_cut:
        #         if bPrice in self.book_data[symbol]['b']:
        #             del self.book_data[symbol]['b'][bPrice]

    def _orderbook_sorter_algo(self, books_dict_base, side):
        book_depth_organised = []

        prices_list = list(books_dict_base.keys())

        if side == 'ask':
            prices_list.sort()
        elif side == 'bid':
            prices_list.sort(reverse=True)

        prices_list = prices_list

        for price in prices_list:
            if price in books_dict_base:
                book_depth_organised.append([price, books_dict_base[price][1]])

        return(book_depth_organised)


import torch

class GECO():

    def __init__(self, goal, step_size, alpha=0.99, beta_init=1.0,
                 beta_min=1e-10, speedup=None):
        self.err_ema = None
        self.goal = goal
        self.step_size = step_size
        self.alpha = alpha
        self.beta = torch.tensor(beta_init)
        self.beta_min = torch.tensor(beta_min)
        self.beta_max = torch.tensor(1e10)
        self.speedup = speedup

    def to_cuda(self):
        self.beta = self.beta.cuda()
        if self.err_ema is not None:
            self.err_ema = self.err_ema.cuda()

    def loss(self, err, kld):
        # Compute loss with current beta
        loss = err + self.beta * kld
        # Update beta without computing / backpropping gradients
        with torch.no_grad():
            if self.err_ema is None:
                self.err_ema = err
            else:
                self.err_ema = (1.0-self.alpha)*err + self.alpha*self.err_ema
            constraint = (self.goal - self.err_ema)
            if self.speedup is not None and constraint.item() > 0:
                factor = torch.exp(self.speedup * self.step_size * constraint)
            else:
                factor = torch.exp(self.step_size * constraint)
            self.beta = (factor * self.beta).clamp(self.beta_min, self.beta_max)
        # Return loss
        return loss


import datetime

from sqlalchemy import Column, String, Integer, Date, Sequence
from sqlalchemy.ext.declarative import declarative_base

from gyma.vdv.Entities.EntityBase import EntityBase

from gyma.vdv.db import DBConnection

Base = declarative_base()

class EntityTariff(EntityBase, Base):
    __tablename__ = 'vdv_tariff'

    vdvid = Column(Integer, Sequence('vdv_tariffs_seq'), primary_key=True)
    months = Column(Integer)
    price = Column(Integer)
    sale = Column(Integer)

    json_serialize_items_list = ['vdvid', 'months', 'price', 'sale']

    def __init__(self, months, price, sale):
        super().__init__()
        self.months = months
        self.price = price
        self.sale = sale

    @classmethod
    def add_from_json(cls, data):
        if 'months' in data and 'price' in data and 'sale' in data:
            months = data['months']
            price = data['price']
            sale = data['sale']

            new_entity = EntityTariff(months, price, sale)
            vdvid = new_entity.add()

            try:
                with DBConnection() as session:
                    session.db.commit()
            except Exception as e:
                EntityTariff.delete(vdvid)
                raise Exception('Internal error')
        else:
            raise Exception('Validation exception')
        return vdvid


import discord
from redbot.core import Config, checks, commands
from redbot.core.utils.menus import menu, commands, DEFAULT_CONTROLS
import asyncio
import logging

from .boorucore import BooruCore

class Boorualias:

    @commands.command()
    @commands.guild_only()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def neko(self, ctx, *, tag=""):
        """Neko images"""

        tag_default = " neko"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "nekos_nsfw_neko", "nekos_sfw_neko"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.is_nsfw()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def anal(self, ctx, *, tag=""):
        """Anal images"""

        tag_default = " anal"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "anal", "nekos_nsfw_anal"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.is_nsfw()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def ass(self, ctx, *, tag=""):
        """Ass images"""

        tag_default = " ass"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "ass", "obutts"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.is_nsfw()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def bdsm(self, ctx, *, tag=""):
        """Bdsm images"""

        tag_default = " bdsm"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "bdsm", "nekos_nsfw_bdsm", "nekos_nsfw_femdom"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.is_nsfw()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def boobs(self, ctx, *, tag=""):
        """Boobs images"""

        tag_default = " boobs"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "boobs", "oboobs", "nekos_nsfw_boobs"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.is_nsfw()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def cumon(self, ctx, *, tag=""):
        """Cum_on images"""

        tag_default = " cum_on_*"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "cumshots", "nekos_nsfw_cum"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def yaoi(self, ctx, *, tag=""):
        """Yaoi images"""

        tag_default = " yaoi"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "gay"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def lesbian(self, ctx, *, tag=""):
        """Lesbian images"""

        tag_default = " lesbian yuri"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "lesbian", "nekos_nsfw_yuri"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def milf(self, ctx, *, tag=""):
        """Milf images"""

        tag_default = " milf"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "safe", "milf"]

        await self.generic_alias_booru(ctx, boards, tag)

    @commands.command()
    @commands.guild_only()
    @commands.is_nsfw()
    @commands.bot_has_permissions(embed_links=True, add_reactions=True)
    async def oral(self, ctx, *, tag=""):
        """Oral images"""

        tag_default = " oral"
        tag += tag_default
        boards = ["dan", "gel", "kon", "yan", "cunnilingus", "blowjob", "deepthroat", "nekos_nsfw_blowjob", "nekos_nsfw_cunnilingus"]

        await self.generic_alias_booru(ctx, boards, tag)


from decimal import Decimal
from django.conf import settings
from django.shortcuts import get_object_or_404
from products.models import Product

def bag_contents(request):
    """
    Context processor
    Returns context dictionary of data for the shopping bag
    The function is avaliable accross all the templates
    """

    # setting up vars
    # list of dicts containing item_id, quantitiy and the product obj
    bag_items = []
    total = 0
    product_count = 0
    # get the var if it exists or assign empty dict
    bag = request.session.get('bag', {})

    # using bag items to calc the total, items and product_count
    # item_id is the key and quantity is its value
    for item_id, quantity in bag.items():
        # checks that the item is not found for the bag if its been deleted
        if Product.objects.filter(pk=item_id).exists():
            # get the product
            product = Product.objects.get(pk=item_id)
            total += quantity * product.price
            product_count += quantity
            bag_items.append({
                'item_id': item_id,
                'quantity': quantity,
                'product': product,
                'sub_total': (product.price * quantity)
            })

    if total < settings.FREE_DELIVERY_THRESHOLD:
        # decimal is more accurate than float for rounding
        delivery = total * Decimal(settings.STANDARD_DELIVERY_PERCENTAGE / 100)

        # to show the user how much more they need to spend for free delivery
        free_delivery_delta = settings.FREE_DELIVERY_THRESHOLD - total
    else:
        delivery = 0
        free_delivery_delta = 0

    grand_total = total + delivery

    context = {
        'bag_items': bag_items,
        'product_count': product_count,
        'total': total,
        'delivery': delivery,
        'grand_total': grand_total,
        'free_delivery_delta': free_delivery_delta,
        'free_delivery_threshold': settings.FREE_DELIVERY_THRESHOLD,
    }

    return context


"""Retriever implementation which provides a NetVLAD global image descriptor to suggest potential image pairs.
Note: Similarity computation based off of Paul-Edouard Sarlin's HLOC:
Reference: https://github.com/cvg/Hierarchical-Localization/blob/master/hloc/pairs_from_retrieval.py
https://openaccess.thecvf.com/content_cvpr_2016/papers/Arandjelovic_NetVLAD_CNN_Architecture_CVPR_2016_paper.pdf
Authors: John Lambert
"""
import math
import os
from dataclasses import dataclass
from pathlib import Path
from typing import List, Optional, Tuple

import matplotlib.pyplot as plt
import numpy as np
import torch

import gtsfm.utils.logger as logger_utils
from gtsfm.retriever.retriever_base import RetrieverBase, ImageMatchingRegime

logger = logger_utils.get_logger()
MAX_NUM_IMAGES = 10000

@dataclass
class SubBlockSimilarityResult:
    i_start: int
    i_end: int
    j_start: int
    j_end: int
    subblock: torch.Tensor

class NetVLADRetriever(RetrieverBase):
    def __init__(self, num_matched: int, min_score: float = 0.1, blocksize: int = 50) -> None:
        """
        Args:
            num_matched: Number of K potential matches to provide per query. These are the top "K" matches per query.
            min_score: Minimum allowed similarity score to accept a match.
            blocksize: Size of matching sub-blocks when creating similarity matrix.
        """
        super().__init__(matching_regime=ImageMatchingRegime.RETRIEVAL)
        self._num_matched = num_matched
        self._blocksize = blocksize
        self._min_score = min_score

    def __repr__(self) -> str:
        return f"""
        NetVLADRetriever:
            Num. frames matched: {self._num_matched}
            Block size: {self._blocksize}
            Minimum score: {self._min_score}
        """

    def get_image_pairs(
        self,
        global_descriptors: Optional[List[np.ndarray]],
        image_fnames: List[str],
        plots_output_dir: Optional[Path] = None,
    ) -> List[Tuple[int, int]]:
        """Compute potential image pairs.

        Args:
            global_descriptors: the global descriptors for the retriever, if needed.
            image_fnames: file names of the images
            plots_output_dir: Directory to save plots to. If None, plots are not saved.

        Returns:
            List of (i1,i2) image pairs.
        """
        if global_descriptors is None:
            raise ValueError("Global descriptors need to be provided")
        sim = self.compute_similarity_matrix(global_descriptors)
        return self.compute_pairs_from_similarity_matrix(
            sim=sim, image_fnames=image_fnames, plots_output_dir=plots_output_dir
        )

    def compute_similarity_matrix(self, global_descriptors: List[np.ndarray]) -> torch.tensor:
        """Compute a similarity matrix between all pairs of images.
        We use block matching, to avoid excessive memory usage.
        We cannot fit more than 50x50 sized block into memory, on a 16 GB RAM machine.
        A similar blocked exhaustive matching implementation can be found in COLMAP:
        https://github.com/colmap/colmap/blob/dev/src/feature/matching.cc#L899

        Args:
            global_descriptors: global descriptors, one per image.

        Returns:
            Similarity matrix as tensor of shape (num_images, num_images).
        """
        num_images = len(global_descriptors)
        if num_images > MAX_NUM_IMAGES:
            raise RuntimeError("Cannot construct similarity matrix of this size.")

        subblock_results: List[SubBlockSimilarityResult] = []
        num_blocks = math.ceil(num_images / self._blocksize)

        # TODO(Ayush, John): do we still need to do subblock computations?
        for block_i in range(num_blocks):
            for block_j in range(block_i, num_blocks):
                subblock_results.append(
                    self._compute_similarity_subblock(
                        global_descriptors=global_descriptors, block_i=block_i, block_j=block_j
                    )
                )

        sim = self._aggregate_subblocks(subblock_results=subblock_results, num_images=num_images)
        return sim

    def _compute_similarity_subblock(self, global_descriptors: List[np.ndarray], block_i: int, block_j: int):
        """Compute a sub-block of an global descriptor based similarity matrix.
        Args:
            global_descriptors: global descriptors, one per image.
            block_i: row index of sub-block.
            block_j: column index of sub-block.
        Returns:
            sub-block similarity result.
        """
        num_images = len(global_descriptors)
        device = "cuda" if torch.cuda.is_available() else "cpu"
        num_blocks = math.ceil(num_images / self._blocksize)
        # only compute the upper triangular portion of the similarity matrix.
        logger.info("Computing matching block (%d/%d,%d/%d)", block_i, num_blocks - 1, block_j, num_blocks - 1)
        i_start = block_i * self._blocksize
        i_end = (block_i + 1) * self._blocksize
        i_end = min(i_end, num_images)
        j_start = block_j * self._blocksize
        j_end = (block_j + 1) * self._blocksize
        j_end = min(j_end, num_images)
        # Form (K,D) for K images.
        block_i_query_descs = torch.from_numpy(np.array(global_descriptors[i_start:i_end]))
        block_j_query_descs = torch.from_numpy(np.array(global_descriptors[j_start:j_end]))
        # Einsum equivalent to (img_descs @ img_descs.T)
        sim_block = torch.einsum("id,jd->ij", block_i_query_descs.to(device), block_j_query_descs.to(device))
        return SubBlockSimilarityResult(i_start=i_start, i_end=i_end, j_start=j_start, j_end=j_end, subblock=sim_block)

    def _aggregate_subblocks(self, subblock_results: List[SubBlockSimilarityResult], num_images: int) -> torch.Tensor:
        """Aggregate results from many independently computed sub-blocks of the similarity matrix into a single matrix.

        Args:
            subblock_results: Metadata and results of similarity matrix sub-block computation.
            num_images: Number of images to compare for matching.

        Returns:
            sim: Tensor of shape (num_images, num_images) representing similarity matrix.
        """
        sim = torch.zeros((num_images, num_images))
        for sr in subblock_results:
            sim[sr.i_start : sr.i_end, sr.j_start : sr.j_end] = sr.subblock
        return sim

    def compute_pairs_from_similarity_matrix(
        self, sim: torch.Tensor, image_fnames: List[str], plots_output_dir: Optional[Path] = None
    ) -> List[Tuple[int, int]]:
        """

        Args:
            sim: Tensor of shape (num_images, num_images) representing similarity matrix.
            image_fnames: the names of the images
            plots_output_dir: Directory to save plots to. If None, plots are not saved.

        Returns:
            pair_indices: (i1,i2) image pairs.
        """
        num_images = len(image_fnames)
        # Avoid self-matching and disallow lower triangular portion
        is_invalid_mat = ~np.triu(np.ones((num_images, num_images), dtype=bool))
        np.fill_diagonal(a=is_invalid_mat, val=True)
        pairs = pairs_from_score_matrix(
            sim, invalid=is_invalid_mat, num_select=self._num_matched, min_score=self._min_score
        )
        named_pairs = [(image_fnames[i], image_fnames[j]) for i, j in pairs]
        if plots_output_dir:
            os.makedirs(plots_output_dir, exist_ok=True)
            # Save image of similarity matrix.
            plt.imshow(np.triu(sim.detach().cpu().numpy()))
            plt.title("Image Similarity Matrix")
            plt.savefig(str(plots_output_dir / "netvlad_similarity_matrix.jpg"), dpi=500)
            plt.close("all")
            # Save values in similarity matrix.
            np.savetxt(
                fname=str(plots_output_dir / "netvlad_similarity_matrix.txt"),
                X=sim.detach().cpu().numpy(),
                fmt="%.2f",
                delimiter=",",
            )

            # Save named pairs and scores.
            with open(plots_output_dir / "netvlad_named_pairs.txt", "w") as fid:
                for _named_pair, _pair_ind in zip(named_pairs, pairs):
                    fid.write("%.4f %s %s\n" % (sim[_pair_ind[0], _pair_ind[1]], _named_pair[0], _named_pair[1]))

        logger.info("Found %d pairs from the NetVLAD Retriever.", len(pairs))
        return pairs

def pairs_from_score_matrix(
    scores: torch.Tensor, invalid: np.array, num_select: int, min_score: Optional[float] = None
) -> List[Tuple[int, int]]:
    """Identify image pairs from a score matrix.

    Note: Similarity computation here is based off of Paul-Edouard Sarlin's HLOC:
    Reference: https://github.com/cvg/Hierarchical-Localization/blob/master/hloc/pairs_from_retrieval.py

    Args:
        scores: (K1,K2) for matching K1 images against K2 images.
        invalid: (K1,K2) boolean array indicating invalid match pairs (e.g. self-matches).
        num_select: Number of matches to select, per query.
        min_score: Minimum allowed similarity score.

    Returns:
        pairs: Tuples representing pairs (i1,i2) of images.
    """
    N = scores.shape[0]
    # if there are only N images to choose from, selecting more than N is not allowed
    num_select = min(num_select, N)
    assert scores.shape == invalid.shape
    invalid = torch.from_numpy(invalid).to(scores.device)
    if min_score is not None:
        # logical OR.
        invalid |= scores < min_score
    scores.masked_fill_(invalid, float("-inf"))
    topk = torch.topk(scores, k=num_select, dim=1)
    indices = topk.indices.cpu().numpy()
    valid = topk.values.isfinite().cpu().numpy()
    pairs = []
    for i, j in zip(*np.where(valid)):
        pairs.append((i, indices[i, j]))
    return pairs


"""
JADN Definition BaseModel
Customized from `jadnschema.schema.baseModel.BaseModel`
"""
from copy import deepcopy
from enum import Enum
from typing import Any, ClassVar
from pydantic import create_model  # pylint: disable=no-name-in-module
from pydantic.main import ModelMetaclass  # pylint: disable=no-name-in-module
from .options import Options
from .field import getFieldSchema, getFieldType
from ..consts import SELECTOR_TYPES, STRUCTURED_TYPES, FIELD_TYPES
from ..baseModel import BaseModel
from ...utils import classproperty, ellipsis_str
__pdoc__ = {
    "DefinitionBase.name": "The definition's valid schema name",
    "DefinitionBase.description": "The definition's description",
    "DefinitionBase.data_type": "The definition's base datatype"
}

class DefinitionMeta(ModelMetaclass):
    def __new__(mcs, name, bases, attrs, **kwargs):  # pylint: disable=bad-classmethod-argument
        base_opts = [b.__options__ for b in reversed(bases) if issubclass(b, BaseModel) and b != BaseModel]
        opts = Options(
            *base_opts,
            attrs.pop("__options__", None), attrs.pop("Options", None),
            kwargs.pop("__options__", None), kwargs.pop("Options", None),
            getattr(mcs, "__options__", None), getattr(mcs, "Options", None),
        )
        new_namespace = {
            **attrs,
            **kwargs,
            "__options__": opts,
        }
        cls = super().__new__(mcs, name, bases, new_namespace)  # pylint: disable=too-many-function-args
        base_names = [b.__name__ for b in bases]
        for idx, (field, opts) in enumerate(cls.__fields__.items()):
            if field != "__root__":
                opts.field_info.extra["parent"] = cls
                field_opts = opts.field_info.extra["options"]
                field_opts.setdefault("name", f"{name}.{opts.alias}")
                field_opts.setdefault("data_type", getFieldType(opts))
                opts.field_info.extra.setdefault("id", idx)
                if not opts.required and field_opts.minc != 0 and "Choice" not in base_names:
                    field_opts.minc = 0
        return cls

class DefinitionBase(BaseModel, metaclass=DefinitionMeta):  # pylint: disable=invalid-metaclass
    __options__: ClassVar[Options]

    def __str__(self):
        cls = self.__class__
        mro = [c for c in cls.__mro__ if not c.__name__ == cls.__name__][0]
        data = self.json(exclude_none=True)
        return f"{self.name}({mro.__name__}: {ellipsis_str(data)})"

    # Pydantic overrides
    @classmethod
    def schema(cls) -> list:
        """
        Format the definition to valid JADN schema format
        :return: formatted JADN
        """
        mro = [c for c in cls.__mro__ if not c.__name__ == cls.__name__][0]
        schema = [cls.name, mro.__name__, cls.__options__.schema(), (cls.__doc__ or "").strip()]
        if cls.__fields__ and "__root__" not in cls.__fields__:
            fields = []
            for opt in cls.__fields__.values():
                fields.append(getFieldSchema(opt))
            schema.append(fields)
        return schema

    # Custom Methods
    @classmethod
    def is_enum(cls) -> bool:
        """
        Determine if the definition is an enumerated type
        :return: True/False if the definition is an enumerated type
        """
        for base in cls.__mro__:
            if base.__name__ == "Enumerated":
                return True
        return False

    @classmethod
    def is_structure(cls) -> bool:
        """
        Determine if the definition is a structure type
        `Array`, `ArrayOf`, `Map`, `MapOf`, & `Record` are structure types
        :return: True/False if the definition is a structure type
        """
        for base in cls.__mro__:
            if base.__name__ in STRUCTURED_TYPES:
                return True
        return False

    @classmethod
    def is_selector(cls) -> bool:
        """
        Determine if the definition is a selector type
        `Enumerated` & `Choice` are selector types
        :return: True/False if the definition is a selector type
        """
        for base in cls.__mro__:
            if base.__name__ in SELECTOR_TYPES:
                return True
        return False

    @classmethod
    def has_fields(cls) -> bool:
        """
        Determine if the definition has fields
        `Enumerated`, `Choice`, `Array`, `Map`, & `Record` should have defined fields
        :return: True/False if the definition has fields
        """
        for base in cls.__mro__:
            if base.__name__ in FIELD_TYPES:
                return True
        return False

    # Helpers
    @classmethod
    def expandCompact(cls, value: Any) -> Any:
        if isinstance(value, dict) and cls.__fields__:
            rtn = {}
            for field in cls.__fields__.values():
                f_id = field.field_info.extra["id"]
                if f_id in value:
                    rtn[field.alias] = field.type_.expandCompact(value[f_id])
            return rtn
        return value

    @classproperty
    def name(cls) -> str:  # pylint: disable=no-self-argument
        """The definition's valid schema name"""
        return cls.__options__.name or cls.__name__

    @classproperty
    def description(cls) -> str:  # pylint: disable=no-self-argument
        """The definition's description"""
        return cls.__doc__

    @classproperty
    def data_type(cls) -> str:  # pylint: disable=no-self-argument
        """The definition's base datatype"""
        return cls.__options__.data_type

    @classmethod
    def enumerated(cls) -> "Enumerated":  # pylint: disable=no-self-argument
        """
        Convert the given class to an 'Enumerated' class if applicable
        :return: converted Enumerated class object
        """
        if cls.data_type in ("Binary", "Boolean", "Integer", "Number", "Null", "String"):
            raise TypeError(f"{cls.name} cannot be extended as an enumerated type")

        if cls.data_type == "Enumerated":  # pylint: disable=comparison-with-callable
            return cls

        from .structures import Enumerated  # pylint: disable=import-outside-toplevel
        name = f"Enum-{cls.name}"
        values = {}
        for k, v in cls.__fields__.items():
            values[k] = deepcopy(v.field_info)
            values[k].default = k

        cls_kwargs = dict(
            __name__=name,
            __doc__=f"Derived Enumerated from {cls.name}",
            __enums__=Enum("__enums__", values),
            __options__=Options(cls.__options__, name=name)
        )
        return create_model(name, __base__=Enumerated, __cls_kwargs__=cls_kwargs)

    class Config:
        arbitrary_types_allowed = True
        orm_mode = True


# -*- coding: utf-8 -*-
"""
Created on Thu Oct 29 15:13:36 2020

@author: deyiwang@qq.com

实现功能: 选取两张图片的控制点, 完成对应视角的转换

参考代码: https://blog.csdn.net/fanzy1234/article/details/103072723?utm_medium=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-2.channel_param&depth_1-utm_source=distribute.pc_relevant.none-task-blog-BlogCommendFromMachineLearnPai2-2.channel_param
以及: https://blog.csdn.net/qq_36281080/article/details/103928487
"""

from imutils import perspective
from skimage.filters import threshold_local
import cv2
import imutils
import numpy as np

def SetPoints(windowname, img):
    """
    输入图片，打开该图片进行标记点，返回的是标记的几个点的字符串
    """
    print('(提示：单击需要标记的坐标，Enter确定，Esc跳过，其它重试。)')
    points = []

    def onMouse(event, x, y, flags, param):
        if event == cv2.EVENT_LBUTTONDOWN:
            cv2.circle(temp_img, (x, y), 10, (102, 217, 239), -1)
            points.append([x, y])
            cv2.imshow(windowname, temp_img)

    temp_img = img.copy()
    cv2.namedWindow(windowname)
    cv2.imshow(windowname, temp_img)
    cv2.setMouseCallback(windowname, onMouse)
    key = cv2.waitKey(0)
    if key == 13:  # Enter
        print('坐标为：', points)
        del temp_img
        cv2.destroyAllWindows()
        return points
    elif key == 27:  # ESC
        print('跳过该张图片')
        del temp_img
        cv2.destroyAllWindows()
        return
    else:
        print('重试!')
        return SetPoints(windowname, img)

def extra_point(aol,img):
    points = SetPoints(str(aol), img)
    temp = np.array(points)
    pts = temp.reshape(4, 2).astype(np.float32)
    return pts

def read_img(path):
    image = cv2.imread(path)
    w=image.shape[0]
    h=image.shape[1]
    ratio = image.shape[0] / 500.0  # 比例
    orig = image.copy()
    image = imutils.resize(image, height=500)
    return image

def main():

    image1 = read_img("real.jpg")
    image2 = read_img("satellite.png")

    point1 = extra_point('real',image1)
    point2 = extra_point("sat", image2)

    #实际坐标点和提取的角点必须一一对应呀，
    M = cv2.getPerspectiveTransform(point1,point2)
    out_img = cv2.warpPerspective(image1,M,(image1.shape[0],700))
    dst=cv2.perspectiveTransform(point2.reshape(1,4,2), M)

    cv2.imshow("Original", image1)
    cv2.imshow("Scanned",cv2.resize(out_img,(image1.shape[0],700)))

if __name__ == "__main__":
    main()


#!/usr/bin/python3
# -*- coding: utf-8 -*

# Photobooth version 1

# Pour info, la taille de l'écran 7 pouces est de 640x480.

import RPi.GPIO as GPIO
import time #Utile pour le sime.sleep pour mettre en pause le programme
from datetime import datetime #Utile pour générer le nom de la photo selon la date du jour. Voir ligne 106. Doc : https://docs.python.org/fr/3.6/library/datetime.html
from PIL import Image #PIL sert à ouvrir, manipuler, sauveguarder des images
import pygame
from pygame.locals import *
import os

#-- Import et création d'un fichier de log --#
import logging
logging.basicConfig(filename='FichierDeLogs.log', encoding='utf-8', level=logging.DEBUG)
logging.info('This message should go the the file')

#--------------------------------------------#

import logging #Utile pour logguer tous les évènements

GPIO.setmode(GPIO.BCM) #Attention au choix du port ; référez-vous au site https://fr.pinout.xyz/
GPIO.setup(25, GPIO.IN, pull_up_down=GPIO.PUD_UP)

pygame.init()
infoObject = pygame.display.Info() #pygame.display.Info sert a creer un objet d'information video destine a l'affichage. Il prend pour proprietes ce qu'il trouvera dans current-h et current_y qui sont des attributs de .display.info.  Voir https://devdocs.io/pygame/ref/display#pygame.display.Info
screen = pygame.display.set_mode((infoObject.current_w,infoObject.current_h), pygame.FULLSCREEN)
screen = pygame.display.set_mode((0,0),pygame.FULLSCREEN)
background = pygame.Surface(screen.get_size())  # Create the background object ; pygame.Surface va creer un nouvel objet image avec la taille pour arguments. Screen est une variable-
background = background.convert()  # Convert it to a background - Pour que le format de pixel soit identique entre le fond et le background

screenPicture = pygame.display.set_mode((infoObject.current_w,infoObject.current_h), pygame.FULLSCREEN)  # Full screen - Idem c la variable screen
backgroundPicture = pygame.Surface(screenPicture.get_size())  # Create the background object
backgroundPicture = background.convert()  # Convert it to a background - Pour que le format de pixel soit identique entre le fond et le background
#screen = pygame.display.set_mode((0,0),pygame.RESIZABLE)
width, height = screen.get_size()
rectScreen = screen.get_rect()
pygame.mouse.set_visible (False)
BUTTON_PIN = 25

#Ppygame.mixer.init()
#son = pygame.mixer.Sound('/home/pi/Photomaton/son.wav')
#canal = son.play()

def PrisePhoto(NomPhoto): #prendre une photo avec Raspistill
    command = "sudo raspistill -t 2000 -w 1200 -h 675 -o "+ NomPhoto +" -q 100" #prend une photo après 5sec de largeur 1200 et hauteur 675 de qualité 100
    os.system(command)

def AfficherPhoto(NomPhoto): # affiche NomPhoto
    print("loading image: " + NomPhoto)
    background = pygame.image.load(NomPhoto);
    background.convert_alpha()
    background = pygame.transform.scale(background,(width,height))
    screen.blit(background,(0,0),(0,0,width,height))
    pygame.display.flip()

def decompte():
  AfficherTexte("Attention")
  time.sleep(1)
  AfficherTexte("--> Photo dans 5 secondes")
  time.sleep(1)

def AfficherTexte(message): # pour pouvoir afficher des messages sur un fond noir
    screen.fill(pygame.Color(0,0,0))
    font = pygame.font.SysFont("verdana", 30, bold=1)
    textsurface = font.render(message, 1, pygame.Color(255,255,255))
  #  textpos = text.get_rect()
  #  textpos.centerx = background.get_rect().centerx
  #  textpos.centery = background.get_rect().centery

  #  if(ImageShowed):
  #      backgroundPicture.blit(text, textpos)
  #  else:
  #      background.blit(text, textpos)

    #screen.blit(textsurface,(240,240)) # Position x et y du texte
    #pygame.display.update()

def AfficherTexteTransparent(message): # pour pouvoir afficher des messages en conservant le fond
    font = pygame.font.SysFont("verdana", 30, bold=1)
    textsurface = font.render(message, 1, pygame.Color(255,255,255))
    screen.blit(textsurface,(35,40))
    pygame.display.update()

def AfficherTexteAccueil(message): # Afficher un Texte sur l'image d'accueil (ou à la place)
    font = pygame.font.SysFont("verdana", 30, bold=1)
    textsurface = font.render(message, 1, pygame.Color(100,150,200))
    screen.blit(textsurface,(35,40))
    pygame.display.update()

def Evenement(): #Est en charge de retourner TRUE ou FALSE
    global pygame

    RienNeSePasse = True #initialisation de la variable RienNeSePasse a TRUE. Il n'y a pas d'evenement, rien ne se passe.
    while RienNeSePasse: #tant qu'il n'y a pas d'evenement.... Tant que RienNeSePasse est au statut defini au-dessus
            input_state = GPIO.input(BUTTON_PIN) #...alors on va lire (GPIO.input()) le PIN (variable BUTTON_PIN initialisé à 25 au début). Correspond a TRUE. La variable input_state correspond a le lecture de l'entree BUTTON_PIN (25). Voir https://deusyss.developpez.com/tutoriels/RaspberryPi/PythonEtLeGpio/#LIII-A
            if input_state == False: #Si l'etat change (appui sur le bouton)...
                    RienNeSePasse = False #Alors on indique cette variable change.
                    return #On retourne l'etat
            for event in pygame.event.get(): #pygame.event.get() va lire les evenements en attente dans la queue ainsi que les y retirer.
                    if event.type == pygame.KEYDOWN: #Si dans la queue il y a un appui sur une touche du clavier...
                        if event.key == pygame.K_ESCAPE: #...et qu'il s'agit de la touche echap...
                            pygame.quit() #...alors on quitte le programme https://www.pygame.org/docs/ref/key.html#comment_pygame_key_name
                        if event.key == pygame.K_DOWN: #Toutefois s'il y a un appuisur la fleche du bas
                            RienNeSePasse = False #Alors on change l'etat et donc on attend 0.2s
                            return #On retourne l'etat
            time.sleep(0.2)

# Début du programme #

if (os.path.isdir("/home/pi/Desktop/photos") == False): # si le dossier pour stocker les photos n'existe pas
    os.mkdir("/home/pi/Desktop/photos")                  # alors on crée le dossier (sur le bureau)
    os.chmod("/home/pi/Desktop/photos",0o777)            # et on change les droits pour pouvoir effacer des photos

#AfficherTexteAccueil("Installez-vous et appuyez sur le bouton pour prendre une photo")

while True : #boucle jusqu'a interruption
    print ("\n attente boucle \n ")
    screensize = screen.get_rect() #Je veux récupérer la taille de l'écran...
    print ("L'écran fait", screensize) #...afin de l'afficher dans la console

     #On affiche la photo d'accueil
    AfficherPhoto("/home/pi/Photobooth/images/appuyezbouton.jpg")

     #on attend que le bouton soit pressé
    Evenement()
     # on a appuyé sur le bouton...
     #on lance le décompte
    decompte()
    pygame.display.flip()
     #on génère le nom de la photo avec heure_min_sec
    date_today = datetime.now()
    nom_image = date_today.strftime('%d-%m-%Y_%Hh-%Mm-%Ss') #Voir strftime de la doc https://docs.python.org/fr/3.6/library/datetime.html#strftime-strptime-behavior
     #on déclenche la prise de photo
    chemin_photo = '/home/pi/Desktop/photos/'+nom_image+'.jpeg'
    PrisePhoto(chemin_photo) #Déclenchement de la prise de photo
    AfficherTexte("--> Merci ! <--")
    time.sleep(1)
     #On affiche la photo...
    AfficherPhoto(chemin_photo)
     #...et par dessus on affiche un message
    AfficherTexteTransparent("OK ; voici ce qui est dans la boite ...")
    time.sleep(2) #Ajout d'un temps d'affichage afin de repartir sur l'accueil ensuite
    pygame.display.flip()
     #on recommence en rechargeant l'écran d'accueil
    #AfficherPhoto("/home/pi/Photobooth/images/appuyezbouton.jpg")
    #pygame.mixer.init()
    #son = pygame.mixer.Sound('/home/pi/Photomaton/son.wav')
    #canal = son.play()

    #if (GPIO.input(25) == 0): #si le bouton est encore enfoncé (son etat sera 0)
          #print ("Ho ; bouton  appuyé !!! Je dois sortir ; c'est le chef qui l'a dit !") #Va afficher ça dans la console
          #break # alors on sort du while

    #for event in pygame.event.get(): #pygame.event.get() va lire les evenements en attente dans la queue ainsi que les y retirer.
        #if event.type == pygame.KEYDOWN: #Si dans la queue il y a un appui sur la fleche du bas sur le clavier alors on attends 0.2s (fin de la fonction)
            #if event.key == pygame.K_ESCAPE: #Mais si il y a un appui sur la touche echap https://www.pygame.org/docs/ref/key.html#comment_pygame_key_name
                #print("Appui sur Echap, fin du programme")
                #pygame.quit() #Alors on quitte le programme

GPIO.cleanup()           # reinitialisation GPIO lors d'une sortie normale


from classes import Collector, DBManager
from utils import connect_or_create_db, create_tables, load_db_tables

company_names = ['LC Group', 'IBS', 'Rover', 'R-Style', 'Rosco', 'Марвел', 'Verysell', 'Yandex', 'Газпром', "Тинькофф"]

if __name__ == '__main__':
    conn, cur = connect_or_create_db('hh_vacancies')
    create_tables(conn, cur)

    collector = Collector()
    for company in company_names:
        collector.add_hh_vacancies(company)

    load_db_tables(cur, collector.vacancies)
    conn.commit()

    db_manager = DBManager(conn, cur)
    # print(db_manager.get_companies_and_vacancies_count())
    # print(db_manager.get_all_vacancies())
    # print(db_manager.get_avg_salary())
    # print(db_manager.get_vacancies_with_higher_salary())
    print(db_manager.get_vacancies_with_keyword(['инженер', 'редактор']))
    cur.close()
    conn.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Jaap.nl scraping - main script
By dr.Pep
"""

# Built-in/Generic imports

# Libs
# import time
# import numpy as np
from datetime import date, datetime
import pandas as pd
import logging
import os, uuid, sys
# from azure.storage.blob import BlockBlobService, PublicAccess, ContentSettings

# Own modules

import ScrapeJaap
from MyConstants import * # This way you get the constants without filename as prefix

# Authorship
__author__ = 'dr.Pep'
__copyright__ = 'Copyright 2019, dr.Pep'
__credits__ = 'Thanks to the magnificent Py4e and PCC'
__licence__ = 'All rights reserved'
__version__ = '0.0.1'
__maintainer__ = 'dr.Pep'
__email__ = 'python@vdlaan.eu'
__status__ = 'In development'

# Set up logging
logging.basicConfig(filename = LOGDATA + LOGFILE,
                    level = logging.DEBUG)
logging.info('Running script MainJaap.py | Time stamp: ' + datetime.now().strftime("%d/%m/%Y %H:%M:%S"))
logging.info('Website to be scraped: ' + str(URL))
logging.info('Number of pages: ' + str(MAXPAGE))

# Scrape all summary pages
for p in range(MAXPAGE):
    # Log where we are
    # logging.info('Scraping summary page ' + str(p+1))
    # Read page
    df_houses_read = ScrapeJaap.read_summary_page(URL,p+1)
    if p == 0:
        df_houses_summary = df_houses_read
    else:
        df_houses_summary = pd.concat(
            [df_houses_summary,df_houses_read],
            ignore_index=True,
            sort =False
        )

# Write summary to file
df_houses_summary.to_csv(PROJECTDATA + SUMMARYFILE,index=False,header=True)

logging.info('Summaries of all ' + str(len(df_houses_summary)) + ' houses have been written to file!')

# Scrape all detail pages
for i in df_houses_summary.index:
    # Start with logging where we are
    # logging.info('Scraping detail page '+ str(i))

    # Read the page with details
    df_detail_read = ScrapeJaap.read_house_detail_page(
        df_houses_summary['link'][i],
        df_houses_summary['pricetype'][i],
        df_houses_summary['id'][i]
    )

    # Some error handling - when page cannot be read
    if type(df_detail_read) == bool:
        logging.warning(
            'Index:' + str(i)
            + ' | Address:' + str(df_houses_summary['address'][i])
            + ' | ID:'+ str(df_houses_summary['id'][i]) + ' | Scraping failed'
        )
        continue

    # Put the scraped data into a data frame
    if i == 0:
        df_houses_detail = df_detail_read
    else:
        df_houses_detail= pd.concat(
            [df_houses_detail,df_detail_read],
            ignore_index=True,
            sort =False
        )

# Write data frame with all house details to file
df_houses_detail.to_csv(PROJECTDATA + DETAILFILE,index=False,header=True)

# Log our success
logging.info('Details of '+str(len(df_houses_detail))+' houses have been written to file!')

# Write Summary and Detail files to blobs
# THE BELOW DOES NOT WORK WITHOUT SAS TOKEN (currently empty)
#
# blob_service = BlockBlobService(account_name = ACCOUNT_NAME, account_key = None, sas_token = SAS_TOKEN)
#
# blob_service.create_blob_from_path(
#     CONTAINER_NAME,
#     LOGFILE,
#     RAWDATA + LOGFILE)
# blob_service.create_blob_from_path(
#     CONTAINER_NAME,
#     SUMMARYFILE,
#     PROJECTDATA + SUMMARYFILE)
# blob_service.create_blob_from_path(
#     CONTAINER_NAME,
#     DETAILFILE,
#     PROJECTDATA + DETAILFILE)


import argparse
import os
import sys
import logging
import traceback

import geopandas as gpd

def get_parser():
    """
    Parser from command line arguments

    :return myparser: Parser
    :rtype: ArgumentParser

    """
    myparser = argparse.ArgumentParser(os.path.basename(__file__))
    myparser.add_argument("-i", type=str, required=True,
                          help="Water mask file")
    myparser.add_argument("-o", type=str, required=True,
                          help="Output file")
    myparser.add_argument("-v", type=float, required=True,
                          help="Height value")
    return myparser

def process_file(inputfile: str, outputfile: str, height: float):
    '''
    Change height of water mask for swot-hydrology-toolbox

    :param inputfile: Input file
    :param outputfile: Output file
    '''

    try:
        data = gpd.read_file(inputfile)
        data.at[0,'height'] = height
        data.to_file(outputfile)
    except:
        traceback.print_exc()
        logging.error(f"Writing output file")
    else:
        logging.info("Write output file: OK")

# Main program
if __name__ == "__main__":
    parser = get_parser()
    args = parser.parse_args()
    level = getattr(logging, "INFO")
    logging.basicConfig(filename=None,format='%(asctime)s [%(levelname)s] %(message)s', level=level)
    process_file(args.i, args.o, args.v)

# End


import numpy as np
import torch

def masked_mse_np(preds_np, labels_np, null_val=np.nan):
    if np.isnan(null_val):
        mask_np = ~np.isnan(labels_np)
    else:
        mask_np = (labels_np != null_val)
    mask_np = mask_np.astype(float)
    mask_np /= np.mean((mask_np))
    mask_np = np.where(np.isnan(mask_np), np.zeros_like(mask_np), mask_np)
    loss_np = (preds_np-labels_np)**2
    loss_np = loss_np * mask_np
    loss_np = np.where(np.isnan(loss_np), np.zeros_like(loss_np), loss_np)
    return np.mean(loss_np)

def masked_rmse_np(preds_np, labels_np, null_val=np.nan):
    return np.sqrt(masked_mse_np(preds_np=preds_np, labels_np=labels_np, null_val=null_val))

def masked_mae_loss_np(y_pred_np, y_true_np):
    mask_np = (y_true_np != 0).float()
    mask_np /= mask_np.mean()
    loss_np = np.abs(y_pred_np - y_true_np)
    loss_np = loss_np * mask_np
    # trick for nans: https://discuss.pytorch.org/t/how-to-set-nan-in-tensor-to-0/3918/3
    loss_np[loss_np != loss_np] = 0
    return loss_np.mean()

def masked_mae_np(preds, labels, null_val=np.nan):
    if np.isnan(null_val):
        mask = ~np.isnan(labels)
    else:
        mask = (labels!=null_val)
    mask = mask.astype(float)
    mask /=  np.mean((mask))
    mask = np.where(np.isnan(mask), np.zeros_like(mask), mask)
    loss = np.abs(preds-labels)
    loss = loss * mask
    loss = np.where(np.isnan(loss), np.zeros_like(loss), loss)
    return np.mean(loss)

def masked_mape_np(preds, labels, null_val=np.nan):
    if np.isnan(null_val):
        mask = ~np.isnan(labels)
    else:
        mask = (labels!=null_val)
    mask = mask.astype(float)
    mask /=  np.mean((mask))
    mask = np.where(np.isnan(mask), np.zeros_like(mask), mask)
    loss = np.abs(preds-labels)/labels
    loss = loss * mask
    loss = np.where(np.isnan(loss), np.zeros_like(loss), loss)
    return np.mean(loss)

def metric_np(pred, real):
    mae = masked_mae_np(pred,real,0.0).item()
    mape = masked_mape_np(pred,real,0.0).item()
    rmse = masked_rmse_np(pred,real,0.0).item()
    return mae,mape,rmse

if __name__ == "__main__":
    data1 = torch.randn(16, 207, 12)
    data2 = torch.randn(16, 207, 12)
    data1_np = data1.numpy()
    data2_np = data2.numpy()
    r1 = metric_np(data1_np, data2_np)


from glob import glob
import numpy as np
import pandas as pd
from keras.preprocessing.image import img_to_array, load_img
import os
import shutil
from definitions import REPO_ROOT_PATH, PROCESSED_DATA_PATH

# set random seed
np.random.seed(10)

###############################################################################
# Set up processed output directories
###############################################################################

# make train validation test subdirectories
train_dir = PROCESSED_DATA_PATH + "train_data/"
validation_dir = PROCESSED_DATA_PATH + "validation_data/"
test_dir = PROCESSED_DATA_PATH + "test_data/"

# if processed data directory exists, remove it
if os.path.exists(PROCESSED_DATA_PATH):
    shutil.rmtree(PROCESSED_DATA_PATH)

# make new processed data directory
os.makedirs(PROCESSED_DATA_PATH)

# make new processed train validation test subdirectories
os.makedirs(train_dir)
os.makedirs(validation_dir)
os.makedirs(test_dir)

###############################################################################
# Separate raw image files into train, validation, and test sets
###############################################################################

# set path for raw data directory
raw_data_path = REPO_ROOT_PATH + "data/raw/"

# load raw test data files
raw_test_files = glob(raw_data_path + "chest_xray/test/*/*.jpeg")

# load raw train data files
raw_train_files = glob(raw_data_path + "chest_xray/train/*/*.jpeg")

# separate raw train files by class
raw_train_bacteria_files = [fn for fn in raw_train_files if 'BACTERIA' in fn]
raw_train_virus_files = [fn for fn in raw_train_files if 'VIRUS' in fn]
raw_train_normal_files = [fn for fn in raw_train_files if 'NORMAL' in fn]

# make list of files in test set
test_files = raw_test_files

# randomly sample image files for validation set
bacteria_val_files = np.random.choice(raw_train_bacteria_files,
                                      size=len(raw_train_bacteria_files) // 4,
                                      replace=False).tolist()

virus_val_files = np.random.choice(raw_train_virus_files,
                                   size=len(raw_train_virus_files) // 4,
                                   replace=False).tolist()

normal_val_files = np.random.choice(raw_train_normal_files,
                                    size=len(raw_train_normal_files) // 4,
                                    replace=False).tolist()

# make lists of files for train set
bacteria_train_files = list(set(raw_train_bacteria_files)
                            - set(bacteria_val_files))
virus_train_files = list(set(raw_train_virus_files) - set(virus_val_files))
normal_train_files = list(set(raw_train_normal_files) - set(normal_val_files))

# concatenate the lists for the train and validation sets
train_files = bacteria_train_files + virus_train_files + normal_train_files
validation_files = bacteria_val_files + virus_val_files + normal_val_files

###############################################################################
# Copy image files into processed train, validation, test subdirectories
###############################################################################

# make image file subdirectories
img_subdir = "image_files/"

os.makedirs(train_dir + img_subdir)
os.makedirs(validation_dir + img_subdir)
os.makedirs(test_dir + img_subdir)

# copy the files to the processed data subdirectories
print("copying image files")

for fn in train_files:
    shutil.copy(fn, train_dir + img_subdir)

for fn in validation_files:
    shutil.copy(fn, validation_dir + img_subdir)

for fn in test_files:
    shutil.copy(fn, test_dir + img_subdir)

###############################################################################
# Process image file data into dataframes
###############################################################################

# function to get class from filename
def get_class_from_filename(filename):
    """Takes a string with the base name of a filepath as input.
    returns the class as a string."""

    if 'NORMAL' in filename:
        str_class = 'normal'
    elif 'BACTERIA' in filename:
        str_class = 'bacterial_pneumonia'
    elif 'VIRUS' in filename:
        str_class = 'viral_pneumonia'
    else:
        raise Exception('Filename not valid!  Class not found.')

    return str_class

# function to make dataframe using list of image filepaths
def df_from_filepath_list(filepath_list, img_size):
    """Takes a list of filepath strings and a tuple of ints
    with image pixel dimensions as input.
    Returns a dataframe with the base path, image class,
    and pixel intensity array for each filepath."""

    list_of_tuples = []

    for fp in filepath_list:
        f_name = os.path.basename(fp)
        f_class = get_class_from_filename(f_name)
        pixel_array = img_to_array(load_img(fp, target_size=img_size))
        list_of_tuples.append((f_name, f_class, pixel_array))

    return pd.DataFrame(list_of_tuples,
                        columns=['image_file_base_path',
                                 'image_class',
                                 'pixel_array_custom_image_size'])

# Set the image size
IMG_SIZE = (224, 224)

# create the train validation test dataframes
print("loading image data into dataframes")

df_train = df_from_filepath_list(train_files, IMG_SIZE)
df_validation = df_from_filepath_list(validation_files, IMG_SIZE)
df_test = df_from_filepath_list(test_files, IMG_SIZE)

###############################################################################
# Save processed data to disk
###############################################################################

print("saving data files")

# save the dataframes to pickle files -- preserves numpy array format
df_validation.to_pickle(validation_dir + "validation_data.pickle")
df_test.to_pickle(test_dir + "test_data.pickle")
df_train.to_pickle(train_dir + "train_data.pickle")

###############################################################################
# Save list of class names
###############################################################################

print("saving class names")

class_list = ['bacterial_pneumonia', 'viral_pneumonia', 'normal']

class_filepath = PROCESSED_DATA_PATH + "y_ohe_class_names.txt"

with open(class_filepath, "w") as f:
    for c in class_list:
        f.write(c + "\n")

print("done")


from __future__ import annotations

from typing import Dict, Iterable, List, Optional, Tuple, Type, TypeVar

from sqlalchemy.exc import DataError
from sqlalchemy.inspection import inspect

from crystalcv.extensions import db

T = TypeVar("T", bound="BaseModel")

class BaseModel(db.Model):
    __abstract__ = True

    @classmethod
    def _get_primary_key_manes(cls):
        return [k.name for k in inspect(cls).primary_key]

    @classmethod
    def create(cls, **kwargs):
        """Create a new record and save it the database."""
        instance = cls(**kwargs)
        return instance.save()

    def update(self, commit=True, **kwargs):
        """Update specific fields of a record."""
        for attr, value in kwargs.items():
            setattr(self, attr, value)
        return commit and self.save() or self

    def save(self, commit=True):
        """Save the record."""
        db.session.add(self)  # pylint: disable=E1101
        if commit:
            db.session.commit()  # pylint: disable=E1101
        return self

    def delete(self, commit=True):
        """Remove the record from the database."""
        db.session.delete(self)  # pylint: disable=E1101
        return commit and db.session.commit()  # pylint: disable=E1101

    @classmethod
    def commit(cls):
        """Just commit any recent changes, useful if you plan to commit only after
        many changes that need to be done in a single transaction"""
        return db.session.commit()

    @classmethod
    def get_by_id(cls: Type[T], record_id) -> Optional[T]:
        """Get record by ID."""
        try:
            return cls.query.get(record_id)
        except DataError:
            # Wrong format/type of id returns None and does not break the session
            db.session.commit()
            return None

    @classmethod
    def get_by_ids(cls: Type[T], record_ids) -> List[T]:
        return cls.query.filter(cls.id.in_(record_ids)).all()

    def get_changed_attrs(self, **kwargs):
        update = {}
        for key, value in kwargs.items():
            try:
                existing_value = getattr(self, key)
            except AttributeError:
                continue
            else:
                if existing_value != value:
                    update[key] = value
        return update

    def update_if_changed(self, **kwargs):
        update = self.get_changed_attrs(**kwargs)
        if update:
            return self.update(**update)
        return self

    @classmethod
    def get_or_create(cls, keys: Dict, rest: Dict = None) -> Tuple[BaseModel, int]:
        """
        raises MultipleResultsFound
        """
        if rest is None:
            rest = dict()
        created = False
        instance = cls.get_one_by_keys(keys=keys)
        if instance is None:
            instance = cls.create(**keys, **rest)
            created = True
        return instance, created

    @classmethod
    def get_one_by_keys(cls, keys):
        return cls.query.filter_by(**keys).one_or_none()

    @classmethod
    def get_list(cls, keys: Dict = None, order_by: List = None) -> Iterable[BaseModel]:
        if keys is None:
            keys = {}
        query = cls.query.filter_by(**keys)
        if order_by:
            query = query.order_by(*order_by)
        return query.all()

    def update_or_create(self, data, keys: Dict = None):
        if keys:
            instance = self.get_one_by_keys(keys=keys)
        else:
            instance = self.get_by_id(
                tuple(getattr(self, k) for k in self._get_primary_key_manes())
            )
        if instance:
            instance.update(**data)
            return instance
        else:
            return self.save()


from collections import Counter

import numpy as np
from imblearn.over_sampling.base import BaseOverSampler
from sklearn.neighbors import NearestNeighbors

# Authors: Dominik Badora, Cezary Szczesny
class Adasyn(BaseOverSampler):

    def __init__(self, sampling_strategy="auto", n_neighbors=5, ratio=1.0):
        super().__init__(sampling_strategy=sampling_strategy)
        self.n_neighbors = n_neighbors
        self.ratio = ratio

    def _fit_resample(self, X, y):
        examplesInEachClass = Counter(y)
        maxNumberOfSample = max(examplesInEachClass.values())
        syntheticExampleList = []

        for classIdx in examplesInEachClass.keys():
            sampleIndexes = np.where(y == classIdx)[0]
            currentClassSamples = X[sampleIndexes]
            syntheticSamplesToGenerateInClass = int(self.ratio * (maxNumberOfSample - examplesInEachClass[classIdx]))

            if syntheticSamplesToGenerateInClass == 0:
                continue

            knn = NearestNeighbors(n_neighbors=self.n_neighbors).fit(currentClassSamples)
            knnOfExample = knn.kneighbors(currentClassSamples, return_distance=False)

            for classSample in range(syntheticSamplesToGenerateInClass):

                randomExample = np.random.randint(len(sampleIndexes))
                exampleRandNNeighbor = np.random.choice(knnOfExample[randomExample])
                diffrenceBetweenExampleAndNeighbor = currentClassSamples[exampleRandNNeighbor] - currentClassSamples[
                    randomExample]
                syntheticExample = currentClassSamples[
                                       randomExample] + np.random.rand() * diffrenceBetweenExampleAndNeighbor

                if examplesInEachClass[classIdx] < maxNumberOfSample:
                    syntheticExampleList.append(syntheticExample)
                    examplesInEachClass[classIdx] += 1
                    y = np.concatenate((y, np.full(1, classIdx)))
                    X = np.concatenate((X, np.array([syntheticExample])))
        return X, y