import time

import requests
from pprint import pprint
import pal
from CEX_Request_V2 import ByBitRequest

######### Bitget #########
def fetch_market_data(product_type):
    url = "https://api.bitget.com/api/mix/v1/market/contracts"
    params = {"productType": product_type}

    response = requests.get(url, params=params)
    response = response.json()
    list = response['data']
    symbol = []
    for i in list:
        symbol.append(i['symbolName'])

    return symbol

product_type = "umcbl"
bitget_symbol = fetch_market_data(product_type)
# pprint(data)

######### MEXC #########
# GET /api/v3/defaultSymbols
# 该API返回的数据都是支持限价订单的。
# GET /api/v3/exchangeInfo
# 如果API返回字段 "isSpotTradingAllowed "为真，并且 "orderTypes "包含 "MARKET"，这意味着支持市场订单。

###### Full Data ######
# base_url = 'https://api.mexc.com'
# endpoint = '/api/v3/exchangeInfo'
#
# headers = {
#     'api_key': pal.mexc_key
# }
#
# response = requests.get(f'{base_url}{endpoint}', headers=headers)
# data = response.json()
#
# spot_trading_pairs = [pair for pair in data['symbols'] if pair['isSpotTradingAllowed']]
#
# pprint(spot_trading_pairs)

###### Market Order Pairs ######

base_url = 'https://api.mexc.com'
endpoint = '/api/v3/exchangeInfo'

headers = {
    'api_key': pal.mexc_key
}

response = requests.get(f'{base_url}{endpoint}', headers=headers)
data = response.json()

desired_order_types = {'LIMIT', 'MARKET', 'LIMIT_MAKER'}

mexc_spot_trading_pair_symbols = [
    pair['symbol'] for pair in data['symbols']
    if pair['isSpotTradingAllowed'] and set(pair['orderTypes']) == desired_order_types
]

######### Bybit #########

bybit_api_key = pal.bybit_key
bybit_api_secret = pal.bybit_secret
bybit_request = ByBitRequest(bybit_api_key, bybit_api_secret)

response = bybit_request.get_tickers('spot')
symbol = response['result']['list']
# pprint(response)
# pprint(symbol)

# symbol['symbol']: print(symbol) for i in symbol

all_pair = [i['symbol'] for i in symbol]
USDC_pair = []
for i in all_pair:
    # if 'USDC' in i or 'USDT' in i:
    if 'USDC' in i:
        USDC_pair.append(i)

###### Compare ######
# print(f'Bitget: {bitget_symbol}')
print(f'MEXC: {mexc_spot_trading_pair_symbols}')
# print(f'Bybit: {USDC_pair}')

time.sleep(1234)
basecoins = ['USDT', 'USDC']

bitget_symbol = [s.replace(coin, '') for s in bitget_symbol for coin in basecoins if coin in s]
mexc_spot_trading_pair_symbols = [s.replace(coin, '') for s in mexc_spot_trading_pair_symbols for coin in basecoins if coin in s]
USDC_pair = [s.replace(coin, '') for s in USDC_pair for coin in basecoins if coin in s]

# print(f'Bitget: {bitget_symbol}')
# print(f'MEXC: {mexc_spot_trading_pair_symbols}')
# print(f'Bybit: {USDC_pair}')

set_1 = set(mexc_spot_trading_pair_symbols)
# set_2 = set(USDC_pair)
set_3 = set(bitget_symbol)

common = set_1 & set_3
common = list(common) #['SUI']

print(f'Common: {common}')

###### Testing ######

crypto/Cross Exchange Market Making/ETH_Deribit-Bybit-Bitget/Get Product Info.py


import os
import sys
import math
import re
import string
import random
import json

def learn(rootdir):
    dictionary = {}
    L = list()
    weights = {}
    max_iterations = 20
    for dirName, subdirList, fileList in os.walk(rootdir):
        for fname in fileList:
            if fname.endswith('.txt'):
                filename = (os.path.join(dirName, fname))
                L.append(filename)
                with open(filename, "r", encoding="latin1") as file:
                    dict_words = {}
                    for line in file:
                        words = line.split()
                        for word in words:
                            if word not in dict_words:
                                dict_words[word] = 1
                            else:
                                dict_words[word] += 1
                            if word not in weights:
                                weights[word] = 0
                dictionary[filename] = dict_words
    #print(L)
    #print(len(L))
    #print(dictionary)
    #print(weights)

    bias = 0
    gamma = 0
    for iter in range(0,max_iterations):

        random.shuffle(L)
        for path in L:
            alpha = 0
            #print(path)
            if path.split('/')[-2] == 'spam':
                gamma = 1
            else:
                gamma = -1
            for word in dictionary.get(path):

                alpha += (weights.get(word)*dictionary.get(path)[word])

            alpha += bias
            #print(alpha)
            y = alpha*gamma
            #print(y)
            if(y <= 0):
                for word in dictionary.get(path):
                    #print(word)
                    #print(gamma)
                    weights[word] += gamma*dictionary.get(path)[word]
                    #print(weights[word])
                bias += gamma
                #print(bias)
    #print(weights)
    #print(len(weights))
    print(bias)
    weights["per_model_bias"] = bias
    with open("per_model.txt",'w') as f:
        json.dump(weights,f)

if __name__ == "__main__":
    learn(sys.argv[1])


import asyncio
import os
import json
from concurrent.futures import ThreadPoolExecutor, as_completed
from fetch import fetch
from parser import parse_page
from urllib.parse import urlparse
from cytoolz import *

ROOT_DIR = os.path.dirname(os.path.abspath('.'))
error_pages = []

def len_parsed(domain):
    with open('../data/{}/parsed_data.json'.format(domain),'r') as f:
        return (domain, len(json.loads(f.read())['products']))

def is_domain(domain, url):
    return urlparse(url).netloc == domain

def get_domain_urls(domain, seed):
    return list(filter(partial(is_domain, domain), seed))

def extract_domain_name(domain):
    return domain[4:-7]

def write_domain_links(file_name, links):
    os.makedirs(os.path.dirname(file_name), exist_ok=True)
    with open(file_name, 'w') as f:
        f.writelines(links)

def write_results(payload, domain):
    result = os.path.join(ROOT_DIR, 'data', domain, 'parsed_data.json')
    with open(result, 'w') as f:
        f.writelines(json.dumps(payload))

async def fetch_and_parse(url, data):
    try:
        page = await fetch(url)
        if page:
            data.append(parse_page(page))
    except Exception as err:
        error_pages.append(url)
        print('Error: {}\nURL: {}\n'.format(err, url))

async def worker(targets, data, counter):
    tasks = asyncio.as_completed([fetch_and_parse(url, data) for url in targets])
    domain = extract_domain_name(urlparse(targets[0]).netloc)

    for task in tasks:
        await task

def main(pair):
    domain, targets = pair
    data = []
    policy = asyncio.get_event_loop_policy()
    policy.set_event_loop(policy.new_event_loop())
    loop = asyncio.get_event_loop()
    counter = 0

    print('[*] Fetching and Parsing targets for {} ...'.format(domain))
    loop.run_until_complete(worker(targets, data, counter))

    payload = {'products': data}
    print('[*] Writing results...')
    write_results(payload, extract_domain_name(domain))

    print('[*] New data is available for {}!'.format(domain))

if __name__ == '__main__':
    with open(os.path.join(ROOT_DIR, 'data', 'targets.txt'), 'r') as f:
        all_urls = set([url[:-1] for url in f.readlines()])

    all_domains = set([urlparse(url).netloc for url in all_urls])

    domain_filter1 = ['www.americanas.com.br', 'www.shoptime.com.br',
                      'www.submarino.com.br']

    domain_filter2 = ['www.casasbahia.com.br','www.extra.com.br',
                      'www.pontofrio.com.br', 'www.walmart.com.br',
                      'www.magazineluiza.com.br']

    #domain_filter = domain_filter1 + domain_filter2
    domain_filter = ['www.walmart.com.br']

    domains = [extract_domain_name(url) for url in domain_filter]
    links = [(domain, get_domain_urls(domain, all_urls)) for domain in domain_filter]

    # execute main for each shop
    with ThreadPoolExecutor(max_workers=4) as ex:
        tasks = as_completed([ex.submit(main, pair) for pair in links])

    with open('../data/error_urls', 'w') as f:
        f.writelines(error_pages)

    print('[*] Results:')
    print([len_parsed(domain) for domain in domains])

    # # write links in separate files
    # for link in links:
    #     domain_name = extract_domain_name(link[0])
    #     file_name = os.path.join(ROOT_DIR, 'data', domain_name, '{}_urls'.format(domain_name))
    #     write_domain_links(file_name, link[1])


"""Tests for hubspot_sync.task_helpers"""
import pytest

from ecommerce.factories import ProductFactory
from hubspot_sync.task_helpers import (
    sync_hubspot_deal,
    sync_hubspot_product,
    sync_hubspot_user,
)

pytestmark = pytest.mark.django_db

@pytest.fixture()
def mock_exception_log(settings, mocker):
    """Return a mocked log.exception object"""
    settings.MITOL_HUBSPOT_API_PRIVATE_TOKEN = "faketoken"
    return mocker.patch("hubspot_sync.task_helpers.log.exception")

@pytest.mark.parametrize("raise_exc", [True, False])
def test_sync_hubspot_deal(mocker, mock_exception_log, hubspot_order, raise_exc):
    """sync_hubspot_deal should call tasks.sync_contact_with_hubspot.applY_async and log any exception"""
    mock_sync = mocker.patch(
        "hubspot_sync.task_helpers.tasks.sync_deal_with_hubspot.apply_async",
        side_effect=(ConnectionError if raise_exc else None),
    )
    sync_hubspot_deal(hubspot_order)
    mock_sync.assert_called_once_with(args=(hubspot_order.id,), countdown=10)
    if raise_exc:
        mock_exception_log.assert_called_once_with(
            "Exception calling sync_deal_with_hubspot for order %d", hubspot_order.id
        )
    else:
        mock_exception_log.assert_not_called()

@pytest.mark.parametrize("raise_exc", [True, False])
def test_sync_hubspot_user(mocker, mock_exception_log, user, raise_exc):
    """sync_hubspot_user should call tasks.sync_contact_with_hubspot.delay and log any exception"""
    mock_sync = mocker.patch(
        "hubspot_sync.task_helpers.tasks.sync_contact_with_hubspot.delay",
        side_effect=(ConnectionError if raise_exc else None),
    )
    sync_hubspot_user(user)
    mock_sync.assert_called_once_with(user.id)
    if raise_exc:
        mock_exception_log.assert_called_once_with(
            "Exception calling sync_contact_with_hubspot for user %s", user.username
        )
    else:
        mock_exception_log.assert_not_called()

@pytest.mark.parametrize("raise_exc", [True, False])
def test_sync_hubspot_product(mocker, mock_exception_log, raise_exc):
    """sync_hubspot_product should call tasks.sync_product_with_hubspot.delay and log any exception"""
    mock_sync = mocker.patch(
        "hubspot_sync.task_helpers.tasks.sync_product_with_hubspot.delay",
        side_effect=(ConnectionError if raise_exc else None),
    )
    product = ProductFactory.build()
    sync_hubspot_product(product)
    mock_sync.assert_called_once_with(product.id)
    if raise_exc:
        mock_exception_log.assert_called_once_with(
            "Exception calling sync_product_with_hubspot for product %d", product.id
        )
    else:
        mock_exception_log.assert_not_called()


import psycopg2
from datetime import datetime

from config import *
import utils
import sys

class DBOperator:
    def __init__(self,):
        # Connect to database
        try:
            self.conn = psycopg2.connect("dbname=%s host=%s user=%s password=%s"
                                         % (db_name, db_host, db_user, db_password))
        except:
            print ("Unable to connect to database: " + db_name)
            sys.exit()
        self.cursor = self.conn.cursor()
        self.conn.set_session(autocommit=True)

    def get_gsb_queryable_slds(self,):
        self.cursor.execute("""
            SELECT distinct sld FROM slds WHERE sld NOT IN
                (SELECT domain FROM gsb WHERE last_flag IS TRUE)
            """)
        return [x[0] for x in self.cursor.fetchall()]

    def get_seed_urls(self,):
        self.cursor.execute("""
            SELECT seed_url_id,seed_url FROM seed_urls WHERE has_permission_request=true and visit_status=1
            """)
        return [x for x in self.cursor.fetchall()]

    def get_gsb_queryable_urls(self,):
        self.cursor.execute("""
            SELECT distinct url FROM urls WHERE url NOT IN
                (SELECT domain FROM gsb WHERE last_flag IS TRUE)
            """)
        return [x[0] for x in self.cursor.fetchall()]

    def get_vt_scan_urls(self,):
        self.cursor.execute("""
            SELECT distinct url FROM urls WHERE url NOT IN
                (SELECT url FROM vt_results)
            """)
        return [x[0] for x in self.cursor.fetchall()]

    def get_vt_check_urls(self,):
        self.cursor.execute("""
            SELECT distinct url FROM urls WHERE url NOT IN
                (SELECT url FROM vt_results WHERE last_flag IS TRUE)
            """)
        return [x[0] for x in self.cursor.fetchall()]

    def update_gsb_table(self, sld, result, query_time):
        flag = (result != "None")
        se_flag = flag and ("SOCIAL_ENGINEERING" in result)
        text = result if flag else ""
        self.cursor.execute("""
            SELECT * FROM gsb WHERE domain = %s""", (sld,))
        if self.cursor.rowcount == 0:
            self.cursor.execute(""" INSERT INTO gsb (first_query_time, first_flag,
                                first_se_flag, first_result, domain) VALUES
                                (%s, %s, %s, %s, %s)""",
                                (query_time, flag, se_flag, text, sld))
        self.cursor.execute(""" UPDATE gsb SET last_query_time = %s,
                            last_flag = %s,
                            last_se_flag = %s,
                            last_result = %s WHERE domain = %s""",
                            (query_time, flag, se_flag, text, sld))

    def update_vt_table(self, result, query_time):
        flag = result['positive']>0
        se_flag = result['positive']>0

        self.cursor.execute("""
            SELECT * FROM vt_results WHERE url = %s""", (result['url'],))
        if self.cursor.rowcount == 0:
            self.cursor.execute(""" INSERT INTO vt_results (first_query_time, first_flag,
                                 first_result, url, vt_link,positive,total,status) VALUES
                                (%s, %s, %s, %s, %s, %s, %s, %s)""",
                                (query_time, flag, result['text'], result['url'], result['vt_link'], result['positive'],result['total'], result['status'] ))
        self.cursor.execute(""" UPDATE vt_results SET last_query_time = %s,
                            last_flag = %s,
                            last_result = %s ,
                            positive = %s,
                            total = %s
                            WHERE url = %s""",
                            (query_time, se_flag, result['text'],result['positive'],result['total'],result['url']))

    def bye(self,):
        self.cursor.close()
        self.conn.close()

    def update_urls_table(self, id, url, campaign, url_type, domain, url_path):
        self.cursor.execute("""
            SELECT count FROM urls WHERE domain = %s
            AND url_path = %s AND campaign = %s""", (domain, url_path, campaign))
        if self.cursor.rowcount == 0:
            self.cursor.execute(
                """
                INSERT INTO urls (site_id, url, domain, url_path, class,
                                  campaign, count)
                VALUES (%s, %s, %s, %s, %s, %s, 1)""",
                (id, url, domain, url_path, url_type, campaign))
        else:
            self.cursor.execute("""
                UPDATE urls SET url = %s, count = count + 1
                WHERE domain = %s AND url_path = %s AND
                campaign = %s
                """, (url, domain, url_path, campaign))

    def update_domains_seen_table(self, campaign, domain, time_seen):
        self.cursor.execute("""
            SELECT * FROM domains_seen WHERE domain = %s AND
            campaign = %s
            """, (domain, campaign))
        if self.cursor.rowcount == 0:
            self.cursor.execute(
                """
                INSERT INTO domains_seen (domain, campaign,
                                  first_seen, last_seen)
                VALUES (%s, %s, %s, %s)""",
                (domain, campaign, time_seen, time_seen))
        else:
            self.cursor.execute("""
                UPDATE domains_seen SET last_seen = %s
                WHERE domain = %s AND campaign = %s
                """, (time_seen, domain, campaign))

    def update_slds_table(self, domain):
        self.cursor.execute("""
            SELECT domain FROM slds WHERE domain = %s
            """, (domain,))
        if self.cursor.rowcount == 0:
            sld = utils.get_sld(domain)
            self.cursor.execute(
                """
                INSERT INTO slds (sld, domain)
                VALUES (%s, %s)""",
                (sld, domain))

    def update_campaigns_table(self, campaign, last_seen):
        self.cursor.execute("""
            SELECT * FROM campaigns WHERE campaign = %s
            """, (campaign,))
        if self.cursor.rowcount == 0:
            self.cursor.execute(
                """
                INSERT INTO campaigns (campaign, last_seen)
                VALUES (%s, %s)""",
                (campaign, last_seen))
        else:
            self.cursor.execute(
                """
                UPDATE campaigns SET last_seen = %s
                WHERE campaign = %s
                """, (last_seen, campaign))

    # Note we only insert a new url if it has a different
    # domain or url path. If both are same, we just update
    # an old entry even if the whole URL is different from the
    # stored one.
    def insert_url(self,id, url, campaign, url_type):
        domain, url_path = utils.split_url(url)
        time_seen = datetime.now()
        self.update_urls_table(id, url, campaign, url_type, domain, url_path)
        self.update_domains_seen_table(campaign, domain, time_seen)
        self.update_slds_table(domain)
        self.update_campaigns_table(campaign, time_seen)

    def insert_notification(self, notification_obj):
        try:
            self.cursor.execute(
                    """
                    INSERT INTO notification_details (sw_url_id,notification_title, notification_body,notification_count, target_url, image_url, sw_url, timestamp)
                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
                    (notification_obj['log_id'],
                    notification_obj['push_notification_title'] ,
                    notification_obj['push_notification_body'] ,
                    notification_obj['notification_count'] ,
                    notification_obj['push_notification_target_url'] ,
                    notification_obj['push_notification_image'] ,
                    notification_obj['frame_url'],
                    notification_obj['timestamp'] ))
            if self.cursor.rowcount == 1:
                return True
        except Exception as e:
            print(e)
            return False
        return False

    def insert_request(self, req_obj):
        try:
            self.cursor.execute(
                    """
                    INSERT INTO page_requests(sw_url_id,frame_url,local_url,request_url,timestamp)
                    VALUES (%s, %s, %s, %s,%s)""",
                    (req_obj['log_id'],
                    req_obj['frame_url'],
                    req_obj['local_frame_root_url'],
                    req_obj['url'],
                    req_obj['timestamp'] ))
            if self.cursor.rowcount == 1:
                return True
        except Exception as e:
            print(e)
            return False
        return False

    def insert_logs(self,iteration, logs_obj):
        try:
            self.cursor.execute(
                    """
                    INSERT INTO desktop_detailed_logs (url_id, iteration, info, url, target_url, landing_url, timestamp)
                    VALUES (%s, %s, %s, %s, %s, %s, %s)""",
                    (logs_obj['log_id'],
                    iteration,
                    logs_obj['info'] ,
                    logs_obj['url'] ,
                    logs_obj['target_url'] ,
                    logs_obj['landing_url'] ,
                    logs_obj['timestamp'] ))
            if self.cursor.rowcount == 1:
                return True
        except Exception as e:
            print(e)
            return False
        return False

    def insert_mobile_logs(self, logs_obj):
        try:
            self.cursor.execute(
                    """
                    INSERT INTO detailed_logs (pid, info, url, target_url, landing_url, timestamp)
                    VALUES (%s, %s, %s, %s, %s, %s)""",
                    (logs_obj['pid'],
                    logs_obj['info'] ,
                    logs_obj['url'] ,
                    logs_obj['target_url'] ,
                    logs_obj['landing_url'] ,
                    logs_obj['timestamp'] ))
            if self.cursor.rowcount == 1:
                return True
        except Exception as e:
            print(e)
            return False
        return False

    def insert_resource_info(self, id, iteration, file_name, file_hash, contacted_urls):
        try:
            self.cursor.execute(
                    """
                    INSERT INTO resource_info (url_id, iteration, file_name, file_hash, contacted_urls)
                    VALUES (%s, %s, %s, %s, %s)""",
                    (id,
                    iteration,
                    file_name ,
                    file_hash ,
                    contacted_urls))
            if self.cursor.rowcount == 1:
                return True
        except Exception as e:
            print(e)
            return False
        return False

    def insert_service_wroker_event(self, sw_obj):
        try:
            self.cursor.execute(
                    """
                    INSERT INTO service_worker_details (sw_url_id,sw_url, target_url, timestamp, sw_event)
                    VALUES (%s, %s, %s, %s, %s)""",
                    (sw_obj['log_id'],
                    sw_obj['sw_url'] ,
                    sw_obj['target_url'] ,
                    sw_obj['timestamp'] ,
                    sw_obj['info']  ))
            if self.cursor.rowcount == 1:
                return True
        except Exception as e:
            print(e)
            return False
        return False

def test():
    dbo = DBOperator()
    dbo.insert_url('http://www.amazon.esp.bravaidea.com/AWS/mobil/signin.php?https://www.amazon.com/gp/product/B00DBYBNEE/ref=nav_prime_try_btn','','other')
    print (dbo.get_gsb_queryable_slds())

if __name__ == "__main__":
    test()


from django.conf.urls import url
from . import views

app_name = 'members'

urlpatterns = [
    url(r'^adhesions/$', views.AdhesionsView.as_view(), name='adhesions'),
    url(r'^adhesions/data/$', views.AdhesionsJsonView.as_view(), name='adhesions_data'),
    url(r'^tranches/$', views.TranchesView.as_view(), name='tranches'),
    url(r'^tranches/data/$', views.TranchesJsonView.as_view(), name='tranches_data'),
]


import urllib.request, json
from urllib.error import URLError

from src.model.WeatherModel import WeatherModel
from src.constants.GenericConstants import base_weather_url, owm_key, imperial_countries

class RetrieveLocationBasedWeatherInfo:
    def __init__(self):
        self.weather_data = WeatherModel()

    def get_weather(self, geo_coordinates) -> WeatherModel:
        """
        This method takes geographical coordinates & calls the open-weather api to determine the weather conditions for
        a given set of geo-coordinates.
        @:param - latitude, longitude
        @:return - current_temperature, weather_condition, icon_id
        """
        # Build the location based url
        weather_url = f'{base_weather_url}lat={geo_coordinates.latitude}&lon={geo_coordinates.longitude}&appid={owm_key}'
        try:
            with urllib.request.urlopen(weather_url) as api_request:
                api_response = api_request.read().decode('utf-8')
                response_data = json.loads(api_response)
            api_request.close()
        except (ConnectionError, URLError) as ce:
            response_data = {'Error': 'Cannot connect to internet at the moment: {}'.format(ce)}
        except (ValueError, TypeError) as e:
            response_data = {'Error': 'Cannot interpret the response due to data or type mismatch: {}'.format(e)}

        # Prepare the final model object from the response json
        self.prepare_weather_data(response_data)
        return self.weather_data

    def prepare_weather_data(self, response_data) -> None:
        if 'Error' not in response_data.keys():
            # country
            self.weather_data.country = response_data['sys']['country']

            # icon-code for current conditions
            self.weather_data.icon_id = response_data['weather'][0]['icon']

            # wind
            self.weather_data.wind_speed = response_data['wind']['speed']
            self.weather_data.wind_degree = response_data['wind']['deg']

            # current conditions
            current_condition_desc = response_data['weather'][0]['description']
            # .title() capitalizes the 1st character of each word
            self.weather_data.condition_description = current_condition_desc.title()

            current_temperature = response_data['main']['temp']
            self.weather_data.general_temperature = self.scale_temperature(current_temperature, self.weather_data)
        else:
            current_temperature = 420.00
            self.weather_data.general_temperature = current_temperature

    def scale_temperature(self, current_temp, weather_data) -> float:
        """
        Helper method to convert the temperature from Kelvin Scale to Fahrenheit or Celsius depending on the metric
        system the given country uses.
        :param - WeatherModel
        :param - float (Kelvin)
        :return - float (Fahrenheit/Celsius)
        """
        if weather_data.country in imperial_countries:
            return (current_temp * (9 / 5)) - 459.67
        else:
            return current_temp - 273


"""
The code in this file is based on https://github.com/ml-jku/hopfield-layers.
This repo accompanies Ramsauer et al. (2020) (https://arxiv.org/abs/2008.02217).
"""
import numpy as np
from itertools import product
from tqdm import tqdm
from scipy.special import softmax
import math

class DenseHopfield:
    def __init__(self, pat_size, beta=1, normalization_option=1):
        self.size = pat_size
        self.beta = beta
        self.max_norm = np.sqrt(self.size)
        if normalization_option == 0:
            self.energy = self.energy_unnormalized
        elif normalization_option == 1: # normalize dot product of patterns by 1/sqrt(pattern_size)
            self.energy = self.energy_normalized
        elif normalization_option == 2: # normalize dot product of patterns by shifting and clamping low exponentials
            self.energy = self.energy_normalized2
        else:
            raise ValueError('unkown option for normalization: %d'% normalization_option)

        return

    def learn(self, patterns):
        """expects patterns as numpy arrays and stores them col-wise in pattern matrix
        """
        self.num_pat = len(patterns)
        assert(all(type(x) is np.ndarray for x in patterns)), 'not all input patterns are numpy arrays'
        assert(all(len(x.shape) == 2 for x in patterns)), 'not all input patterns have dimension 2'
        assert(all(1 == x.shape[1] for x in patterns)), 'not all input patterns have shape (-1,1) '
        self.patterns = np.array(patterns).squeeze(axis=-1).T # save patterns col-wise
        # without squeeze axis would result in problem with one pattern
        self.max_pat_norm = max(np.linalg.norm(x) for x in patterns)

    def retrieve(self, partial_pattern, max_iter=np.inf, thresh=0.5):
        # partial patterns have to be provided with None/0 at empty spots
        if partial_pattern.size != self.size:
            raise ValueError("Input pattern %r does not match state size: %d vs %d"
                %(partial_pattern, len(partial_pattern), self.size))

        if None in partial_pattern:
            raise NotImplementedError("None elements not supported")

        assert type(partial_pattern) == np.ndarray, 'test pattern was no numpy array'
        assert len(partial_pattern.shape) <=2 and 1 == partial_pattern.shape[1], 'test pattern with shape %r is not a col-vector' %(partial_pattern.shape,)

        pat_old = partial_pattern.copy()
        iters = 0

        for iters in tqdm(range(max_iter)):
            pat_new = np.zeros(partial_pattern.shape)
            # jj = np.random.randint(self.size)
            for jj in range(self.size):
                # simple variant:
                E = 0
                temp = pat_old[jj].copy()
                pat_old[jj] = +1
                E -= self.energy(pat_old)
                pat_old[jj] = -1
                E += self.energy(pat_old)

                pat_old[jj] = temp
                pat_new[jj] = np.where(E >0 , 1, -1)

            if np.count_nonzero(pat_old != pat_new)<= thresh:
                break
            else:
                pat_old = pat_new

        return pat_new

    @staticmethod
    def _lse(z, beta):
        return 1/beta * np.log(np.sum(np.exp(beta*z)))

    def energy_unnormalized(self, pattern):
        # return -1*np.exp(self._lse(self.patterns.T @pattern, beta=self.beta))
        # this is equal, but faster
        return -1*np.sum(np.exp(self.patterns.T @pattern ))

    def energy_normalized(self, pattern):
        # normalize dot product of patterns by 1/sqrt(pattern_size)
        return -1*np.sum(np.exp((self.patterns.T @pattern)/self.max_norm ))

    def energy_normalized2(self, pattern):
        # normalize dot product of patterns by shifting by -sqrt(pattern_size)
        # also clamp exponential for exponents smaller then -73 to 0
        exponents = self.patterns.T @pattern
        norm_exponents = exponents - self.max_pat_norm
        norm_exponents[norm_exponents < -73] = -np.inf

        return -1*np.sum(np.exp(norm_exponents))

    def energy_landscape(self):
        for pat in product([1,-1], repeat=self.size):
            pat = np.array(pat)
            print("energy(%r)=%.3f"%(pat, self.energy(pat)))

class ContinuousHopfield:
    def __init__(self, pat_size, beta=1, do_normalization=True):
        self.size = pat_size # size of individual pattern
        self.beta = beta
        print(self.beta)
        self.max_norm = np.sqrt(self.size)
        if do_normalization:
            self.softmax = self.softmax_normalized
            self.energy = self.energy_normalized
        else:
            self.softmax = self.softmax_unnormalized
            self.energy = self.energy_unnormalized

        return

    def learn(self, patterns):
        """expects patterns as numpy arrays and stores them col-wise in pattern matrix
        """
        self.num_pat = len(patterns)
        assert(all(type(x) is np.ndarray for x in patterns)), 'not all input patterns are numpy arrays'
        assert(all(len(x.shape) == 2 for x in patterns)), 'not all input patterns have dimension 2'
        assert(all(1 == x.shape[1] for x in patterns)), 'not all input patterns have shape (-1,1) '
        self.patterns = np.array(patterns).squeeze(axis=-1).T # save patterns col-wise
        # without squeeze axis would result in problem with one pattern
        # return -1*np.sum(np.exp([(self.patterns[:,ii].T @pattern)/self.max_norm for ii in range(self.patterns.shape[1])]))
        self.M = max(np.linalg.norm(vec) for vec in patterns)# maximal norm of actually stored patterns
        return

    def retrieve(self, partial_pattern, max_iter=np.inf, thresh=0.5):
        # partial patterns have to be provided with None/0 at empty spots
        if partial_pattern.size != self.size:
            raise ValueError("Input pattern %r does not match state size: %d vs %d"
                %(partial_pattern, len(partial_pattern), self.size))

        if None in partial_pattern:
            raise NotImplementedError("None elements not supported")

        assert type(partial_pattern) == np.ndarray, 'test pattern was not numpy array'
        assert len(partial_pattern.shape) ==2 and 1 == partial_pattern.shape[1], 'test pattern with shape %r is not a col-vector' %(partial_pattern.shape,)

        pat_old = partial_pattern.copy()
        iters = 0

        while iters < max_iter:
            pat_new = self.patterns @ self.softmax(self.beta*self.patterns.T @ pat_old)

            if np.count_nonzero(pat_old != pat_new)<= thresh: # converged
                break
            else:
                pat_old = pat_new
            iters += 1

        return pat_new

    def softmax_unnormalized(self, z):
        return softmax(z)  # Scipy's softmax is numerically stable

    def softmax_normalized(self, z):
        return softmax(z / self.max_norm)

    @staticmethod
    def _lse(z, beta):
        return 1/beta * np.log(np.sum(np.exp(beta*z)))

    def energy_unnormalized(self, pattern):
        return -1*self._lse(self.patterns.T @pattern, 1) + 0.5 * pattern.T @ pattern\
            + 1/self.beta*np.log(self.num_pat)\
            + 0.5*self.M**2

    def energy_normalized(self, pattern):
        # normalize dot product of patterns by 1/sqrt(pattern_size)
        return -1*self._lse((self.patterns.T @pattern )/self.max_norm, 1) + 0.5 * pattern.T @ pattern\
            + 1/self.beta*np.log(self.num_pat)\
            + 0.5*self.M**2

    def energy_landscape(self):
        for pat in product([1,-1], repeat=self.size):
            pat = np.array(pat)
            print("energy(%r)=%.3f"%(pat, self.energy(pat)))


import pygame
import time

class UI:
    def last_zone(self):
        return self.zones[self.last_zone_name]

    def add_zone(self,title,name):
        new_y = self.last_zone().y + self.last_zone().get_height() + title.get_height() / 2 if len(self.zones) else 5
        z = Zone(self.x, new_y, title)
        self.zones.update( { name : z })
        self.last_zone_name = name

    def create_buttons(self, game, team):
        self.panel_buttons.clear()
        cell_size = 150
        for i in range(4):
            id = i + 2 + 10 * (team + 1)
            sprite = game.board.GetPiece(id)(0,0,team).sprite
            self.panel_buttons.append(Button(self.panel.x + i * cell_size + 5, self.panel.y + 5, cell_size,cell_size, sprite, id))

    def create_panel(self, game):
        panel_width, panel_height = 150*4+10, 160
        board_rect = pygame.Rect(5,5, game.board.w, game.board.h)
        panel_rect = pygame.Rect(0,0, panel_width, panel_height)
        panel_rect.center = board_rect.center
        self.panel = panel_rect

        self.add_zone(self.titleFont.render("Chess", 1, (15,15,15)), "Turn")

        text = "'Turn: {}'.format('White' if game.board.playingTeam else 'Black')"
        self.zones["Turn"].add_info_text(self.turnFont, text, game)

        self.add_zone(self.titleFont.render("Score", 1, (15,15,15)), "Score")

        text = "'White: {}'.format(game.board.whiteScore)"
        self.zones["Score"].add_info_text(self.turnFont, text, game)

        text = "'Black: {}'.format(game.board.blackScore)"
        self.zones["Score"].add_info_text(self.turnFont, text, game)

        self.add_zone(self.titleFont.render("Time", 1, (15,15,15)), "Time")

        text = "'{}'.format(time.strftime('%M:%S', time.gmtime(game.board.elapsedTime)))"
        self.zones["Time"].add_info_text(self.timeFont, text, game)

        self.add_zone(self.titleFont.render("Last Moves", 1, (15,15,15)), "Last Moves")

    def draw(self,game):
        pygame.draw.rect(game.win, (200,200,200), (self.x, self.y, self.w, self.h)) #BACKGROUND

        for k,v in self.zones.items(): v.draw(game)

        if game.board.player_in_panel:
            pygame.draw.rect(game.win, (255,255,255), (self.panel.x, self.panel.y, self.panel.w, self.panel.h))
            [btn.draw(game) for btn in self.panel_buttons]

    def add_last_move(self, game,msg):
        if len(self.zones["Last Moves"].info_texts) >= game.board.max_last_moves: self.zones["Last Moves"].remove(0)
        self.zones["Last Moves"].add_info_text(self.turnFont, msg.msg, game, True, (255,255,255) if msg.team else (0,0,0))

    def __init__(self, x,y,w,h):
        self.x, self.y, self.w, self.h = x,y,w,h
        self.titleFont = pygame.font.SysFont("Arial", 22)
        self.turnFont = pygame.font.SysFont("Arial", 18)
        self.timeFont = pygame.font.SysFont("Arial", 28)
        self.panel = None
        self.panel_buttons = []
        self.zones = {}
        self.last_zone_name = ""

class Button:
    def __init__(self, x,y,w,h,sprite, id):
        self.x,self.y,self.w,self.h = x,y,w,h
        self.sprite = pygame.transform.scale(sprite, (self.w, self.h))
        self.is_hovered = False
        self.id = id

    def set_hovered(self,game):
        self.is_hovered = self.x + self.w > game.mouse_position[0] > self.x and self.y + self.h > game.mouse_position[1] > self.y

    def check_clicked(self, game):
        return self.is_hovered and game.mouse_button_down

    def draw(self, game):
        self.set_hovered(game)
        pygame.draw.rect(game.win, (0,0,0), (self.x,self.y,self.w,self.h), 1) #DRAW BORDER
        if self.is_hovered:
            pygame.draw.rect(game.win, (0,255,0), (self.x + 1, self.y + 1, self.w - 2, self.h - 2))
        game.win.blit(self.sprite, (self.x, self.y))

class Zone:
    def __init__(self, x,y,title):
        self.x, self.y = x,y
        self.title = title
        self.info_texts = []
        self.y_offset = 5
        self.bar_offset = 5

    def draw(self, game):
        game.win.blit(self.title, (self.x + game.ui.w / 2- self.title.get_width() / 2, self.y))
        for t in self.info_texts:
            t.update_text(game)
            t.draw(game)
        self.draw_bar(game)

    def add_info_text(self, font,condition, game, static = False, color = (15,15,15)):
        last_y = self.info_texts[-1].y if len(self.info_texts) else self.y
        text_offset = self.info_texts[-1].get_height() if len(self.info_texts) else self.title.get_height()
        next_y = last_y + self.y_offset + text_offset
        info_t = info_text(self.x,next_y, font, condition, game, static, color)
        self.info_texts.append(info_t)

    def get_height(self):
        total = self.title.get_height()
        for i in self.info_texts:
            total += i.text.get_height() + self.y_offset
        return (total + self.bar_offset + 2)

    def draw_bar(self, game):
        pygame.draw.line(game.win, (15,15,15), (self.x, self.y + self.get_height() + self.bar_offset),(self.x + game.ui.w, self.y + self.get_height() + self.bar_offset), 2)

    def remove(self, id):
        sy = self.info_texts[id].get_height()
        for i in self.info_texts:
            i.y -= (sy + self.y_offset)
        del self.info_texts[id]

class info_text:
    def __init__(self, x,y, font,condition, game, static, color):
        self.x, self.y = x,y
        self.static = static
        self.freeze = False
        self.color = color
        self.text = ""
        self.condition = condition
        self.font = font
        self.update_text(game)

    def draw(self, game):
        game.win.blit(self.text, (self.x, self.y))

    def get_height(self):
        return self.text.get_height()

    def get_text(self, game):
        return str(eval(self.condition)) if not self.static else self.condition

    def update_text(self, game):
        if not self.freeze:
            self.text = self.font.render(self.get_text(game), 1, self.color)


from collections import Counter
from dataclasses import dataclass
from functools import cached_property
from textwrap import dedent

import aoc

test_input = dedent("""\
    0,9 -> 5,9
    8,0 -> 0,8
    9,4 -> 3,4
    2,2 -> 2,1
    7,0 -> 7,4
    6,4 -> 2,0
    0,9 -> 2,9
    3,4 -> 1,4
    0,0 -> 8,8
    5,5 -> 8,2\
""")

def parse(data):
    @dataclass(frozen=True)
    class Point:
        x: int
        y: int

    class Line:
        def __init__(self, origin: Point, dest: Point):
            self.origin = origin
            self.dest = dest

            self.x_step = self.get_step(origin.x, dest.x)
            self.y_step = self.get_step(origin.y, dest.y)

        @staticmethod
        def get_step(v1, v2):
            if v1 < v2:
                return 1
            elif v1 > v2:
                return -1
            else:
                return 0

        @cached_property
        def points(self):
            def iter_points():
                p = self.origin
                while p != self.dest:
                    yield p
                    p = Point(x=p.x + self.x_step,
                              y=p.y + self.y_step)
                yield self.dest

            return list(iter_points())

    lines = []
    for input_line in data.strip().splitlines():
        input_line = input_line.strip()
        left, right = input_line.split(" -> ")
        x1, y1 = [int(n) for n in left.split(",")]
        x2, y2 = [int(n) for n in right.split(",")]
        origin = Point(x1, y1)
        dest = Point(x2, y2)
        line = Line(origin, dest)
        lines.append(line)

    return lines

def part1(data):
    lines = parse(data)

    covered_points = Counter()

    for line in lines:
        if line.origin.x != line.dest.x and line.origin.y != line.dest.y:
            continue

        for point in line.points:
            covered_points[point] += 1

    return sum(1 for x in covered_points.values() if x > 1)

def part2(data):
    lines = parse(data)

    covered_points = Counter()

    for line in lines:
        for point in line.points:
            covered_points[point] += 1

    return sum(1 for x in covered_points.values() if x > 1)

def test_part1():
    assert 5 == part1(test_input)

def test_part2():
    assert 12 == part2(test_input)

if __name__ == '__main__':
    data = aoc.read_input(__file__)
    print("Part 1", part1(data))
    print("Part 2", part2(data))


import sys
import os
import time
import numpy as np
import argparse

import torch
import torchvision
import torch.nn as nn
import torch.optim as optim
from torch.optim.lr_scheduler import StepLR
from torch.utils.tensorboard import SummaryWriter
from torch.nn.parallel import DistributedDataParallel
import torch.multiprocessing

import logging
from utils import logging_utils
logging_utils.config_logger()
from utils.YParams import YParams
from utils.data_loader import get_data_loaders_distributed
from utils.plotting import generate_images, PDF_and_metrics
from networks import UNet

def adjust_LR(optimizer, params, iternum):
  """Piecewise constant rate decay"""
  if params.distributed and iternum<5000:
    lr = params.lr*(iternum/5000.) #warmup for distributed training

  elif iternum<params.LRsched[0]:
    lr = params.lr
  elif iternum>params.LRsched[1]:
    lr = params.lr/4.
  else:
    lr = params.lr/2.
  for param_group in optimizer.param_groups:
    param_group['lr'] = lr

def train(params, args, world_rank):
  logging.info('rank %d, begin data loader init'%world_rank)
  train_data_loader, test_data_loader = get_data_loaders_distributed(params, world_rank)
  logging.info('rank %d, data loader initialized'%world_rank)
  model = UNet.UNet(params).cuda()
  if not args.resuming:
    model.apply(model.get_weights_function(params.weight_init))

  optimizer = optim.Adam(model.parameters(), lr = params.lr)
  if params.distributed:
    model = DistributedDataParallel(model)

  iters = 0
  startEpoch = 0
  checkpoint = None
  if args.resuming:
    if world_rank==0:
      logging.info("Loading checkpoint %s"%params.checkpoint_path)
    checkpoint = torch.load(params.checkpoint_path, map_location='cuda:{}'.format(args.local_rank))
    model.load_state_dict(checkpoint['model_state'])
    iters = checkpoint['iters']
    startEpoch = checkpoint['epoch'] + 1
    optimizer.load_state_dict(checkpoint['optimizer_state_dict'])

  if world_rank==0:
    logging.info(model)
    logging.info("Starting Training Loop...")

  device = torch.cuda.current_device()
  if params.LRsched:
      step, decay = params.LRsched[0], params.LRsched[1]
      scheduler = StepLR(optimizer, step_size=step, gamma=decay)
  for epoch in range(startEpoch, startEpoch+params.num_epochs):
    start = time.time()
    tr_time = 0.
    log_time = 0.

    trainloss = []
    for i, data in enumerate(train_data_loader, 0):
      iters += 1
      inp, tar = map(lambda x: x.to(device), data)
      tr_start = time.time()

      model.zero_grad()
      gen = model(inp)
      loss = UNet.loss_func(gen, tar, params)
      loss.backward()
      optimizer.step()

      tr_end = time.time()
      tr_time += tr_end - tr_start
      trainloss.append(loss.item())
      if params.LRsched:
        scheduler.step()
        if iters%params.LRsched[0] == 0:
          logging.info('LR decayed to %f'%optimizer.param_groups[-1]['lr'])

    # Output training stats
    if world_rank==0:
      log_start = time.time()
      gens = []
      tars = []
      with torch.no_grad():
        for i, data in enumerate(test_data_loader, 0):
          inp, tar = map(lambda x: x.to(device), data)
          gen = model(inp)
          gens.append(gen.detach().cpu().numpy())
          tars.append(tar.detach().cpu().numpy())
      gens = np.concatenate(gens, axis=0)
      tars = np.concatenate(tars, axis=0)

      # Tensorboard
      args.tboard_writer.add_scalar('Loss/train', np.mean(trainloss), iters)
      args.tboard_writer.add_scalar('Loss/test', np.mean(np.abs(gens - tars)), iters)

      fig, chi, KLscore = PDF_and_metrics(tars, gens, params)
      args.tboard_writer.add_figure('pixhist', fig, iters, close=True)
      args.tboard_writer.add_scalar('Metrics/chi', chi, iters)
      args.tboard_writer.add_scalar('Metrics/KL', KLscore, iters)
      fig = generate_images(inp[-1].detach().cpu().numpy(), gens[-1], tars[-1], params.diff_precision)
      args.tboard_writer.add_figure('genimg', fig, iters, close=True)
      log_end = time.time()
      log_time += log_end - log_start

      # Save checkpoint
      torch.save({'iters': iters, 'epoch':epoch, 'model_state': model.state_dict(),
                  'optimizer_state_dict': optimizer.state_dict()}, params.checkpoint_path)

    end = time.time()
    if world_rank==0:
        logging.info('Time taken for epoch {} is {} sec'.format(epoch + 1, end-start))
        logging.info('train step time={}, logging time={}'.format(tr_time, log_time))

if __name__ == '__main__':

  parser = argparse.ArgumentParser()
  parser.add_argument("--local_rank", default=0, type=int)
  parser.add_argument("--run_num", default='00', type=str)
  parser.add_argument("--yaml_config", default='./config/UNet.yaml', type=str)
  parser.add_argument("--config", default='default', type=str)
  args = parser.parse_args()

  run_num = args.run_num

  params = YParams(os.path.abspath(args.yaml_config), args.config)

  params.distributed = False
  if 'WORLD_SIZE' in os.environ:
    params.distributed = int(os.environ['WORLD_SIZE']) > 1

  world_rank = 0
  if params.distributed:
    torch.cuda.set_device(args.local_rank)
    torch.distributed.init_process_group(backend='nccl',
                                         init_method='env://')
    args.gpu = args.local_rank
    world_rank = torch.distributed.get_rank()

  torch.backends.cudnn.benchmark = True

  args.resuming = False

  # Set up directory
  baseDir = './expts/'
  expDir = os.path.join(baseDir, args.config+'/'+str(run_num)+'/')
  if  world_rank==0:
    if not os.path.isdir(expDir):
      os.makedirs(expDir)
      os.mkdir(expDir+'training_checkpoints/')

    logging_utils.log_to_file(logger_name=None, log_filename=os.path.join(expDir, 'out.log'))
    params.log()
    args.tboard_writer = SummaryWriter(log_dir=os.path.join(expDir, 'logs/'))

  params.experiment_dir = os.path.abspath(expDir)
  params.checkpoint_path = os.path.join(params.experiment_dir, 'training_checkpoints/ckpt.tar')
  if os.path.isfile(params.checkpoint_path):
    args.resuming=True

  train(params, args, world_rank)
  if world_rank == 0:
    args.tboard_writer.flush()
    args.tboard_writer.close()
  logging.info('DONE ---- rank %d'%world_rank)


import os
from setuptools import find_packages, setup

from codecs import open

here = os.path.abspath(os.path.dirname(__file__))

with open('README.md', 'r') as fh:
    long_description = fh.read()

about = {}
with open(os.path.join(here, 'infynipy', '__version__.py'), 'r', 'utf-8') as f:
    exec(f.read(), about)

setup(
    name=about['__title__'],
    version=about['__version__'],
    author=about['__author__'],
    author_email=about['__author_email__'],
    description=about['__description__'],
    long_description=long_description,
    long_description_content_type='text/markdown',
    url=about['__url__'],
    license=about['__license__'],
    classifiers=[
        'License :: OSI Approved :: MIT License',
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
    ],
    install_requires=[
        'requests==2.20.0',
    ],
    packages=find_packages(exclude=["tests", "tests.*"]),
    setup_requires=['pytest-runner'],
    tests_require=[
        "pytest >=5.0.1",
        "python-dotenv >=0.10.3",
        "vcrpy >=2.0.1",
    ],
    test_suite="tests",
)


import streamlit as st
import matplotlib.pyplot as plt
import base64

def app():

    # page and sidebar background images
    main_bg = "background5.jpg"
    main_bg_ext = "jpg"
    side_bg = "background4.jpg"
    side_bg_ext = "jpg"
    st.markdown(f"""
        <style>
        .reportview-container {{
            background: url(data:image/{main_bg_ext};base64,{base64.b64encode(open(main_bg, "rb").read()).decode()})
        }}
        .css-17eq0hr {{
            background: url(data:image/{side_bg_ext};base64,{base64.b64encode(open(side_bg, "rb").read()).decode()})
        }}
        </style>
        """,
                unsafe_allow_html=True)

    st.title('About Us')
    st.write('SERSA Team')

    team = {
        '01': 'Filippo Colonna',
        '02': 'Ia Vaquilar',
        '03': 'Michael Michael',
        '04': 'Pankaj Patel'
    }

    fig = plt.figure()
    fig.subplots_adjust(hspace=0.4, wspace=0.4)
    for i in team.keys():
        filename = f'bios/team{i}.jpg'
        fig.add_subplot(2, 2, int(i))
        plt.title(team[i], fontsize=6)
        image = plt.imread(filename)
        plt.axis('off')
        plt.imshow(image)
    st.pyplot(fig)