import math

from file_utils import file_reader
from utils.IntPoint import IntPoint

def detect_asteroids(data):
    points = load_points(data)
    return find_all_in_sight(points)

def load_points(data):
    points = []
    for y, line in enumerate(data):
        for x, obj in enumerate(line):
            if obj == '#':
                points.append(IntPoint(x, y))
    return points

def find_all_in_sight(asteroids):
    max_in_sight = 0
    for asteroid in asteroids:
        to_check = asteroids.copy()
        angles = calculate_angles(to_check, asteroid)
        curr_in_sight = len(to_check)
        for k, v in angles.items():
            if len(v) > 1:
                curr_in_sight = curr_in_sight - (len(v) - 1)
        if curr_in_sight > max_in_sight:
            max_in_sight = curr_in_sight
            best_point = asteroid
    return [max_in_sight, best_point]

def is_on_the_same_line(first: IntPoint, second: IntPoint, test: IntPoint):
    return first.x * (second.y - test.y) + second.x * (test.y - first.y) + test.x * (first.y - second.y) == 0

def calculate_angles(asteroids: [IntPoint], middle: IntPoint):
    angles = {}
    asteroids.remove(middle)
    for point in asteroids:
        dx = middle.x - point.x
        dy = middle.y - point.y
        angle = math.atan2(-dx, dy)
        value = round(math.degrees(angle if angle >= 0 else 2 * math.pi + angle), 4)

        if value in angles:
            angles[value].append(point)
        else:
            angles[value] = [point]
    return angles

def destroy_asteroids(data, middle: IntPoint, elem_pos):
    angles = list(data.keys())
    angles.sort()

    for angle in angles:
        curr = list(data[angle])
        curr.sort(key=lambda p: abs(middle.x - p.x) + abs(middle.y - p.y))
        data[angle] = curr

    idx = 1
    angle_idx = 0
    while idx <= elem_pos:
        angle = angles[angle_idx]
        angle_points = list(data[angle])
        to_remove = angle_points.pop(0)
        if len(angle_points) == 0:
            angles.pop(angle_idx)
            data.pop(angle, None)
        else:
            data[angle] = angle_points
            angle_idx += 1

        if idx == elem_pos:
            return to_remove

        idx += 1
        if angle_idx == len(angles) - 1:
            angle_idx = 0

    raise Exception('no point at pos {}', elem_pos)

def get_removed_element_at_pos(data, pos):
    points = load_points(data)
    middle = find_all_in_sight(points)[1]
    angles = calculate_angles(points, middle)
    return destroy_asteroids(angles, middle, pos)

def solve_1():
    values = file_reader.read_str_from_file('input/day10_input.txt')
    return detect_asteroids(values)

def solve_2():
    values = file_reader.read_str_from_file('input/day10_input.txt')
    result = get_removed_element_at_pos(values, 200)
    return result.x * 100 + result.y

assert is_on_the_same_line(IntPoint(0, 0), IntPoint(5, 5), IntPoint(2, 2))
assert is_on_the_same_line(IntPoint(1, 1), IntPoint(1, 4), IntPoint(1, 5))
assert is_on_the_same_line(IntPoint(0, 0), IntPoint(5, 5), IntPoint(11, 11))
assert not is_on_the_same_line(IntPoint(0, 0), IntPoint(5, 5), IntPoint(11, 12))
assert not is_on_the_same_line(IntPoint(1, 5), IntPoint(2, 5), IntPoint(4, 6))

test_case_1 = '''.#..#
.....
#####
....#
...##'''

test_case_2 = '''......#.#.
#..#.#....
..#######.
.#.#.###..
.#..#.....
..#....#.#
#..#....#.
.##.#..###
##...#..#.
.#....####'''

test_case_3 = '''.#..##.###...#######
##.############..##.
.#.######.########.#
.###.#######.####.#.
#####.##.#.##.###.##
..#####..#.#########
####################
#.####....###.#.#.##
##.#################
#####.##.###..####..
..######..##.#######
####.##.####...##..#
.#####..#.######.###
##...#.##########...
#.##########.#######
.####.#.###.###.#.##
....##.##.###..#####
.#.#.###########.###
#.#.#.#####.####.###
###.##.####.##.#..##'''

test_case_4 = '''.#..#..###
####.###.#
....###.#.
..###.##.#
##.##.#.#.
....###..#
..#.#..#.#
#..#.#.###
.##...##.#
.....#.#..'''

# part1
result_1 = detect_asteroids(test_case_1.split('\n'))
assert 8 == result_1[0]
assert IntPoint(3, 4) == result_1[1]
result_2 = detect_asteroids(test_case_2.split('\n'))
assert 33 == result_2[0]
assert IntPoint(5, 8) == result_2[1]
result_3 = detect_asteroids(test_case_3.split('\n'))
assert 210 == result_3[0]
assert IntPoint(11, 13) == result_3[1]
result_4 = detect_asteroids(test_case_4.split('\n'))
assert 41 == result_4[0]
assert IntPoint(6, 3) == result_4[1]
assert 253 == solve_1()[0]

# part2
assert IntPoint(4, 3) == get_removed_element_at_pos(test_case_1.split('\n'), 4)
middle = IntPoint(11, 13)
points = [IntPoint(11, 12), IntPoint(12, 1), IntPoint(12, 2), middle, IntPoint(9, 6), IntPoint(8, 2),
          IntPoint(12, 8), IntPoint(16, 0), IntPoint(16, 9), IntPoint(10, 16)]
assert IntPoint(8, 2) == destroy_asteroids(calculate_angles(points, middle), middle, 9)
assert IntPoint(8, 2) == get_removed_element_at_pos(test_case_3.split('\n'), 200)
assert 815 == solve_2()


import pytest
import xarray
import dask.array as da
from itertools import product
from collections import namedtuple
from daskms.experimental.zarr import xds_to_zarr
from quartical.config.internal import gains_to_chain
from quartical.gains.gain import gain_spec_tup
from quartical.interpolation.interpolate import (
    load_and_interpolate_gains
)
import numpy as np
from copy import deepcopy

# TODO: These deliberately have enough points to work with all interpolation
# methods. Add tests for the case when we don't.

GAIN_PROPERTIES = {
    "between": ((0, 2, 2, 2, 0, 2, 2, 2), (2, 2, 2, 1, 2, 2, 2, 1)),
    "aligned": ((0, 4, 4, 1, 0, 4, 4, 1), (0, 4, 4, 1, 0, 4, 4, 1)),
    "overlap": ((0, 2, 2, 2, 0, 2, 2, 2), (1, 4, 2, 1, 1, 4, 2, 1)),
    "contain": ((0, 4, 4, 1, 0, 4, 4, 1), (1, 2, 6, 1, 1, 2, 6, 1)),
    "outside": ((2, 4, 4, 1, 2, 4, 4, 1), (0, 2, 4, 2, 0, 2, 4, 2)),
}

BOUNDS = namedtuple("BOUNDS", "min_t max_t min_f max_f")

def mock_gain_xds_list(start_time,
                       n_time,
                       gap_time,
                       n_xds_time,
                       start_freq,
                       n_freq,
                       gap_freq,
                       n_xds_freq):

    n_ant = 3
    n_dir = 1
    n_corr = 4

    _gain_xds_list = []

    for t_ind, f_ind in product(range(n_xds_time), range(n_xds_freq)):

        time_lb = start_time + t_ind*(n_time + gap_time)
        time_range = np.arange(time_lb, time_lb + n_time)

        freq_lb = start_freq + f_ind*(n_freq + gap_freq)
        freq_range = np.arange(freq_lb, freq_lb + n_freq)

        coords = {
            "gain_time": time_range,
            "gain_freq": freq_range,
            "antenna": np.arange(n_ant),
            "direction": np.arange(n_dir),
            "correlation": np.arange(n_corr)
        }

        gains = da.zeros((n_time, n_freq, n_ant, n_dir, n_corr),
                         dtype=np.complex128)
        gains += da.array([1, 0, 0, 1])

        flags = da.zeros((n_time, n_freq, n_ant, n_dir),
                         dtype=np.int8)

        gain_axes = (
            "gain_time",
            "gain_freq",
            "antenna",
            "direction",
            "correlation"
        )

        # Include a dummy data_var to check that it doesn't break anything.
        data_vars = {
            "gains": (gain_axes, gains),
            "gain_flags": (gain_axes[:-1], flags),
            "dummy": (("antenna",), np.arange(n_ant))
        }

        attrs = {
            "NAME": "G",
            "TYPE": 'complex',
            "GAIN_AXES": gain_axes,
            "GAIN_SPEC": gain_spec_tup((n_time,), (n_freq,), (n_ant,),
                                       (n_dir,), (n_corr,))
        }

        xds = xarray.Dataset(
            data_vars=data_vars,
            coords=coords,
            attrs=attrs
        )

        _gain_xds_list.append(xds)

    return _gain_xds_list

@pytest.fixture(scope="function")
def opts(base_opts, interp_mode, interp_method, tmp_path_factory):

    # Don't overwrite base config - instead duplicate and update.

    _opts = deepcopy(base_opts)

    _opts.solver.terms = ["G", "B"]
    _opts.output.gain_directory = str(tmp_path_factory.mktemp("writes.qc"))
    _opts.G.load_from = str(tmp_path_factory.mktemp("loads.qc")) + "/G"
    _opts.G.interp_method = interp_method
    _opts.G.interp_mode = interp_mode

    return _opts

@pytest.fixture(scope="function")
def chain(opts):
    return gains_to_chain(opts)

@pytest.fixture(scope="function", params=GAIN_PROPERTIES.values())
def _params(request):
    return request.param[0], request.param[1]

@pytest.fixture(scope="function")
def load_params(_params):
    return _params[0]

@pytest.fixture(scope="function")
def gain_params(_params):
    return _params[1]

@pytest.fixture(scope="function")
def gain_xds_lod(gain_params):
    return [{"G": xds, "B": xds} for xds in mock_gain_xds_list(*gain_params)]

@pytest.fixture(scope="function")
def term_xds_list(gain_xds_lod):
    return [xds_list["G"] for xds_list in gain_xds_lod]

@pytest.fixture(scope="function")
def load_xds_list(load_params, opts):

    mock_loads = mock_gain_xds_list(*load_params)

    path = '::'.join(opts.G.load_from.rsplit('/', maxsplit=1))

    da.compute(xds_to_zarr(mock_loads, path))

    return mock_loads

@pytest.fixture(scope="function", params=["reim", "ampphase"])
def interp_mode(request):
    return request.param

@pytest.fixture(scope="function", params=["2dlinear", "2dspline"])
def interp_method(request):
    return request.param

# -----------------------------make_interp_xds_list----------------------------

@pytest.fixture(scope="function")
def interpolated_xds_lod(
    gain_xds_lod,
    chain,
    opts,
    load_xds_list
):

    return load_and_interpolate_gains(
        gain_xds_lod,
        chain,
        opts.output.gain_directory
    )

def test_has_gains(interpolated_xds_lod):
    assert all(
        all(hasattr(xds, "gains") for xds in d.values())
        for d in interpolated_xds_lod
    )

def test_chunking(interpolated_xds_lod, term_xds_list):
    # TODO: Chunking behaviour is tested but not adequately probed yet.
    interpolated_xds_list = [ixds['G'] for ixds in interpolated_xds_lod]

    assert all(ixds.chunks == txds.chunks
               for ixds, txds in zip(interpolated_xds_list, term_xds_list))

# -----------------------------load_and_interpolate----------------------------

@pytest.fixture(scope="function")
def compute_interpolated_xds_lod(interpolated_xds_lod):
    return da.compute(interpolated_xds_lod)[0]

def test_cixl_has_gains(compute_interpolated_xds_lod):
    assert all([hasattr(xds, "gains")
               for xds_dict in compute_interpolated_xds_lod
               for xds in xds_dict.values()])

def test_cixl_gains_ident(compute_interpolated_xds_lod):
    # NOTE: Splines will not be exactly identity due to numerical precision.
    assert all(np.allclose(xds.gains.values, np.array([1, 0, 0, 1]))
               for xds_dict in compute_interpolated_xds_lod
               for xds in xds_dict.values())

# -----------------------------------------------------------------------------


import logging
import torch

from .autoencoder import TCNAutoencoder
from .supervised import SupervisedTCN

import json
from datetime import datetime, date

class ModelManager:
    def __init__(self, args, training=True, **kwargs):
        self.is_training = training
        self.load_network(args, **kwargs)

    def load_network(self, args, **kwargs):
        if args.model_pos or args.model_vel:
            print('Load network')
            if self.is_training:
                self.network, self.optim = self._load_pretrained_model(args.model_pos or args.model_vel)
            else:
                self.network = {}
                vel_net = self._load_pretrained_model(args.model_vel)
                if vel_net:
                    self.network['vel'] = vel_net.eval()
                pos_net = self._load_pretrained_model(args.model_pos)
                if pos_net:
                    self.network['pos'] = pos_net.eval()
        #erstelle Netzwerk
        else:
            self.network = TCNAutoencoder(args)

            params_basis = [*self.network.encoder.parameters(), *self.network.decoder.parameters()]
            params_adversarial = [*self.network.adversary_decoder.parameters()]

            self.optim_basis = torch.optim.Adam(params_basis, lr=args.learning_rate)
            self.optim_adversarial = torch.optim.Adam(params_adversarial, lr=args.learning_rate)

            self._log(self.network)

    def _load_pretrained_model(self, model_name):
        if not model_name:
            return None

        logging.info('Loading saved model {}...'.format(model_name))
        model = torch.load('../models/' + model_name)
        try:
            network = model['network']
        except KeyError:
            network = model['model']
        self._log(network)

        network.load_state_dict(model['model_state_dict'])

        if self.is_training:
            optim = torch.optim.Adam(network.parameters())
            optim.load_state_dict(model['optimizer_state_dict'])
            return network, optim

        return network

    def _log(self, network):
        #TODO prints network out: logging.info('\n ' + str(network))
        logging.info('# of Parameters: ' +
                     str(sum(p.numel() for p in network.parameters()
                             if p.requires_grad)))

    def save(self, e, run_identifier, losses, losses_100, name_run, args, is_adv=False):
        model_filename = '../models/' + args.signal_type + '-e' + str(e) + '-hz' + str(args.hz)
        torch.save(
            {
                'epoch': e,
                'network': self.network,
                'model_state_dict': self.network.state_dict(),
                'optimizer_state_dict': self.optim_basis.state_dict(),
                'optimizer_state_dict_adv': self.optim_adversarial.state_dict() if is_adv else [],
                'losses': losses
            }, model_filename)
        logging.info('Model saved to {}'.format(model_filename))

        #save params in extra file
        params = {
            "run_identifier": run_identifier,
            "signal_type"   : args.signal_type,
            "lr"            : args.learning_rate,
            "hz"            : args.hz,
            "viewing time"  : args.viewing_time,
            "bs"            : args.batch_size,
            "slice-time-windows": args.slice_time_windows,
            "last loss"     : list(losses)[-1],
            "current day"   : date.today().strftime("%d.%m.%Y"),
            "current time"  : datetime.now().strftime("%H:%M:%S"),
            "epoch losses train"  : list(losses['train']),
            "epoch losses val"  : list(losses['val']),
            "global losses 100 train" : list(losses_100['train']),
            "global losses 100 val": list(losses_100['val']),
            "CEL epoch losses train"  : list(losses['CELtrain']) if is_adv else [],
            "CEL epoch losses val"  : list(losses['CELval']) if is_adv else [],
            "CEL global losses 100 train" : list(losses_100['CELtrain']) if is_adv else [],
            "CEL global losses 100 val": list(losses_100['CELval'] if is_adv else [])
        }

        jsonFile = json.dumps(params)

        with open(model_filename + '.json', 'w') as jasonfile: # function opens a file, and returns it as a file object., Write - Opens a file for writing, creates the file if it does not exist
            jasonfile.write(jsonFile)


import requests, re
from urlextract import URLExtract
#from lipoja https://github.com/lipoja/URLExtract
from bs4 import BeautifulSoup
import csv,time
from datetime import datetime

artes = 0
written = False
printables=""
timetaken = datetime.now()
printables += f"\nData Taken at {timetaken}\n"
yearz = str(datetime.now().year)
montz = str(datetime.now().month)
dias = str(datetime.now().day)
hrs = str(datetime.now().hour)
minz = str(datetime.now().minute)
timetaken2 = f'{yearz}{montz}{dias}{hrs}{minz}'
pagenums=1

questionable_a = 0
straight_nsfw_a = 0

keyword = ''
keyword = input('Type a deviantart user to see their works!>')
keyword = keyword.lower()

url = 'https://www.deviantart.com/{}/gallery/all?page={}'.format(keyword,pagenums)
headers = {
    'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Mobile Safari/537.36'
}
req = requests.get(url,headers=headers)

nd = str(req)
soup = BeautifulSoup(req.content,'html.parser')

#To print entire HTML Page. Used for maintenance.
def print_txt(keyword):
    with open(f'{keyword}.txt','w',encoding='utf-8',newline='') as fyle:
        fyle.write(soup.prettify())
        fyle.close()

#To determine an active account. An active account should return Response 200 at all times.
if nd == '<Response [200]>':
    print(' ')
    print('This user is available!\n')

else:
    print(' ')
    deact = str(soup.prettify())
    if "DeviantArt: Deactivated Account" in deact:
        print('Deactivated Account!\n')
    elif "DeviantArt: 404" in deact:
        print('Account has never existed!\n')
    print('This user is not available! Please run this program again.\n')
    exit()

#Saving in .csv
def save_csv(keyword, lists,favelists,arttype,maturelevel,linklists):
    global written,timetaken,timetaken2
    csv_header = ['Title', 'Faves','Type','Artist','Mature','Link']
    keyword=keyword.lower()
    with open(f'{keyword}_{timetaken2}.csv','a+',encoding='utf-8',newline='') as f:
        arts = csv.writer(f)
        if written == False:
            arts.writerow(csv_header)
            written = True
        for i in range(len(lists)):
            arts.writerow([lists[i],favelists[i],arttype[i],keyword,maturelevel[i],linklists[i]])
    f.close()

def save_csv2(keyword):
    global timetaken,timetaken2
    keyword=keyword.lower()
    csv_header = [f'Timestamp taken at {timetaken}']
    with open(f'{keyword}_{timetaken2}.csv','a+',encoding='utf-8',newline='') as f:
        arts = csv.writer(f)
        arts.writerow(csv_header)
    f.close()

#Saving in .txt
def printprintables(printables_a):
    global timetaken2
    with open(f'{keyword.lower()}_written_data_{timetaken2}.txt','w',encoding='utf-8',newline='') as dataz:
        dataz.write(printables_a)
        dataz.close()

#Differ Visual and Literature arts and then
#Determining the contents
def addappend(lists,sensiti,maturelebels):
    for i in range (len(sensiti)):
        x = str(sensiti[i])
        if 'visual art' in x:
            for q in range (10):
                if lists[q] in x:
                    arttype[q] = 'Visual Art'
                    break
        elif 'literature' in x:
            for q in range (10):
                if lists[q] in x:
                    arttype[q] = 'Literature'
                    break
        if '
May contain sensitive content
' in x:
            for p in range (10):
                if lists[p] in x:
                    maturelebels[p] = 'Suggestive'
                    break

        elif '
Sensitive content
' in x:
            for p in range (10):
                if lists[p] in x:
                    maturelebels[p] = 'Mature'
                    break

        else:
            for p in range (10):
                if lists[p] in x:
                    maturelebels[p] = 'Safe'
                    break

#To determine name change of an user
name = soup.find_all('span',{'class':'_2UI2c'})
username = str(name[0])
del name
username = username[21:]
username = username[:-7]

if keyword.lower() != username.lower():
    printables+=f'\nThis user used to be around Deviantart with the name of {keyword.lower()},\nand now their name is {username}\n'
    print(printables)
    nowname = input("Would you like to use their current name (Y/N)?>")
    if nowname == 'Y' or nowname == 'y':
        keyword = username
    else:
        keyword = keyword

keyword = keyword.upper()
#print_txt(keyword)
#exit()
printme = input("\nPrint .csv proof (Y/N)?>")

printme2 = input("\nPrint .txt proof (Y/N)?>")

printables += '\nLIST OF ALL '+ keyword +' WORKS'
printables += '\n\n'

totaldevs = soup.find_all('span',{'class':'_2USdI'})
if len(totaldevs) > 0:
    devsnumber = str(totaldevs[0])
    devsnumber = devsnumber[21:]
    devsnumber = devsnumber[:-7]
    devsnumber = int(devsnumber)
    numofdeviation = devsnumber
else:
    devsnumber = 0

if devsnumber == 0:
    print("They got no posts for now.\n")
    printables += '\nThey got no posts for now.\n'
    printprintables(printables)
    exit()

#Printing their recent art titles
while devsnumber > 0:
    url = 'https://www.deviantart.com/{}/gallery/all?page={}'.format(keyword,pagenums)
    req = requests.get(url,headers=headers)
    nd = str(req)
    soup = BeautifulSoup(req.content,'html.parser')
    questionable = 0
    straight_nsfw = 0
    maturelebels = ['']*10
    arttype = ['']*10
    #saving number of faves
    dicts = {}

    items = soup.find_all('div',{'class':'_22J_R'})
    lists = []
    for i in items:
        i=str(i)
        i = i[20:]
        i = i[:-6]
        lists.append(i)

    #For visual arts
    sensiti1 = soup.find_all('a',{'class':'_1mCeE'})
    for i in range (len(sensiti1)):
        addappend(lists,sensiti1,maturelebels)

    #For literature arts
    sensiti2 = soup.find_all('div',{'class':'_1mCeE'})
    for i in range (len(sensiti2)):
        addappend(lists,sensiti2,maturelebels)

    #Extracting URL
    linkss= soup.find_all('a',{'data-hook':'deviation_link'})
    linklists = []
    a = 0
    for i in range (len(linkss)):
        if i % 2 == 0:
            linksone = str(linkss[i])
            extractor = URLExtract()
            linksone = extractor.find_urls(linksone)
            linklists.append(linksone[0])

    #Counting Mature and Suggestive Arts
    for i in range (len(maturelebels)):
        if maturelebels[i] == 'Suggestive':
            questionable += 1
        elif maturelebels[i] == 'Mature':
            straight_nsfw += 1
        else:
            continue

    #Counting faves and storing it on a dictionary
    faves = soup.find_all('button',{'class':'_3Vvhk x48yz'})
    favelists = []

    a = 0

    for i in faves:
        try:
            faveone = faves[a].find_all('span')
        except IndexError as e:
            print("This user needs to draw more art!")
        favecount = str(faveone[1])
        favecount = favecount.strip('')
        favecount = favecount.strip('')

        if "." in favecount:
            favecount=favecount.replace(".","")
            if "K" in favecount:
                favecount=favecount.replace("K","00")
        else:
            if "K" in favecount:
                favecount=favecount.replace("K","000")

        favecountint = int(favecount)
        favelists.append(favecount)
        dicts.update({lists[a]:favecountint})
        a = a+1

    #Collecting results
    for i in range(len(favelists)):
        artes += 1
        printables+=f'\n{artes}. '+lists[i]+" - "+favelists[i]+" favorites,\n"+"Art Type: "+arttype[i]+",\nMature Level: "+maturelebels[i]+",\nLink: "+linklists[i]+"\n"
    if printme == 'Y' or printme == 'y':
        save_csv(keyword, lists,favelists,arttype,maturelebels,linklists)

    devsnumber -= 10
    pagenums += 1
    questionable_a += questionable
    straight_nsfw_a += straight_nsfw
    totaldone=numofdeviation-devsnumber
    totaldone = (totaldone*100)/numofdeviation
    totaldonewritten = "%.2f" % totaldone
    if totaldone < 100:
        print (f'\nCalculation {totaldonewritten}% done!')
    else:
        print('\nCalculation done!')
    #To give requests some delay
    time.sleep(1)

questionable_arts = questionable_a/numofdeviation
questionable_arts = questionable_arts*100
questionable_arts = "%.2f" % questionable_arts
nsfw_arts = straight_nsfw_a/numofdeviation
nsfw_arts=nsfw_arts*100
nsfw_arts = "%.2f" % nsfw_arts
printables_a = ''
printables_a += f'\nFrom {numofdeviation} total arts, \n{questionable_a} ({questionable_arts}%) are questionable, \nand {straight_nsfw_a} ({nsfw_arts}%) are not safe for work.\n'
printables_a += printables

del printables

keyword = keyword.lower()
#Displaying results
printmea = input("\nDisplay results in terminal (Y/N)?>")
if printmea == 'Y' or printmea == 'y':
    print(printables_a)

if printme == 'Y' or printme == 'y':
    save_csv2(keyword)

if printme2 == 'Y' or printme2 == 'y':
    #print_txt(keyword) #--> Only uncomment in maintenances or mods.
    printprintables(printables_a)

print("\nThank you for using this checker!\n")
exit()


from glob import glob
import os
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from dateutil.relativedelta import relativedelta
import numpy as np
import pandas as pd
from datetime import datetime
import metpy.calc as mpcalc
from metpy.plots import Hodograph, SkewT
from metpy.units import units

#%%
add_log = True
if add_log == True :
    log_file = 'metpy_python.log'
    err_log_file = 'metpy_python_err.log'

def write_log(log_file, log_str):
    with open(log_file, 'a') as log_f:
        log_f.write(log_str+'\n')
    return print (log_str)
#for checking time
cht_start_time = datetime.now()
def print_working_time():
    working_time = (datetime.now() - cht_start_time) #total days for downloading
    return print('working time ::: %s' % (working_time))

#%%
###########################################
# Upper air data can be obtained using the siphon package, but for this example we will use
# some of MetPy's sample data.
dir_name = '../rawin_data/47138/'
save_dir_name = 'skew_T-log_P-diagram/'

if not os.path.exists(dir_name+save_dir_name):
    os.makedirs(dir_name+save_dir_name)
    print ('*'*80)
    print (dir_name+save_dir_name, 'is created')
else :
    print ('*'*80)
    print (dir_name+save_dir_name, 'is exist')

filename = 'UPPER_SONDE_47138_ALL_2018_2018_2019.csv'

fullname = dir_name + filename
fullname =sorted(glob(os.path.join(dir_name, '*.csv')))[-1]
#%%
fullname_el = fullname.split('\\')
filename = fullname_el[-1]
filename_el = filename.split('_')
obs_year = int(filename_el[-3])

#지점,일시(UTC),기압(hPa),고도(gpm),기온(°C),이슬점온도(°C),풍향(deg),풍속(knot),지상 FLAG(null),권계면 FLAG(null),최대풍 FLAG(null)
df = pd.read_csv(dir_name+filename, skiprows=1, sep=',', header=None, index_col=0,
                   names = ['site', 'time', 'pressure', 'height', 'temperature', 'dewpoint', 'direction', 'speed', 'FLAG1', 'FLAG2', 'FLAG3'],
                   skipfooter=1, engine='python')

df['u_wind'], df['v_wind'] = mpcalc.wind_components(df['speed'],
                                                    np.deg2rad(df['direction']))
#%%
# Drop any rows with all NaN values for T, Td, winds
#df = df.dropna(subset=('temperature', 'dewpoint', 'direction', 'speed', 'u_wind', 'v_wind'), how='all').reset_index(drop=True)
df = df.dropna(subset=('temperature', 'dewpoint'), how='all').reset_index(drop=True)

#%%
# Rows that do not meet the condition alpha + num are eliminated
s_start_date = datetime(obs_year, 1, 1) #convert startdate to date type
s_end_date = datetime(obs_year, 12, 31)

date1 = s_start_date
selected_times = []
while date1 < s_end_date :
    date1_strf = date1.strftime('%Y-%m-%d %H:%M')
    selected_times.append(date1_strf)
    date1 = date1 + relativedelta(hours=12)

#%%
#for selected_time in selected_times[0:10] :
selected_time  = selected_times[0]
print(selected_time)

f = lambda s: selected_time in s
ids  = df['time'].apply(f)
df_selected_time = df[ids]

#df_selected_time = df_selected_time.dropna()
df_selected_time = df_selected_time.sort_values('pressure', ascending=False)
print(df_selected_time)

df_selected_time.to_csv(r'{0}{1}{2}_{3}.csv'.format(dir_name, save_dir_name, filename_el[-5], selected_time[:13]))
###########################################
# We will pull the data out of the example dataset into individual variables and
# assign units.

##########################################################################
# We will pull the data out of the example dataset into individual variables and
# assign units.

p = df_selected_time['pressure'].values * units.hPa
T = df_selected_time['temperature'].values * units.degC
Td = df_selected_time['dewpoint'].values * units.degC
wind_speed = df_selected_time['speed'].values * units.knots
wind_dir = df_selected_time['direction'].values * units.degrees
u, v = mpcalc.wind_components(wind_speed, wind_dir)

##########################################################################
# Thermodynamic Calculations
# --------------------------
#
# Often times we will want to calculate some thermodynamic parameters of a
# sounding. The MetPy calc module has many such calculations already implemented!
#
# * **Lifting Condensation Level (LCL)** - The level at which an air parcel's
#   relative humidity becomes 100% when lifted along a dry adiabatic path.
# * **Parcel Path** - Path followed by a hypothetical parcel of air, beginning
#   at the surface temperature/pressure and rising dry adiabatically until
#   reaching the LCL, then rising moist adiabatially.

# Calculate the LCL
lcl_pressure, lcl_temperature = mpcalc.lcl(p[0], T[0], Td[0])

print(lcl_pressure, lcl_temperature)

# Calculate the parcel profile.
parcel_prof = mpcalc.parcel_profile(p, T[0], Td[0]).to('degC')

##########################################################################
# Basic Skew-T Plotting
# ---------------------
#
# The Skew-T (log-P) diagram is the standard way to view rawinsonde data. The
# y-axis is height in pressure coordinates and the x-axis is temperature. The
# y coordinates are plotted on a logarithmic scale and the x coordinate system
# is skewed. An explanation of skew-T interpretation is beyond the scope of this
# tutorial, but here we will plot one that can be used for analysis or
# publication.
#
# The most basic skew-T can be plotted with only five lines of Python.
# These lines perform the following tasks:
#
# 1. Create a ``Figure`` object and set the size of the figure.
#
# 2. Create a ``SkewT`` object
#
# 3. Plot the pressure and temperature (note that the pressure,
#    the independent variable, is first even though it is plotted on the y-axis).
#
# 4. Plot the pressure and dewpoint temperature.
#
# 5. Plot the wind barbs at the appropriate pressure using the u and v wind
#    components.

# Create a new figure. The dimensions here give a good aspect ratio
fig = plt.figure(figsize=(9, 9))
skew = SkewT(fig)

# Plot the data using normal plotting functions, in this case using
# log scaling in Y, as dictated by the typical meteorological plot
skew.plot(p, T, 'r', linewidth=2)
skew.plot(p, Td, 'g', linewidth=2)
skew.plot_barbs(p, u, v)

# Show the plot
plt.show()

##########################################################################
# Advanced Skew-T Plotting
# ------------------------
#
# Fiducial lines indicating dry adiabats, moist adiabats, and mixing ratio are
# useful when performing further analysis on the Skew-T diagram. Often the
# 0C isotherm is emphasized and areas of CAPE and CIN are shaded.

# Create a new figure. The dimensions here give a good aspect ratio
fig = plt.figure(figsize=(9, 9))
skew = SkewT(fig, rotation=30)

# Plot the data using normal plotting functions, in this case using
# log scaling in Y, as dictated by the typical meteorological plot
skew.plot(p, T, 'r')
skew.plot(p, Td, 'g')
skew.plot_barbs(p, u, v)
skew.ax.set_ylim(1050, 100)
skew.ax.set_xlim(-40, 60)

# Plot LCL temperature as black dot
skew.plot(lcl_pressure, lcl_temperature, 'ko', markerfacecolor='black')

# Plot the parcel profile as a black line
skew.plot(p, parcel_prof, 'k', linewidth=2)

# Shade areas of CAPE and CIN
skew.shade_cin(p, T, parcel_prof)
skew.shade_cape(p, T, parcel_prof)

# Plot a zero degree isotherm
skew.ax.axvline(0, color='c', linestyle='--', linewidth=2)

# Add the relevant special lines
skew.plot_dry_adiabats()
skew.plot_moist_adiabats()
skew.plot_mixing_lines()

# Show the plot
plt.show()

##########################################################################
# Adding a Hodograph
# ------------------
#
# A hodograph is a polar representation of the wind profile measured by the rawinsonde.
# Winds at different levels are plotted as vectors with their tails at the origin, the angle
# from the vertical axes representing the direction, and the length representing the speed.
# The line plotted on the hodograph is a line connecting the tips of these vectors,
# which are not drawn.

# Create a new figure. The dimensions here give a good aspect ratio
fig = plt.figure(figsize=(9, 9))
skew = SkewT(fig, rotation=30)

# Plot the data using normal plotting functions, in this case using
# log scaling in Y, as dictated by the typical meteorological plot
skew.plot(p, T, 'r')
skew.plot(p, Td, 'g')
skew.plot_barbs(p, u, v)
skew.ax.set_ylim(1050, 100)
skew.ax.set_xlim(-40, 60)

# Plot LCL as black dot
skew.plot(lcl_pressure, lcl_temperature, 'ko', markerfacecolor='black')

# Plot the parcel profile as a black line
skew.plot(p, parcel_prof, 'k', linewidth=2)

# Shade areas of CAPE and CIN
skew.shade_cin(p, T, parcel_prof)
skew.shade_cape(p, T, parcel_prof)

# Plot a zero degree isotherm
skew.ax.axvline(0, color='c', linestyle='--', linewidth=2)

# Add the relevant special lines
skew.plot_dry_adiabats()
skew.plot_moist_adiabats()
skew.plot_mixing_lines()

# Create a hodograph
# Create an inset axes object that is 40% width and height of the
# figure and put it in the upper right hand corner.
ax_hod = inset_axes(skew.ax, '40%', '40%', loc=1)
h = Hodograph(ax_hod, component_range=80.)
h.add_grid(increment=20)
h.plot_colormapped(u, v, wind_speed)  # Plot a line colored by wind speed

# Show the plot
plt.show()


import cv2
import numpy as np

# Read the video from file
cap = cv2.VideoCapture("video.mp4")

while True:
    ret, frame = cap.read()
    if ret == False:
        break

    cv2.imshow("Frame", frame)
    if cv2.waitKey(1) == 27:
        break

cap.release()
cv2.destroyAllWindows()


import csv
import logging
from collections import OrderedDict
from itertools import islice

from seleniumrequests import Chrome

from vgrabber.base.importaction import ImportAction
from vgrabber.model import Subject, Test, HomeWork, FinalExam, Student
from .actionexecutor import ActionExecutor

class MoodleGradesActionExecutor(ActionExecutor):
    __force_data = {
        'export_onlyactive': '1',
        'display[real]': '1',
        'decimals': '2',
        'separator': 'comma',
        'id_checkbox_controller1': '1'
    }

    def __init__(self, state):
        super().__init__(state)
        self.__state = state

        self.__import_final_exams = bool(self.__state.requested_actions & {ImportAction.moodle_final_exam_grades})
        self.__import_home_works = bool(self.__state.requested_actions & {ImportAction.moodle_home_work_grades})
        self.__import_tests = bool(self.__state.requested_actions & {ImportAction.moodle_test_grades})

    def exec(self):
        browser: Chrome = self.__state.browser
        model: Subject = self.__state.model

        if self.__import_final_exams:
            model.clear_final_exam_points()

        if self.__import_home_works:
            model.clear_home_work_points()

        if self.__import_tests:
            model.clear_test_points()

        browser.find_element_by_xpath("//span[text()='Export']").click()
        browser.find_element_by_link_text('Čistý textový súbor (TXT)').click()

        to_send = OrderedDict()

        for input_element in browser.find_elements_by_xpath("//form[@id='mform1']//input[@type='hidden']"):
            input_name = input_element.get_attribute('name')
            input_value = input_element.get_attribute('value')
            if input_name not in self.__force_data and not input_name.startswith('itemids['):
                to_send[input_name] = input_value

        to_send.update(self.__force_data)

        form_url = browser.find_element_by_xpath("//form[@id='mform1']").get_attribute('action')

        exported_csv = browser.request(
            'POST',
            form_url,
            data = to_send
        ).text

        for row in islice(csv.reader(exported_csv.splitlines()), 1, None):
            student_email = row[5]
            student = model.get_student_by_email(student_email)

            grades = []
            for grade in row[6:-1]:
                if grade == '-' or not grade:
                    grades.append(0)
                else:
                    try:
                        grades.append(float(grade))
                    except:
                        grades.append(0)

            if not any(grades):
                continue

            if student is None:
                logging.log(
                    logging.WARN,
                    "Student {0} {1} with email {2} not found".format(row[0], row[1], student_email)
                )
                student = Student(model, None, row[0], row[1], None)
                student.moodle_email = student_email
                model.add_student(student)

            for grade_item, grade in zip(self.__state.grade_items, grades):
                if grade_item is None or grade == 0:
                    continue
                if isinstance(grade_item, Test) and self.__import_tests:
                    student.add_test_points(grade_item, grade)
                if isinstance(grade_item, HomeWork) and self.__import_home_works:
                    student.add_home_work_points(grade_item, grade)
                if isinstance(grade_item, FinalExam) and self.__import_final_exams:
                    student.add_final_exam_points(grade_item, grade)


from scrapy.spiders import CrawlSpider
from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import Selector
from Crawler.items import NewsItem
from Crawler.utils import *
import re
import Crawler.settings
import scrapy

def get_xin_allow_url():
    """
    获得允许的url匹配,通过日期匹配
    :return:
    """
    start_time = NOW - datetime.timedelta(END_DAY)
    allow_url = list()
    if END_DAY < 21:
        return get_allow_date('%Y-%m/%d')
    if start_time.year == NOW.year:
        if start_time.month == NOW.month:
            for x in range(start_time.day, NOW.day + 1):
                string = str(start_time.strftime('%m')) + '/' + (str(x) if x >= 10 else '0' + str(x))
                allow_url.append('.*?/%d-%s/.*?' % (start_time.year, string))
        else:
            for x in range(start_time.month, NOW.month + 1):
                allow_url.append(
                    ".*?/%d-%s/\d+.*?" % (start_time.year, (str(x) if x >= 10 else '0' + str(x))))
    else:
        for x in range(start_time.year, NOW.year + 1):
            allow_url.append(".*?/%d-\d+/.*?" % x)
    return allow_url

class XinHuaNewsSpider(CrawlSpider):
    name = "xinhua_news"
    allowed_domains = [
        "news.xinhuanet.com", "www.xinhuanet.com",
        "education.news.cn", "sports.xinhuanet.com"
    ]

    start_urls = [
        'http://news.xinhuanet.com',
        'http://xinhuanet.com'
    ]
    deny_urls = [
        r'.*?xinhuanet.com.*?/\d{4}-\d{2}/\d{2}/.*?',
        r'.*?.photo.*?',
        r'.*?.video.*?',
        r'.*?.comments.*?',
        r'.*?.auto.*?',
        r'.*?.forum.*?',
        r'.*?.caipiao.*?',
        r'.*?fashion.*?'
    ]
    deny_urls_news = [
        r'.*?.photo.*?',
        r'.*?.video.*?',
        r'.*?.comments.*?',
        r'.*?.auto.*?',
        r'.*?.forum.*?',
        r'.*?.caipiao.*?'
    ]

    rules = (
        Rule(LinkExtractor(allow=".*?xinhuanet.com.*?", deny=deny_urls), follow=True),
        Rule(LinkExtractor(allow=get_xin_allow_url(), deny=deny_urls_news), callback="parse_item", follow=True)
    )

    @staticmethod
    def parse_item(response):
        sel = Selector(response)
        url = response.request.url
        if re.match(r'.*?xinhuanet.com.*?/\d{4}-\d{2}/\d{2}/.*?', url):
            content = response.xpath('//*[@id="p-detail"]//p//text()').extract()
            if content:
                item = NewsItem(
                    domainname='http://xinhuanet.com',
                    chinesename='新华网',
                    url=sel.root.base,
                    title=sel.css('div > div.h-title::text').extract_first(),
                    subtitle=sel.css('.sub::text').extract_first(),
                    language='中文',
                    encodingtype='utf-8',
                    corpustype='网络',
                    timeofpublish=sel.re(
                        r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')[0],
                    content=''.join(content),
                    source=sel.css('#source::text').extract_first(),
                    author=None
                )
                item = judge_time_news(item)
                if item:
                    yield item


#!/usr/bin/env python
import config as cfg
import json
import urllib.request
import datetime
import smtplib, ssl
from dateutil import relativedelta
from email.message import EmailMessage

today = datetime.datetime.today().strftime('%y%m%d')
plusOneMonth = datetime.date.today() + relativedelta.relativedelta(months=1)
nextMonth = plusOneMonth.strftime('%y%m%d')

# print(today)
# print(nextMonth)

# Create a secure SSL context
context = ssl.create_default_context()
emailMessage = EmailMessage()
emailMessage['To'] = cfg.to_email
emailMessage['From'] = cfg.from_email
emailMessage['Subject'] = "Vaccin"

with urllib.request.urlopen(f'https://booking-api.mittvaccin.se/clinique/{cfg.clinique_id}/appointments/{cfg.appointment_id}/slots/{today}-{nextMonth}') as url:
    jsonPlanning = json.loads(url.read().decode())
    for date in jsonPlanning:
        day = date['date']
        slots = date['slots']

        for slot in slots:
            time = slot['when']
            available = slot['available']
            if available:
                booking = f'Booking available on {day} at {time}: https://bokning.mittvaccin.se/klinik/{cfg.clinique_id}/bokning'
                # print(booking)
                with smtplib.SMTP_SSL("smtp.gmail.com", 465, context=context) as server:
                    server.login(cfg.from_email, cfg.smtp_token)
                    emailMessage.set_content(booking)
                    server.send_message(emailMessage)


#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author: KrianJ
# datetime:2022/5/30 22:12
# software: PyCharm-train

import os
import warnings
import torch
import torch.nn as nn
import numpy as np
from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score

warnings.filterwarnings("ignore")

def train(model, train_loader, dev_loader, output_dir, device='cpu'):
    """Preparation"""
    best_path = os.path.join(output_dir, 'best_model.pth')  # 最优模型
    checkpoint_dir = os.path.join(output_dir, 'checkpoint')  # 创建checkpoint目录
    os.makedirs(checkpoint_dir, exist_ok=True)

    device = torch.device(device)
    model.to(device)
    epoches = 2
    learning_rate = 1e-5
    weight_decay = 1e-4
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)

    """Starting Training"""
    model.train()
    true_labels = []
    pred_labels = []
    train_losses = []
    best_f1 = 0.  # 记录最好的F1值
    for epoch in range(epoches):
        for i, batch in enumerate(train_loader):
            batch.to(device)
            label = batch.label  # 真实标签
            logits = model(batch)  # 预测值
            preds = torch.argmax(logits, dim=-1)
            loss = criterion(logits, label)  # 交叉熵

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            label, preds = label.cpu().numpy(), preds.cpu().numpy()
            loss = float(loss.detach().cpu().numpy())
            true_labels.extend(label)
            pred_labels.extend(preds)
            train_losses.append(loss)
            # 计算得分
            acc = accuracy_score(label, preds)
            precision = precision_score(label, preds)
            recall = recall_score(label, preds)
            f1 = f1_score(label, preds)

            if i % 10 == 0:
                print("Epoch{}-{}, Loss: {}, Acc: {}, P: {}, R: {}, F1: {}".
                      format(epoch, i, loss, acc, precision, recall, f1))
            if i % 100 == 0 and i:  # 建立checkpoint
                dev_acc, dev_p, dev_r, dev_f1 = evaluate(model, criterion, dev_loader, device)
                if dev_f1 > best_f1:
                    best_f1 = dev_f1
                    checkpoint_path = os.path.join(checkpoint_dir, 'checkpoint_{}_{}.pth'.format(epoch, dev_f1))
                    torch.save(model, checkpoint_path)

        acc = accuracy_score(true_labels, pred_labels)
        precision = precision_score(true_labels, pred_labels)
        recall = recall_score(true_labels, pred_labels)
        f1 = f1_score(true_labels, pred_labels)
        print('Epoch {} Finished\nTraining: acc: {}, p: {}, r: {}, f1: {}'.format(epoch, acc, precision, recall, f1))

        # evaluate
        dev_acc, dev_p, dev_r, dev_f1 = evaluate(model, criterion, dev_loader, device)
        print("Evaluate: acc:{}, p: {}, r: {}, f1: {}".format(dev_acc, dev_p, dev_r, dev_f1))
        # 保存模型
        if dev_f1 > best_f1:
            best_f1 = dev_f1
            torch.save(model, best_path)

def evaluate(model, criterion, dev_loader, device):
    model.to(device)
    model.eval()
    true_labels = []
    pred_labels = []
    dev_loss = []
    with torch.no_grad():
        for batch in dev_loader:
            batch.to(device)
            labels = batch.label
            logits = model(batch)
            preds = torch.argmax(logits, dim=-1)
            loss = float(criterion(logits, labels).cpu().numpy())

            true_labels.extend(labels.cpu().numpy())
            pred_labels.extend(preds.cpu().numpy())
            dev_loss.append(loss)
    acc = accuracy_score(true_labels, pred_labels)
    precision = precision_score(true_labels, pred_labels)
    recall = recall_score(true_labels, pred_labels)
    f1 = f1_score(true_labels, pred_labels)
    return acc, precision, recall, f1


import logging
import csv
from flask import Flask, render_template, request
from flask_mobility import Mobility
from parsers import DriveThruCards

app = Flask(__name__)
Mobility(app)

if __name__ != '__main__':
    gunicorn_logger = logging.getLogger('gunicorn.error')
    app.logger.handlers = gunicorn_logger.handlers
    app.logger.setLevel(gunicorn_logger.level)

app.logger.info("Getting cards...")
card_source = DriveThruCards()
cards = card_source.get_cards()
app.logger.info("Ready!")

app.logger.info("Parsing CSV...")
with open('resources/twda_usage.csv') as csvfile:
    next(csvfile)
    reader = csv.reader(csvfile, delimiter=',')
    card_usage = {row[0]: int(row[1]) for row in reader}
app.logger.info("Ready!")

def add_missing_cards():
    global cards
    global card_usage
    for card in cards:
        if card['id'] not in card_usage:
            app.logger.info("Card missing:" + card['id'])
            card_usage[card['id']] = 0

add_missing_cards()

@app.route("/", methods=['GET'])
def cards_panel():
    sorting_method = request.args.get('sort', None)
    global cards
    global card_usage
    local_cards = cards[:]

    if sorting_method == 'twda':
        local_cards.sort(key=lambda c: card_usage[c['id']], reverse=True)
    elif sorting_method == 'name':
        local_cards.sort(key=lambda c: c['name'])
    elif sorting_method == 'type':
        local_cards.sort(key=lambda c: (c['type'], c['name']))
    elif sorting_method == 'artist':
        local_cards.sort(key=lambda c: (c['artist'], c['name']))

    return render_template('cards_panel.html', cards=local_cards, is_mobile=request.MOBILE)

if __name__ == '__main__':
    app.run(port=8080)


import discord
from sqlalchemy import and_, or_, exists

import db
import discord_bot
from logger import logger

def has_user_reached(level, user_id):
    if level.unlocks:
        return db.session.query(db.UserUnlock).where(
            and_(db.UserUnlock.level_id == level.id, db.UserUnlock.user_id == str(user_id))
        ).scalar() is not None
    for parent_level in level.parent_levels:
        if parent_level.solutions:
            has_solved = db.session.query(db.UserSolve).where(
                and_(db.UserSolve.level_id == parent_level.id, db.UserSolve.user_id == str(user_id))
            ).scalar() is not None
            if not has_solved:
                return False
        elif not has_user_reached(parent_level, user_id):
            return False
    return True

def has_user_solved(level, user_id):
    return db.session.query(db.UserSolve).where(
        and_(db.UserSolve.level_id == level.id, db.UserSolve.user_id == str(user_id))
    ).scalar() is not None

def get_solvable_levels(user_id):
    levels = []
    for level in db.session.query(db.Level).all():
        if can_user_solve(level, user_id):
            if level.solutions:
                levels.append(level)
            elif not level.child_levels:
                levels.append(level)
            elif not any(not l.unlocks and has_user_reached(l, user_id) for l in level.child_levels):
                levels.append(level)
    return levels

def get_solved_levels(user_id, name=None, start='', limit=None):
    query = db.session.query(db.Level).where(
        and_(
            db.Level.name == name if name else db.Level.name.startswith(start),
            exists().where(and_(db.UserSolve.user_id == str(user_id), db.Level.id == db.UserSolve.level_id))
        )
    )
    if limit is not None:
        query = query.limit(limit)
    return query.all()

def get_solved_or_unlocked_levels(user_id, name=None, start='', limit=None):
    query = db.session.query(db.Level).where(
        and_(
            db.Level.name == name if name else db.Level.name.startswith(start),
            or_(
                exists().where(and_(db.UserSolve.user_id == str(user_id), db.Level.id == db.UserSolve.level_id)),
                exists().where(and_(db.UserUnlock.user_id == str(user_id), db.Level.id == db.UserUnlock.level_id))
            )
        )
    )
    if limit is not None:
        query = query.limit(limit)
    return query.all()

def get_user_level_suffixes(user_id):
    levels = list(get_solvable_levels(user_id))
    levels.sort(key=lambda l: ((l.category.ordinal or 0) if l.category else -1, l.name))
    user_level_suffixes = []
    for level in levels:
        if level.nickname_suffix and (not level.nickname_merge or level.nickname_suffix not in user_level_suffixes):
            user_level_suffixes.append(level.nickname_suffix)
    return user_level_suffixes

def has_user_solved_everything(user_id):
    unsolved_level = db.session.query(db.Level).where(
        and_(
            exists().where(db.Level.id == db.Solution.level_id),
            ~exists().where(and_(db.UserSolve.user_id == str(user_id), db.Level.id == db.UserSolve.level_id)),
        )
    ).first()
    locked_level = db.session.query(db.Level).where(
        and_(
            exists().where(db.Level.id == db.Unlock.level_id),
            ~exists().where(and_(db.UserUnlock.user_id == str(user_id), db.Level.id == db.UserUnlock.level_id)),
        )
    ).first()
    return unsolved_level is None and locked_level is None

def get_used_role_ids():
    roles = set()
    for level in db.session.query(db.Level).all():
        if level.discord_role:
            roles.add(level.discord_role)
    completionist_role = db.get_setting('completionist_role')
    if completionist_role:
        roles.add(completionist_role)
    return roles

def get_starting_levels():
    return db.session.query(db.Level).where(~db.Level.parent_levels.any())

def is_member_admin(member):
    admin_role_id = db.get_setting('admin_role')
    if not admin_role_id:
        return False
    return member.get_role(int(admin_role_id)) is not None

async def update_user_nickname(user_id):
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    member = guild.get_member(int(user_id)) or await guild.fetch_member(int(user_id))
    if not member:
        logger.error('member %s not found in guild %s', user_id, guild.name)
        return
    if member.bot:
        return
    user = db.session.get(db.User, str(user_id))
    if user is None:
        user = db.User(id=str(user_id))
        db.session.add(user)
    if db.get_setting('admin_enable') == 'true' and is_member_admin(member):
        name_suffix = db.get_setting('admin_badge', '')
    elif db.get_setting('completionist_enable_nickname') == 'true' and has_user_solved_everything(user_id):
        name_suffix = db.get_setting('completionist_badge', '*')
    elif db.get_setting('nickname_enable') == 'true':
        level_suffixes = get_user_level_suffixes(user_id)
        prefix = db.get_setting('nickname_prefix', ' [')
        separator = db.get_setting('nickname_separator', ', ')
        suffix = db.get_setting('nickname_suffix', ']')
        name_suffix = f'{prefix}{separator.join(level_suffixes)}{suffix}' if level_suffixes else ''
    else:
        name_suffix = None
    user.nick = user.name or member.name
    if name_suffix:
        user.nick = user.nick[:32 - max(0, len(name_suffix))] + name_suffix[:32]
    db.session.commit()
    if member.nick == user.nick:
        return
    logger.info(f'updating nickname for %s (%s) to %s in %s', member.name, member.id, user.nick, guild.name)
    try:
        await member.edit(nick=user.nick)
    except discord.Forbidden:
        logger.error(f'missing permission to update nickname for %s (%s)', member.name, member.id)

async def update_all_user_nicknames():
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    for i, member in enumerate(guild.members):
        if not member.bot:
            await update_user_nickname(str(member.id))
        yield f'{i}/{guild.member_count}'

async def update_user_roles(user_id, used_role_ids=None):
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    if used_role_ids is None:
        used_role_ids = get_used_role_ids()

    member = guild.get_member(int(user_id)) or await guild.fetch_member(user_id)
    if member is None or member.bot:
        return
    roles_user_has = set(map(lambda r: str(r.id), member.roles)) & used_role_ids
    roles_user_should_have = set()

    if not (db.get_setting('admin_enable') == 'true' and is_member_admin(member)):
        for level in db.session.query(db.Level).all():
            if has_user_reached(level, user_id):
                if any(not l.unlocks and has_user_reached(l, user_id) for l in level.child_levels):
                    continue
                for parent_level in get_parent_levels_until_role_or_unlock(level):
                    roles_user_should_have.add(parent_level.discord_role)

        if db.get_setting('completionist_enable_role') == 'true' and has_user_solved_everything(user_id):
            completionist_role = db.get_setting('completionist_role')
            if completionist_role:
                roles_user_should_have.add(completionist_role)

    role_ids_to_add = roles_user_should_have - roles_user_has
    roles_to_add = list(filter(lambda r: r is not None, map(lambda r: guild.get_role(int(r)), role_ids_to_add)))
    role_ids_to_remove = roles_user_has - roles_user_should_have
    roles_to_remove = list(filter(lambda r: r is not None, map(lambda r: guild.get_role(int(r)), role_ids_to_remove)))
    await member.add_roles(*roles_to_add)
    await member.remove_roles(*roles_to_remove)

async def update_all_user_roles():
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    used_role_ids = get_used_role_ids()
    for i, member in enumerate(guild.members):
        if not member.bot:
            await update_user_roles(str(member.id), used_role_ids=used_role_ids)
        yield f'{i}/{guild.member_count}'

def can_user_solve(level, user_id):
    if db.session.query(db.UserSolve).where(
            and_(db.UserSolve.level_id == level.id, db.UserSolve.user_id == str(user_id))
    ).scalar():
        return False
    return has_user_reached(level, user_id)

def can_user_unlock(level, user_id):
    if db.session.query(db.UserUnlock) \
            .where(and_(db.UserUnlock.level_id == level.id, db.UserUnlock.user_id == str(user_id))).scalar():
        return False
    for parent_level in level.parent_levels:
        if not has_user_reached(parent_level, user_id):
            return False
    return True

async def add_role_to_user(user_id, role_id):
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id)
    if guild is None:
        raise Exception(f'guild not set or wrong: {guild_id}')
    member = guild.get_member(int(user_id)) or await guild.fetch_member(int(user_id))
    if member is None:
        raise Exception(f'failed to find member')
    role = guild.get_role(int(role_id))
    logger.info('assigning role %s to user %s (%s)', role.name, member.name, member.id)
    await member.add_roles(role)

def get_parent_levels_until_role_or_unlock(level):
    if level.discord_role:
        return {level}
    res = set()
    if not level.unlocks:
        for parent_level in level.parent_levels:
            res.update(get_parent_levels_until_role_or_unlock(parent_level))
    return res

def get_child_ids_recursively(level):
    child_ids = {level.id}
    for child_level in level.child_levels:
        if child_level.id not in child_ids:
            child_ids |= get_child_ids_recursively(child_level)
    return child_ids

async def move_level_to_category(level):
    if level.discord_channel and level.category and level.category.discord_category:
        discord_channel = discord_bot.client.get_channel(int(level.discord_channel)) \
                          or await discord_bot.client.fetch_channel(level.discord_channel)
        if discord_channel.category_id != int(level.category.discord_category):
            discord_category = discord_bot.client.get_channel(int(level.category.discord_category)) \
                               or await discord_bot.client.fetch_channel(level.category.discord_category)
            if discord_category and discord_category.type == discord.ChannelType.category:
                child_ids = get_child_ids_recursively(level)
                position = None
                found_child = False
                for other_channel in discord_category.channels:
                    for other_level in db.session.query(db.Level).where(
                            db.Level.discord_channel == str(other_channel.id)):
                        if other_level.id in child_ids:
                            position = other_channel.position
                            found_child = True
                    if found_child:
                        break
                    position = other_channel.position + 1
                if position is not None:
                    await discord_channel.edit(category=discord_category, position=position)
                else:
                    await discord_channel.edit(category=discord_category)

async def move_all_levels_to_categories():
    for level in db.session.query(db.Level).all():
        await move_level_to_category(level)

def get_parent_levels_recursively(level, initial_level=None):
    if initial_level is None:
        initial_level = level
    parent_levels = {level}
    for parent_level in level.parent_levels:
        if parent_level.id == initial_level.id:
            raise ValueError('loop in level dependencies')
        parent_levels |= get_parent_levels_recursively(parent_level, initial_level)
    return parent_levels

def check_loops():
    for level in db.session.query(db.Level).all():
        get_parent_levels_recursively(level)

async def update_role_permissions():
    levels = [level for level in db.session.query(db.Level).all()]
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    if guild is None:
        raise Exception(f'guild not set or wrong: {guild_id}')
    channel_permissions = {level.discord_channel: {
        guild.default_role: discord.PermissionOverwrite(read_messages=False)
    } for level in levels if level.discord_channel}
    for level in db.session.query(db.Level).all():
        role = guild.get_role(int(level.discord_role)) if level.discord_role else None
        if role is None:
            continue
        for parent_level in get_parent_levels_recursively(level):
            if parent_level.discord_channel and parent_level.discord_channel in channel_permissions.keys():
                parent_child = parent_level.discord_channel
                channel_permissions[parent_child][role] = discord.PermissionOverwrite(read_messages=True)
    progress = 0
    for channel_id, permissions in channel_permissions.items():
        channel = guild.get_channel(int(channel_id))
        if channel is not None:
            permissions_as_list = list(permissions.items())
            current_permissions = {chid: overwrite for chid, overwrite in channel.overwrites.items() if
                                   overwrite.read_messages}
            roles_to_add = set(permissions.keys()) - set(current_permissions.keys())
            roles_to_remove = set(current_permissions.keys()) - set(permissions.keys())
            if len(roles_to_add) == 0 and len(roles_to_remove) == 0:
                continue
            if len(roles_to_add) + len(roles_to_remove) < len(permissions_as_list) - 100:
                for role in roles_to_remove:
                    await channel.set_permissions(role, overwrite=None)
                for role in roles_to_add:
                    await channel.set_permissions(role, overwrite=permissions[role])
                continue
            first_batch = dict(permissions_as_list[:100])
            await channel.edit(overwrites=first_batch)
            for role, overwrite in permissions_as_list[100:]:
                await channel.set_permissions(role, overwrite=overwrite)
        yield f'{progress}/{len(channel_permissions)}'

async def skip_user_to_level(user_id, level, include_self=False):
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    member = guild.get_member(int(user_id)) or await guild.fetch_member(int(user_id))
    if not member:
        raise Exception('invalid author')
    solved_level_names = []
    unlocked_level_names = []
    for parent_level in get_parent_levels_recursively(level):
        if parent_level.unlocks and not db.session.query(db.UserUnlock).where(
                and_(db.UserUnlock.level_id == str(parent_level.id), db.UserUnlock.user_id == str(member.id))).scalar():
            unlocked_level_names.append(parent_level.name)
            db.session.add(db.UserUnlock(user_id=str(member.id), level=parent_level))
        if level.unlocks and level in parent_level.child_levels:
            continue
        if parent_level == level and not include_self:
            continue
        if parent_level.solutions and not db.session.query(db.UserSolve).where(
                and_(db.UserSolve.level_id == parent_level.id, db.UserSolve.user_id == str(member.id))).scalar():
            solved_level_names.append(parent_level.name)
            db.session.add(db.UserSolve(user_id=str(member.id), level=parent_level))
    db.session.commit()
    await update_user_roles(str(member.id))
    await update_user_nickname(str(member.id))
    message_parts = []
    if solved_level_names:
        message_parts.append(f'{len(solved_level_names)} solves ({", ".join(reversed(solved_level_names))})')
    if unlocked_level_names:
        message_parts.append(f'{len(unlocked_level_names)} unlocks ({", ".join(reversed(unlocked_level_names))})')
    return ('Added ' + ' and '.join(message_parts)) if message_parts else 'Nothing to do'

def get_invalid_user_solves():
    return db.session.query(db.UserSolve).where(
        ~exists(db.Solution).where(db.UserSolve.level_id == db.Solution.level_id)).all()

def get_leaderboard(categories=None):
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id)
    if categories is not None:
        levels = db.session.query(db.Level).where(db.Level.category_id.in_(categories))
    else:
        levels = db.session.query(db.Level).all()
    scores = {}
    for level in levels:
        if not level.solutions:
            continue
        for user_solve in level.user_solves:
            scores[user_solve.user_id] = scores.get(user_solve.user_id, 0) + 1
    groups = {}
    for uid, score in scores.items():
        user = db.session.get(db.User, uid)
        member = guild.get_member(int(user.id))
        if member and is_member_admin(member):
            continue
        if score not in groups:
            groups[score] = []
        groups[score].append(user)
    return sorted(groups.items(), reverse=True)

def get_users_dict():
    users = {}
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id)
    for user in db.session.query(db.User).all():
        member = guild.get_member(int(user.id))
        users[user.id] = {
            'nick': user.nick,
            'avatar': user.avatar,
            'admin': member and is_member_admin(member)
        }
    return users

def get_scores_dict():
    scores = {}
    for solve in db.session.query(db.UserSolve).all():
        if solve.user_id not in scores:
            scores[solve.user_id] = {}
        scores[solve.user_id][solve.level.category_id] = scores[solve.user_id].get(solve.level.category_id, 0) + 1
    return scores

def update_avatar(member):
    user = db.session.get(db.User, str(member.id)) or db.User(id=member.id)
    avatar_url = member.display_avatar.url if user is not None else None
    if avatar_url != user.avatar:
        user.avatar = avatar_url
        db.session.commit()

async def update_all_avatars():
    guild_id = int(db.get_setting('guild'))
    guild = discord_bot.client.get_guild(guild_id) or await discord_bot.client.fetch_guild(guild_id)
    for user in db.session.query(db.User).all():
        member = guild.get_member(int(user.id))
        if member is not None:
            update_avatar(member)
        else:
            user.avatar = None
            db.session.commit()

def get_used_categories():
    used_categories = []
    for category in db.session.query(db.Category).order_by('ordinal').all():
        for level in category.levels:
            if level.solutions:
                break
        else:
            continue
        used_categories.append(category)
    return used_categories


import openai
import pyttsx3 # Text to speech
import speech_recognition as sr # for speech recognition
import datetime # for date and time
import pyaudio
# import wikipedia # for wikipedia
# import webbrowser # for opening browser
# import os # for operating system
# import smtplib # for sending email
# import random # for random number
# import pywhatkit # for playing youtube
# import pyjokes # for jokes
# import requests # for getting weather

# API key
openai.api_key = " "

# Text to speech
engine = pyttsx3.init()

def transcribe_audio_to_text(filename):
    r = sr.Recognizer()
    with sr.Microphone() as source:
        print("Listening...")
        r.pause_threshold = 1
        audio = r.record(source)

    try:
        print("Recognizing...")
        query = r.recognize_google(audio, language='en-in')
        print(f"User said: {query}\n")
        return r.recognize_google(audio, language='en-in')

    except Exception as e:
        print(e)
        print("Say that again please...")
        return "None"
    return query

def generate_response(query):
    response = openai.Completion.create(
        engine="text-davinci-003",
        prompt=query,
        temperature=0.9,
        max_tokens=4000,
        n=1,
        frequency_penalty=0,
        presence_penalty=0.6,
        stop=None,

    )
    return response["choices"][0]["text"]

def speak_text(text):
    engine.say(text)
    engine.runAndWait()

def main():
    while True:
        print("Listening...")
        print(" say 'genius' to start recording your question...")
        with sr.Microphone() as source:
            r = sr.Recognizer()
            r.pause_threshold = 1
            audio = r.listen(source)
            try:
                transcription = r.recognize_google(audio, language='en-in')
                #record audio if transcription.lower() == "genius":
                if transcription.lower() == "genius":
                    recognizer = sr.Recognizer()
                    source.pause_threshold = 1
                    audio = recognizer.listen(source, phrase_time_limit=None, timeout=None)
                    with open(filename, "wb") as f:
                        f.write(audio.get_wav_data())
                #transcribe audio to text
                text = transcribe_audio_to_text(filename)
                if text:
                    print(f"you said: {text}")
                    #generate response
                    response = generate_response(text)
                    print(f"genius said: {response}")
                    #read response using text to speech
                    speak_text(response)
            except Exception as e:
                print(e)
                print("Say that again please...")
                print("an error occurred {}".format(e))
                # return "None"

if __name__ == '__main__':
    main()


from typing import Any

from tensorflow.compiler.mlir.quantization.tensorflow.python import py_function_lib
from tensorflow.compiler.mlir.quantization.tensorflow.python import representative_dataset as rd

# LINT.IfChange(static_range_ptq)
def static_range_ptq(
    src_saved_model_path: str,
    dst_saved_model_path: str,
    quantization_options_serialized: bytes,
    *,
    signature_keys: list[str],
    signature_def_map_serialized: dict[str, bytes],
    function_aliases: dict[str, str],
    py_function_library: py_function_lib.PyFunctionLibrary,
    representative_dataset: rd.RepresentativeDatasetOrMapping,
) -> Any: ...  # Status

# LINT.ThenChange()


import torch
from torch import nn

class BasicArbitraryCNNmodel(nn.Module):
    def __init__(self, in_channels=3, num_outputs=10):
        super(BasicArbitraryCNNmodel, self).__init__()
        hidden_channels = 64
        self.input_block = self.build_intermediate_block(in_channels, hidden_channels, 7, 2, 3)
        self.max_pool1 = nn.MaxPool2d(2, 2, 0)
        self.hidden_block1 = self.build_intermediate_block(hidden_channels, hidden_channels * 2, 3, 1, 0)
        self.max_pool2 = nn.MaxPool2d(2, 2, 0)
        self.hidden_block2 = self.build_intermediate_block(hidden_channels * 2, hidden_channels * 4, 3, 1, 0)
        self.adaptive_avg_pool = nn.AdaptiveAvgPool2d((2, 2))
        self.output_block = self.build_output_block(1024, num_outputs)

    def build_intermediate_block(self, in_channels, out_channels, kernel_size=3, stride=1, padding=0):
        return nn.Sequential(
            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride,
                      padding=padding),
            nn.BatchNorm2d(num_features=out_channels),
            nn.ReLU()
        )

    def build_output_block(self, in_features, num_outputs):
        return nn.Sequential(
            nn.Linear(in_features=in_features, out_features=in_features // 2),
            nn.ReLU(),
            nn.Linear(in_features=in_features // 2, out_features=num_outputs)
        )

    def forward(self, x):
        out = self.input_block(x)
        out = self.max_pool1(out)
        out = self.hidden_block1(out)
        out = self.max_pool2(out)
        out = self.hidden_block2(out)
        out = self.adaptive_avg_pool(out)
        out = torch.flatten(out, start_dim=1)
        return self.output_block(out)


'''
This program designed as daemon for OSX (tested with macOS Catalina 10.15.6) works on:
	- MacBook Air 13 mid 13

The main Idea is:
	IF camera OR microphone IS active THEN we have to turn led light "On AIR" ON
	ELSE turn led light "On AIR" OFF

This daemon paired with special ESP in this way:
	- microcontroller starts
	- mini-web-server started on microcontroller
	- THIS daemon sends on or off depends on state of mic and cam

PLEASE USE YOUR IP-ADDRESS in code instead of 192.168.0.42 or ESP module (or use "airscreen.local" instead)

This daemon paired with SLACK account, you should get own key for application, using
	this instruction:
		How to is here : https://github.com/witnessmenow/arduino-slack-api
		Codes are here : https://www.webfx.com/tools/emoji-cheat-sheet/
	Also you should put this key to: export SLKKEY="xoxp-............" and put it into ~/.bash_profile

PLEASE USE YOUR gags in updateSlackStatus call, mine are funny for me personally (and are in Russian)

The Architecture is easy: daemon uses IOREG keys to find states of devices like cam and mics
'''
import 	subprocess
import 	threading
import 	requests
import 	os
from 	time 		import sleep, strftime

def getMicState():
	ioreg = subprocess.check_output( ['ioreg -r -w0 -c AppleHDAEngineInput'], stderr=subprocess.STDOUT, shell=True )
	lines = ioreg.decode().split("\n")
	for line in lines:
		if "IOAudioEngineState" in line :
			try :
				i = int(line[-1:])
				if i < 1 :
					return False
				else :
					return True
			except ValueError :
				return True
			except :
				return True

def getCamState():
	ioreg = subprocess.check_output( ['ioreg -r -w0 -c AppleCamIn'], stderr=subprocess.STDOUT, shell=True )
	lines = ioreg.decode().split("\n")
	for line in lines:
		if "CameraActive" in line :
			try :
				i = line.index("= ")
				i = i + 2

				YesOrNo = line[i:]

				if YesOrNo.startswith("Yes") :
					return True
				else :
					return False

			except ValueError:
				return True
			except :
				return True

def updateSlackStatus ( status_text, status_emoji ):
	API_URL = "https://slack.com/api/users.profile.set"
	HEADERS = {
		"Content-type" : "application/json; charset=utf-8",
		"Authorization": "Bearer {}".format(os.environ['SLKKEY'])
		}
	NWSTTSM = {
		"profile": {
			"status_text"	: status_text,
			"status_emoji"	: status_emoji
			}
		}

	try :
		rq = requests.post(API_URL, json=NWSTTSM, headers=HEADERS, timeout=3)
	except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as err:
		print( strftime('%Y-%m-%d %H:%M:%S'), "Got an SLACK error", err )
		pass

def updateLocalStatusBis ( busy = False, reason = "air") :
	try :
		if busy:
			resp = requests.get("http://192.168.0.42/{}".format(reason), timeout=2)
		else :
			resp = requests.get("http://192.168.0.42/vacant", timeout=2)
	except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError) as err:
		print( strftime('%Y-%m-%d %H:%M:%S'), "Got an error on connection", err )
		pass

if __name__ == '__main__':

	print("Daemon started")
	prev_state = [False, False]

	while True :
		micbusy = getMicState()
		cambusy = getCamState()

		if True in [micbusy, cambusy] :
			if micbusy and cambusy :
				b_thread = threading.Thread( target=updateLocalStatusBis, args=(True, "air") )
				s_thread = threading.Thread( target=updateSlackStatus, args=("Говорит и показывает в Москву", ":phone:") )
			elif cambusy :
				b_thread = threading.Thread( target=updateLocalStatusBis, args=(True, "video") )
				s_thread = threading.Thread( target=updateSlackStatus, args=("Камера, мотор!", ":phone:") )
			elif micbusy :
				b_thread = threading.Thread( target=updateLocalStatusBis, args=(True, "voice") )
				s_thread = threading.Thread( target=updateSlackStatus, args=("Говорит ротом", ":phone:") )
		else :
			b_thread = threading.Thread( target=updateLocalStatusBis, args=(False, None) )
			s_thread = threading.Thread( target=updateSlackStatus, args=("", "") )

		if prev_state != [micbusy, cambusy] :
			prev_state = [micbusy, cambusy]
			b_thread.start()
			s_thread.start()
		elif True in [micbusy, cambusy]:
			b_thread.start()

		sleep(7)

#
#
# WHAT by katurov@gmail.com for katurov@gmail.com


from numpy import concatenate, nan, ndarray, append, array, std
from sklearn.feature_selection import f_regression
from multiprocessing import Pool
from pandas import DataFrame
import os

class InteractionChecker:
    def __init__(self, alpha=0.05):
        """
        Instantiates InteractionChecker
        :param alpha: Float value of the significance level to test each interaction
        :return: InteractionChecker
        """
        self.alpha        = alpha
        self.x_data       = None
        self.n            = 0
        self.y_data       = None
        self.prange       = None
        self.mat          = None
        self.int_list     = None
        self.interactions = None
        self.interaction_terms = None

    def fit(self, x_data, y_data, mp=True):
        """
        Loops through all potential interactions and adds to self.interactions if the p-value of the interaction coefficient is less than alpha.
        :param x_data: Pandas DataFrame of x values
        :param y_data: Pandas DataFrame of y values
        :param mp: set to False to use a single process for lower-dimensional data sets.
        :return: None
        """
        self.x_data    = x_data
        self.y_data    = y_data
        self.y         = self.y_data.values
        self.mat       = self.x_data.values.T
        self.n         = len(self.mat[0])
        self.prange    = range(0,len(self.x_data.columns)-1)
        self.int_list  = [(a, b) for a in self.prange for b in self.prange if a < b]
        # Test all potential interactions
        if mp:
            with Pool(processes=os.cpu_count() - 1) as pool:
                self.interactions =  pool.map(self.test_interaction,self.int_list)
        else:
            self.interactions =  [self.test_interaction(i) for i in self.int_list]

    def test_interaction(self, arg):
        """
        Tests a single interaction term with a linear regression model containing both terms and the ineraction term
        :param arg: Tuple of two predictor columns
        :return: False if interaction is rejected, an array of both predictors and the interaction values if interaction is not rejected
        """
        p1, p2 = arg
        inter = self.mat[p1]*self.mat[p2]
        # Reject if the standard devation of the interaction term is 0
        if std(inter) == 0:
            return False
        # Ndarray of both terms, plus the interaction term
        x = concatenate((self.mat[p1], self.mat[p2], inter)).reshape(self.n, 3)
        fr = f_regression(x,self.y_data)
        # Return both column indices and the interaction if the p-value of the interaction coefficient is less than alpha
        if fr[1][2] < self.alpha:
            return (self.x_data.columns[p1], self.x_data.columns[p2])
        # Reject if the p-value of the interaction coefficient is greater than alpha
        return False

    def transform(self, data):
        """
        Reshapes all interaction terms not rejected into a dataframe containing all interactions not rejected
        :return: Pandas dataframe object of all the interactions not rejected by test_interaction
        """
        # Column indices and values for non-rejected interactions
        sign_interacts = [i for i in self.interactions if i]
        columns=["{}*{}".format(i,j) for i, j in sign_interacts]
        m = [(data[i].values*data[j].values) for i, j in sign_interacts]
        return DataFrame(array(m).T, index=data.index, columns=columns)


# code for updating gene expression from ..2018

from scipy.io import loadmat
import pandas as pd
import os

#raw_dir = './RAW'
raw_dir = '../../../obsbrain_RAW/Arnatkeviciute2018/RAW/'
mat = loadmat(os.path.join(raw_dir, 'ROIxGene_aparcaseg.mat'))
mat.keys()

new_dir = './obs.DAT'
if not os.path.exists(new_dir):
    os.mkdir(new_dir)

gene_list = [gene[0][0] for gene in mat['probeInformation'][0][0][1]]
"""
for i, gene in enumerate(gene_list):
    col_idx = i+1
    gene_dat = mat['parcelExpression'][:,col_idx]
    out_file = os.path.join(new_dir, gene+'.obs')
    gene_dat_df = pd.DataFrame()
    gene_dat_df['X'] = gene_dat
    gene_dat_df.to_csv(out_file, index=False, header=False)
"""
# save gene list in the database
gene_list_pf = pd.DataFrame()
gene_list_pf['X'] = gene_list
gene_list_pf.to_csv('obs.gene_list.csv', index=False, header=False)

# save hemi list
hemi_list = ['lh']*34
hemi_list_pd = pd.DataFrame()
hemi_list_pd['Hemi'] = hemi_list
hemi_list_pd.to_csv('obs.hemi_list.csv', index=False, header=False)

gene_dat = mat['parcelExpression'][:,1:]
out_file = os.path.join(new_dir, 'obsDat.csv')
gene_dat_df = pd.DataFrame(data=gene_dat, columns=gene_list)
gene_dat_df.to_csv(out_file, index=False)

out_file = os.path.join(new_dir, 'obsDat.feather')
gene_dat_df.reset_index(inplace=True)
gene_dat_df['index']=u'r' + gene_dat_df['index'].astype(str)
gene_dat_df.columns = gene_dat_df.columns.map(str)
gene_dat_df.to_feather(out_file)


from rest_framework.views import APIView
from rest_framework.permissions import IsAuthenticated
from library.permissions import HasProduct, IsProductAuthenticated
from .models import Product
from .models_serializers import ProductSerializers
from .requests_serializers import ConfigureProductSerializer, \
    GetUserProductSerializer, ProductAuthSerializer #, FireSerializer
from library.api_response import ApiResponse
from rest_framework import status
from rest_framework.permissions import AllowAny
from library.helper import print_error_in_console

api_response = ApiResponse()

class ConfigureProductView(APIView):
    permission_class = [IsAuthenticated]

    def post(self, request):
        api_response.__init__()

        serializer = ConfigureProductSerializer(data=request.data, context={'request': request})

        if not serializer.is_valid():
            print_error_in_console("Configure New Product Request", serializer.errors)
            return api_response.set_status_code(status.HTTP_400_BAD_REQUEST).merge_dict(serializer.errors).response()

        product = ProductSerializers(serializer.save()).data

        return api_response.set_status_code(status.HTTP_200_OK).set_data("product", product).set_data("message",
                                                                                f"The product with ID [ {product.get('id')} ] is configured successfully").response()

class GetUserProductsView(APIView):
    permission_classes = [IsAuthenticated & HasProduct]

    def get(self, request, id=None):

        api_response.__init__()

        serializer = GetUserProductSerializer(data=request.data, context={'request': request, 'id': id})

        if not serializer.is_valid():
            print_error_in_console("Get All User Products Request", serializer.errors)
            return api_response.set_status_code(status.HTTP_400_BAD_REQUEST).merge_dict(serializer.errors).response()

        if id == None:
            products = ProductSerializers(serializer.save(), many=True).data

            return api_response.set_status_code(status.HTTP_200_OK).set_data("products", products).response()

        product = ProductSerializers(serializer.save()).data

        return api_response.set_status_code(status.HTTP_200_OK).set_data("product", product).response()

class UpdateProductView(APIView):
    permission_classes = [IsAuthenticated & HasProduct]

    def post(self, request, id):
        api_response.__init__()

        try:
            product = Product.objects.get(pk=id, user=self.request.user, is_active=True)
        except Product.DoesNotExist:
            return api_response.set_status_code(status.HTTP_404_NOT_FOUND).set_data("message",
                                                                                "This product not found").response()

        serializer = ProductSerializers(instance=product, data=request.data, context={'request': request, 'id': id},
                                        partial=True)

        if not serializer.is_valid():
            print_error_in_console("Update Product Request", serializer.errors)
            return api_response.set_status_code(status.HTTP_400_BAD_REQUEST).merge_dict(serializer.errors).response()

        product = serializer.save()

        return api_response.set_status_code(status.HTTP_200_OK).set_data("message", "Updated successfully").set_data(
            "product", serializer.data).response()

class ProductLoginView(APIView):
    permission_classes = [AllowAny]

    def post(self, request):
        api_response.__init__()

        serializer = ProductAuthSerializer(data=request.data)

        if not serializer.is_valid():
            print_error_in_console("Product Login Request", serializer.errors)
            return api_response.set_status_code(status.HTTP_400_BAD_REQUEST).merge_dict(serializer.errors).response()

        product, token = serializer.login()

        return api_response.set_status_code(status.HTTP_200_OK).set_data("product",
                                                                         ProductSerializers(product).data).set_data(
            "token", token).response()

class ProductLogoutView(APIView):
    permission_classes = [IsProductAuthenticated]

    def get(self, request):
        api_response.__init__()

        token = request.headers["Token"]
        product = Product.objects.get(token=token)
        product.token = None
        product.save()

        return api_response.set_status_code(status.HTTP_200_OK).set_data("message", "Loggedout successfully").response()

# class FireView(APIView):
#     permission_classes = [IsProductAuthenticated]
#
#     def get(self, request):
#         api_response.__init__()
#         serializer = FireSerializer(data=request.data, context={'token': request.headers["AUTHORIZATION"]})
#         if not serializer.is_valid():
#             print_error_in_console("Fire Alarm Request", serializer.errors)
#             return api_response.set_status_code(status.HTTP_400_BAD_REQUEST).merge_dict(serializer.errors).response()
#
#         serializer.send_mail()
#         # serializer.send_sms()
#
#         return api_response.set_status_code(status.HTTP_200_OK).set_data("message", "User Notified").response()
#


"""
This test script is to test payment ledger back-population for following Scenario:
- Citation Creation
- Citation Edit
- Citation Voiding
"""

from utility.citation_utility import edit_citation, void_citation
from env.session_manager import current_session
import random, json

test_prefix = "coincident/test_case_8:"

def editing_citation():
    data_to_validate = current_session.get_variable("test_case_8")
    for data in data_to_validate:
        ticket_num = data["ticket_num"]
        new_fine = random.randrange(start=150, stop=300, step=10)
        data_dict = {"ticket_num": ticket_num, "fine": new_fine}
        edit_citation(data_dict)
        data["new_fine"] = new_fine
    current_session.update_variable("test_case_8", data_to_validate)

def citation_voiding():
    data_to_validate = current_session.get_variable("test_case_8")
    for data in data_to_validate:
        ticket_num = data["ticket_num"]
        ticket_type = "local"
        void_citation(ticket_num=ticket_num, ticket_type=ticket_type, void_description="abcd")

def main(*args):

    editing_citation()
    citation_voiding()


import logging, torch, numpy as np, re, cv2 as cv, open3d as op3d

from typing import Union
from pytorch3d.structures import Meshes as Meshes_torch
from pytorch3d.io import load_objs_as_meshes
from pytorch3d.renderer import (
        PointLights,
        RasterizationSettings,
        MeshRenderer,
        MeshRasterizer,
        SoftPhongShader,
        FoVPerspectiveCameras,
        look_at_view_transform,
        TexturesVertex,
)
from pathlib import Path

path_mean_face_mesh = Path('/tmp/mean_face_mesh_basel.obj')

class DtypeTransformator():

    _default_bit_sys: int = 32
    _available_bit_sys: tuple = (32, 64)
    _available_types: tuple = ("int", "float")
    _torch_dtypes: dict = {'float32': torch.float32, 'float64': torch.float64,
                           'int32': torch.int32, 'int64': torch.int64}

    @staticmethod
    def wrapp_tensor(x :Union[np.ndarray, torch.Tensor], device:str, dtype:str, add_batch_axis:bool=True):
        if isinstance(x, np.ndarray):
            x = torch.from_numpy(x)
        if add_batch_axis:
            x = x[None]

        if str(x.device) != device:
            if device.find(':') != -1:
                try:
                    device_name, index = device.split(':')
                    x = x.to(f'{device_name}:{index}')
                    assert all(isinstance(v, str) for v in (device_name, index))
                except Exception as e:
                    logging.info(f'Incorrect device parameter: {device} with exception: {e}')
                    assert False
            else:
                x = x.to(device)
        if dtype:

            type = re.search(r'(float|int)', dtype)
            bit_system = re.search(r'\d+', dtype)
            if type:
                type = type.group()
                assert type in DtypeTransformator._available_types, f'Incorrect dtype: {dtype} parameter'
            else:
                assert False, f'Incorrect dtype: {dtype} parameter'
            if bit_system:
                bit_int = int(bit_system.group())
                assert bit_int in DtypeTransformator._available_bit_sys, f'Incorrect dtype: {dtype} parameter'
            else:
                # by default 32
                bit_int = DtypeTransformator._default_bit_sys
            key_dtype = f'{type}{bit_int}'
            assert key_dtype in DtypeTransformator._torch_dtypes, 'Incorrect class method construction!'

            dtype_torch = DtypeTransformator._torch_dtypes[key_dtype]
            x = x.to(dtype_torch)

        return x

class Render():

    def __init__(self, device:str=None):
        if device is None or device == 'cuda:0':
            if torch.cuda.is_available():
                self._device = torch.device("cuda:0")
                torch.cuda.set_device(self._device)
                logging.info(f'Set cuda:0 device!')
            else:
                self._device = torch.device("cpu")
                logging.info(f'Set cpu device!')

        assert self._device.type in ("cpu", "cuda"), f"Incorrect device parameter: {device}"

    @property
    def device_str_repr(self):
        return f"{self._device.type}:{self._device.index}"

    # TODO: something incorrect
    def get_torch_mesh_object_from_mesh(self, mesh, white_texture:bool=False):
        data = mesh.tensor_mesh_objects
        verts, faces, textures = data['vertices'], data['triangles'], data['textures']
        verts = DtypeTransformator.wrapp_tensor(verts, device=self.device_str_repr, dtype='float32',
                                                add_batch_axis=True)
        faces = DtypeTransformator.wrapp_tensor(faces, device=self.device_str_repr, dtype='float32',
                                                add_batch_axis=True)
        if white_texture:
            textures = torch.ones_like(verts, device=self.device_str_repr, dtype=torch.float32)
        else:
            textures = DtypeTransformator.wrapp_tensor(textures,
                                                       device=self.device_str_repr,
                                                       dtype='float32',
                                                       add_batch_axis=True)
        texture_vertex = TexturesVertex(verts_features=textures)
        return Meshes_torch(verts=verts, faces=faces, textures=texture_vertex)

    def get_torch_mesh_obj_from_file(self, path_mesh_obj, extract_texture:bool=True):
        if isinstance(path_mesh_obj, str):
            path_mesh_obj = Path(path_mesh_obj)
        assert path_mesh_obj.is_file(), 'The file {} not found!'
        mesh = load_objs_as_meshes([str(path_mesh_obj)], device=self._device)
        return mesh

    # for experiment purposes
    def define_render_instance(self, use_look_at_view:bool=True):
        if use_look_at_view:
            R, T = look_at_view_transform(3.0, 0, 0, degrees=True)
            cameras = FoVPerspectiveCameras(device=self._device, R=R, T=T)
        else:
            # default
            cameras = FoVPerspectiveCameras(device=self._device)

        raster_settings = RasterizationSettings(image_size=512, blur_radius=0.0, faces_per_pixel=1)
        lights = PointLights(device=self._device,
                             ambient_color=((0.8, 0.8, 0.8),),
                             specular_color=((0.5, 0.5, 0.5),),
                             diffuse_color=((0.1, 0.1, 0.1), ),
                             location=[[3.0, 3.0, -3.0]])

        renderer = MeshRenderer(
            rasterizer=MeshRasterizer(cameras=cameras, raster_settings=raster_settings),
            shader=SoftPhongShader(device=self._device, cameras=cameras, lights=lights)
        )
        return renderer


# WebDriver 对象有好几种方法，用于在页面中寻找元素。它们被分成find_element_*和
# find_elements_*方法。find_element_*方法返回一个 WebElement 对象，代表页面中
# 匹配查询的第一个元素。find_elements_*方法返回 WebElement_*对象的列表，包含
# 页面中所有匹配的元素。
#
# selenium 的 WebDriver 方法，用于寻找元素
# 方法名                                                 返回的 WebElement 对象/列表
# browser.find_element_by_class_name(name)              使用 CSS 类 name 的元素
# browser.find_elements_by_class_name(name)
#
# browser.find_element_by_css_selector(selector)        匹配 CSS selector 的元素
# browser.find_elements_by_css_selector(selector)
#
# browser.find_element_by_id(id)                        匹配 id 属性值的元素
# browser.find_elements_by_id(id)
#
# browser.find_element_by_link_text(text)               完全匹配提供的 text 的元素
# browser.find_elements_by_link_text(text)
#
# browser.find_element_by_partial_link_text(text)       包含提供的 text 的元素
# browser.find_elements_by_partial_link_text(text)
#
# browser.find_element_by_name(name)                    匹配 name 属性值的元素
# browser.find_elements_by_name(name)
#
# browser.find_element_by_tag_name(name)                匹配标签 name 的元素
# browser.find_elements_by_tag_name(name)               (大小写无关，元素匹配'a'和'A')
#
#
# 除了*_by_tag_name()方法，所有方法的参数都是区分大小写的。如果页面上没
# 有元素匹配该方法要查找的元素，selenium 模块就会抛出 NoSuchElement 异常。如
# 果你不希望这个异常让程序崩溃，就在代码中添加 try 和 except 语句。
# 一旦有了 WebElement 对象，就可以其中的属性，或调用其中的方法
#
# WebElement 的属性和方法
# 属性或方法                                描述
# tag_name                                标签名，例如 'a'表示元素
# get_attribute(name)                     该元素 name 属性的值
# text                                    该元素内的文本，例如hello中的'hello'
# clear()                                 对于文本字段或文本区域元素，清除其中输入的文本
# is_displayed()                          如果该元素可见，返回 True，否则返回 False
# is_enabled()                            对于输入元素，如果该元素启用，返回 True，否则返回 False
# is_selected()                           对于复选框或单选框元素，如果该元素被选中，选择 True，否则返回 False
# location                                一个字典，包含键'x'和'y'，表示该元素在页面上的位置

from selenium import webdriver

chrome_drive = r"/home/ryan/Downloads/chromedriver"
browser = webdriver.Chrome(executable_path=chrome_drive)
browser.get("https://www.baidu.com")

try:
    ret = browser.find_element_by_class_name('title-content-title')
    print(f"Find the {ret.tag_name} element with that class name!")
except:
    print("Fail to find the a element with that class name")