from flask import Flask, request, jsonify
from flask_restful import Resource, Api
from flask_sqlalchemy import SQLAlchemy
from flask_marshmallow import Marshmallow

DEPLOYMENT_ENVIRONMENT = 'docker'

app = Flask(__name__)
api = Api(app)
if DEPLOYMENT_ENVIRONMENT == 'docker':
    app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////data/container.db'
elif DEPLOYMENT_ENVIRONMENT == 'local':
    app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:////tmp/test.db'
else:
    app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///:memory:'
db = SQLAlchemy(app)
ma = Marshmallow(app)

class AppToBackendMapping(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    app_version = db.Column(db.String(20), unique=True, nullable=False)
    server_version = db.Column(db.String(20), unique=False, nullable=False)
    server_domain = db.Column(db.String(100), unique=False, nullable=False)

    def __repr__(self):
        return '<App Version: {app} | Server Version: {server} | Domain: {domain}>'.format(app=self.app_version, server=self.server_version, domain=self.server_domain)

class RuleSchema(ma.Schema):
    class Meta:
        # we want to expose:
        fields = ('app_version', 'server_version', 'server_domain')

rule_schema = RuleSchema()

def map_app_version_to_server_version(app_version = 'DEFAULT'):
    rule = AppToBackendMapping.query.filter_by(app_version=app_version).first()
    if rule is None:
        rule = AppToBackendMapping.query.filter_by(app_version='DEFAULT').first()
    return rule_schema.dump(rule).data, 200

class APIRouter(Resource):
    def get(self, app_version):
        return map_app_version_to_server_version(app_version=app_version)

api.add_resource(APIRouter, '/<string:app_version>')

if __name__ == '__main__':
    import migrate
    migrate.migrate_db()
    app.run(debug=True)


from jinja2 import Environment, PackageLoader, select_autoescape

class CombinedLicenseRenderer(object):
    def render(self, pkg_infos: list):
        env = Environment(
            loader=PackageLoader("leafhopper", "templates"),
            autoescape=select_autoescape()
        )
        template = env.get_template("LICENSES.txt.template")
        license_txt = template.render(pkg_infos=pkg_infos)
        return license_txt


from fastapi import APIRouter, File
from schemas.response import resp
import config
from utils.tool_func import get_random_filename, get_random_str
from common.logger import logger

router = APIRouter()

@router.post("/upload_wav")
async def create_file(file: bytes = File(...)):
    h = get_random_str()
    try:
        path = "{}/{}.wav".format(config.STATIC_DIR,h)
        f = open(path, 'wb')
        f.write(file)
        return resp.ok(data=path)
    except Exception:
        logger.error("写入错误")
        return resp.fail(resp.BusinessError)
    finally:
        f.close()


import collections
import gc
import os
from typing import List, Optional, Union, Tuple, Dict, Any

import albumentations
import numpy as np
import pandas as pd
import torch
from hydra.utils import instantiate
from pytorch_toolbelt.inference import (
    ApplySigmoidTo,
    ApplySoftmaxTo,
    Ensembler,
    GeneralizedTTA,
    MultiscaleTTA,
    d2_image_augment,
    d4_image_augment,
    d4_image_deaugment,
    d2_image_deaugment,
    flips_image_deaugment,
    flips_image_augment,
    fliplr_image_augment,
    fliplr_image_deaugment,
)
from pytorch_toolbelt.utils import to_numpy, fs
from torch import nn
from torch.utils.data import DistributedSampler
from tqdm import tqdm
from pytorch_toolbelt.utils.distributed import is_main_process, get_rank, get_world_size, all_gather

from xview3.centernet.bboxer import (
    MultilabelCircleNetDecodeResult,
    MultilabelCircleNetCoder,
)
from xview3.centernet.constants import (
    CENTERNET_OUTPUT_SIZE,
    CENTERNET_OUTPUT_OFFSET,
    CENTERNET_OUTPUT_OBJECTNESS_MAP,
    CENTERNET_OUTPUT_VESSEL_MAP,
    CENTERNET_OUTPUT_FISHING_MAP,
)
from xview3.centernet.models.inference import (
    multilabel_centernet_tiled_inference,
    get_box_coder_from_model,
)
from xview3.dataset import (
    XView3DataModule,
    read_multichannel_image,
    stack_multichannel_image,
)

__all__ = [
    "average_checkpoints",
    "ensemble_from_checkpoints",
    "model_from_checkpoint",
    "ensemble_from_config",
    "wrap_multilabel_model_with_tta",
    "maybe_run_inference",
]

def average_checkpoints(inputs):
    """Loads checkpoints from inputs and returns a model with averaged weights. Original implementation taken from:
    https://github.com/pytorch/fairseq/blob/a48f235636557b8d3bc4922a6fa90f3a0fa57955/scripts/average_checkpoints.py#L16
    Args:
      inputs (List[str]): An iterable of string paths of checkpoints to load from.
    Returns:
      A dict of string keys mapping to various values. The 'model' key
      from the returned dict should correspond to an OrderedDict mapping
      string parameter names to torch Tensors.
    """
    params_dict = collections.OrderedDict()
    params_keys = None
    new_state = None
    num_models = len(inputs)
    for fpath in inputs:
        with open(fpath, "rb") as f:
            state = torch.load(
                f,
                map_location="cpu",
            )
        # Copies over the settings from the first checkpoint
        if new_state is None:
            new_state = state
        model_params = state["model_state_dict"]
        model_params_keys = list(model_params.keys())
        if params_keys is None:
            params_keys = model_params_keys
        elif params_keys != model_params_keys:
            raise KeyError("For checkpoint {}, expected list of params: {}, " "but found: {}".format(f, params_keys, model_params_keys))
        for k in params_keys:
            p = model_params[k]
            if isinstance(p, torch.HalfTensor):
                p = p.float()
            if k not in params_dict:
                params_dict[k] = p.clone()
                # NOTE: clone() is needed in case of p is a shared parameter
            else:
                params_dict[k] += p
    averaged_params = collections.OrderedDict()
    for k, v in params_dict.items():
        averaged_params[k] = v
        if averaged_params[k].is_floating_point():
            averaged_params[k].div_(num_models)
        else:
            averaged_params[k] //= num_models
    new_state["model_state_dict"] = averaged_params
    return new_state

def model_from_checkpoint(checkpoint_config: Union[str, Dict], **kwargs) -> Tuple[nn.Module, Dict]:
    if isinstance(checkpoint_config, collections.Mapping):
        if "average_checkpoints" in checkpoint_config:
            checkpoint = average_checkpoints(checkpoint_config["average_checkpoints"])
        else:
            checkpoint_name = checkpoint_config["checkpoint"]
            if os.path.isfile(checkpoint_name):
                checkpoint = torch.load(checkpoint_name, map_location="cpu")
            else:
                checkpoint = torch.hub.load_state_dict_from_url(checkpoint_name)

        model_config = checkpoint["checkpoint_data"]["config"]["model"]
    else:
        checkpoint_name = checkpoint_config

        if os.path.isfile(checkpoint_name):
            checkpoint = torch.load(checkpoint_name, map_location="cpu")
        else:
            checkpoint = torch.hub.load_state_dict_from_url(checkpoint_name)

        model_config = checkpoint["checkpoint_data"]["config"]["model"]

    model_state_dict = checkpoint["model_state_dict"]

    model = instantiate(model_config, _recursive_=False)
    model.load_state_dict(model_state_dict, strict=False)

    return model.eval(), checkpoint

def wrap_multilabel_model_with_tta(model, tta_mode, with_offset=True, size_offsets=(0, -32, -64, +32, +64)):
    from xview3.centernet import (
        CENTERNET_OUTPUT_VESSEL_MAP,
        CENTERNET_OUTPUT_FISHING_MAP,
        CENTERNET_OUTPUT_OBJECTNESS_MAP,
        CENTERNET_OUTPUT_OFFSET,
        CENTERNET_OUTPUT_SIZE,
    )

    keys_to_deaug = [
        CENTERNET_OUTPUT_VESSEL_MAP,
        CENTERNET_OUTPUT_FISHING_MAP,
        CENTERNET_OUTPUT_OBJECTNESS_MAP,
        CENTERNET_OUTPUT_SIZE,
    ]
    if with_offset:
        keys_to_deaug.append(CENTERNET_OUTPUT_OFFSET)

    def _make_deaug_dict(keys, fn):
        return dict((key, fn) for key in keys)

    if tta_mode == "d4":
        return GeneralizedTTA(model, augment_fn=d4_image_augment, deaugment_fn=_make_deaug_dict(keys_to_deaug, d4_image_deaugment))
    elif tta_mode == "ms":
        return MultiscaleTTA(model, size_offsets)
    elif tta_mode == "d2-ms":
        return MultiscaleTTA(GeneralizedTTA(model, d2_image_augment, d2_image_deaugment), size_offsets)
    elif tta_mode == "d2":
        model = GeneralizedTTA(model, d2_image_augment, deaugment_fn=_make_deaug_dict(keys_to_deaug, d2_image_deaugment))
    elif tta_mode == "flips":
        model = GeneralizedTTA(model, flips_image_augment, deaugment_fn=_make_deaug_dict(keys_to_deaug, flips_image_deaugment))
    elif tta_mode == "fliplr":
        model = GeneralizedTTA(model, fliplr_image_augment, deaugment_fn=_make_deaug_dict(keys_to_deaug, fliplr_image_deaugment))
    elif tta_mode is None:
        return model
    else:
        raise KeyError("Unusupported TTA mode '" + tta_mode + "'")

    return model

def ensemble_from_checkpoints(
    checkpoint_fnames: List[str],
    strict=True,
    sigmoid_outputs=None,
    softmax_outputs=None,
    activation: str = "after_model",
    tta: Optional[str] = None,
    with_offset=True,
):
    if activation not in {None, "None", "after_model", "after_tta", "after_ensemble"}:
        raise KeyError(activation)

    models = []
    checkpoints = []

    for ck in checkpoint_fnames:
        model, checkpoint = model_from_checkpoint(ck, strict=strict)
        models.append(model)
        checkpoints.append(checkpoint)

    if activation == "after_model":
        if sigmoid_outputs is not None:
            models = [ApplySigmoidTo(m, output_key=sigmoid_outputs) for m in models]
            print("Applying sigmoid activation to", sigmoid_outputs, "after each model", len(models))

        if softmax_outputs is not None:
            models = [ApplySoftmaxTo(m, output_key=softmax_outputs) for m in models]
            print("Applying softmax activation to", softmax_outputs, "after each model", len(models))

    if len(models) > 1:
        model = Ensembler(models)
        if activation == "after_ensemble":
            if sigmoid_outputs is not None:
                model = ApplySigmoidTo(model, output_key=sigmoid_outputs)
                print("Applying sigmoid activation to", sigmoid_outputs, "after ensemble")

            if softmax_outputs is not None:
                model = ApplySoftmaxTo(model, output_key=softmax_outputs)
                print("Applying softmax activation to", softmax_outputs, "after ensemble")

    else:
        assert len(models) == 1
        model = models[0]

    if tta not in {None, "None"}:
        model = wrap_multilabel_model_with_tta(model, tta, with_offset=with_offset)
        print("Wrapping models with TTA", tta)

    if activation == "after_tta":
        if sigmoid_outputs is not None:
            model = ApplySigmoidTo(model, output_key=sigmoid_outputs)
            print("Applying sigmoid activation to ", sigmoid_outputs, " after TTA")
        if softmax_outputs is not None:
            model = ApplySoftmaxTo(model, output_key=softmax_outputs)
            print("Applying softmax activation to", softmax_outputs, "after TTA")

    return model.eval(), checkpoints

def ensemble_from_config(config: Dict[str, Any]):
    model, checkpoints = ensemble_from_checkpoints(
        checkpoint_fnames=config["ensemble"]["models"],
        strict=True,
        activation=config["ensemble"]["activation_after"],
        tta=config["ensemble"]["tta"],
        sigmoid_outputs=config["ensemble"]["sigmoid_outputs"],
        softmax_outputs=config["ensemble"]["softmax_outputs"],
        with_offset=config["ensemble"]["with_offset"],
    )
    box_coder = get_box_coder_from_model(model)
    model = model.eval().cuda()
    return model, checkpoints, box_coder

@torch.jit.optimized_execution(False)
def predict_multilabel_scenes(
    model,
    box_coder: MultilabelCircleNetCoder,
    scenes: List[str],
    channels: List[str],
    tile_step: int,
    tile_size: int,
    objectness_thresholds_lower_bound: float,
    normalization: Dict[str, albumentations.ImageOnlyTransform],
    accumulate_on_gpu: bool,
    fp16: bool,
    batch_size: int,
    apply_activation: bool,
    save_raw_predictions: bool,
    max_objects: int,
    channels_last: bool,
    output_predictions_dir=None,
) -> pd.DataFrame:
    if output_predictions_dir is not None:
        os.makedirs(output_predictions_dir, exist_ok=True)

    all_predictions = []

    scenes = np.array(scenes)
    world_size, local_rank = get_world_size(), get_rank()

    if world_size > 1:
        sampler = DistributedSampler(scenes, world_size, local_rank, shuffle=False)
        rank_local_indexes = np.array(list(iter(sampler)))
        scenes = scenes[rank_local_indexes]
        print("Node", local_rank, "got", len(scenes), "to process")
        torch.distributed.barrier()

    for scene in tqdm(scenes, desc=f"Inference at Node {local_rank}/{world_size}", position=local_rank):
        gc.collect()
        scene_id = fs.id_from_fname(scene)
        predictions = maybe_run_inference(
            model=model,
            box_coder=box_coder,
            scene=scene,
            output_predictions_dir=output_predictions_dir,
            accumulate_on_gpu=accumulate_on_gpu,
            tile_size=tile_size,
            tile_step=tile_step,
            fp16=fp16,
            batch_size=batch_size,
            save_raw_predictions=save_raw_predictions,
            apply_activation=apply_activation,
            max_objects=max_objects,
            channels_last=channels_last,
            normalization=normalization,
            channels=channels,
            objectness_thresholds_lower_bound=objectness_thresholds_lower_bound,
        )

        all_predictions.append(predictions)
        if output_predictions_dir is not None:
            predictions.to_csv(os.path.join(output_predictions_dir, scene_id + ".csv"), index=False)

    all_predictions = pd.concat(all_predictions).reset_index(drop=True)

    if world_size > 1:
        torch.distributed.barrier()
        all_predictions = pd.concat(all_gather(all_predictions)).reset_index(drop=True)

    return all_predictions

def maybe_run_inference(
    model,
    box_coder,
    scene,
    output_predictions_dir,
    channels,
    normalization,
    objectness_thresholds_lower_bound: float,
    tile_size,
    tile_step,
    accumulate_on_gpu,
    fp16,
    batch_size,
    save_raw_predictions,
    apply_activation,
    max_objects,
    channels_last,
):
    scene_id = fs.id_from_fname(scene)
    predictions_computed_offline = False

    if output_predictions_dir is not None:
        raw_predictions_file = os.path.join(output_predictions_dir, scene_id + ".npz")
        decoded_predictions_file = os.path.join(output_predictions_dir, scene_id + ".csv")

        if os.path.isfile(decoded_predictions_file):
            try:
                predictions = pd.read_csv(decoded_predictions_file)
                return predictions
            except Exception as e:
                print(e)
                predictions_computed_offline = False
        elif os.path.isfile(raw_predictions_file):
            try:
                saved_predictions = np.load(raw_predictions_file, allow_pickle=True)
                outputs = dict(
                    CENTERNET_OUTPUT_OBJECTNESS_MAP=torch.from_numpy(saved_predictions[CENTERNET_OUTPUT_OBJECTNESS_MAP]),
                    CENTERNET_OUTPUT_VESSEL_MAP=torch.from_numpy(saved_predictions[CENTERNET_OUTPUT_VESSEL_MAP]),
                    CENTERNET_OUTPUT_FISHING_MAP=torch.from_numpy(saved_predictions[CENTERNET_OUTPUT_FISHING_MAP]),
                    CENTERNET_OUTPUT_SIZE=torch.from_numpy(saved_predictions[CENTERNET_OUTPUT_SIZE]),
                    CENTERNET_OUTPUT_OFFSET=torch.from_numpy(saved_predictions[CENTERNET_OUTPUT_OFFSET])
                    if CENTERNET_OUTPUT_OFFSET in saved_predictions
                    else None,
                )
                predictions_computed_offline = True
            except Exception as e:
                print(e)
                predictions_computed_offline = False

    if not predictions_computed_offline:
        image = read_multichannel_image(scene, channels)
        for channel_name in set(channels):
            image[channel_name] = normalization[channel_name](image=image[channel_name])["image"]
        image = stack_multichannel_image(image, channels)

        outputs = multilabel_centernet_tiled_inference(
            model,
            image,
            box_coder=box_coder,
            tile_size=tile_size,
            tile_step=tile_step,
            accumulate_on_gpu=accumulate_on_gpu,
            fp16=fp16,
            batch_size=batch_size,
            channels_last=channels_last,
        )

        if save_raw_predictions and output_predictions_dir is not None:
            raw_predictions_file = os.path.join(output_predictions_dir, scene_id + ".npz")
            predictions_dict = dict(
                CENTERNET_OUTPUT_OBJECTNESS_MAP=to_numpy(outputs[CENTERNET_OUTPUT_OBJECTNESS_MAP]),
                CENTERNET_OUTPUT_VESSEL_MAP=to_numpy(outputs[CENTERNET_OUTPUT_VESSEL_MAP]),
                CENTERNET_OUTPUT_FISHING_MAP=to_numpy(outputs[CENTERNET_OUTPUT_FISHING_MAP]),
                CENTERNET_OUTPUT_SIZE=to_numpy(outputs[CENTERNET_OUTPUT_SIZE]),
            )
            if CENTERNET_OUTPUT_OFFSET in outputs:
                predictions_dict[CENTERNET_OUTPUT_OFFSET] = to_numpy(outputs[CENTERNET_OUTPUT_OFFSET])
            np.savez(raw_predictions_file, **predictions_dict)

    preds: MultilabelCircleNetDecodeResult = box_coder.decode(
        objectness_map=outputs[CENTERNET_OUTPUT_OBJECTNESS_MAP],
        is_vessel_map=outputs[CENTERNET_OUTPUT_VESSEL_MAP],
        is_fishing_map=outputs[CENTERNET_OUTPUT_FISHING_MAP],
        length_map=outputs[CENTERNET_OUTPUT_SIZE],
        offset_map=outputs.get(CENTERNET_OUTPUT_OFFSET, None),
        apply_activation=apply_activation,
        max_objects=max_objects,
    )

    pos_mask = preds.scores[0] >= objectness_thresholds_lower_bound

    centers = to_numpy(preds.centers[0][pos_mask]).astype(int)
    scores = to_numpy(preds.scores[0, pos_mask]).astype(np.float32)
    lengths = XView3DataModule.decode_lengths(preds.lengths[0, pos_mask])

    is_vessel_prob = to_numpy(preds.is_vessel[0, pos_mask]).astype(np.float32)
    is_fishing_prob = to_numpy(preds.is_fishing[0, pos_mask]).astype(np.float32)

    predictions = collections.defaultdict(list)
    for (
        (detect_scene_column, detect_scene_row),
        objectness_score,
        is_vessel_p,
        is_fishing_p,
        vessel_length_m,
    ) in zip(centers, scores, is_vessel_prob, is_fishing_prob, lengths):
        predictions["vessel_length_m"].append(vessel_length_m)
        predictions["detect_scene_row"].append(detect_scene_row)
        predictions["detect_scene_column"].append(detect_scene_column)
        predictions["scene_id"].append(scene_id)
        # Scores
        predictions["objectness_p"].append(objectness_score)
        predictions["is_vessel_p"].append(is_vessel_p)
        predictions["is_fishing_p"].append(is_fishing_p)
        # Thresholds
        predictions["objectness_threshold"].append(objectness_thresholds_lower_bound)

    predictions = pd.DataFrame.from_dict(predictions)
    return predictions


import pandas as pd
import matplotlib.pyplot as plt

EulerCromerData = pd.read_csv("../data/EulerCromer.csv")
VerletData = pd.read_csv("../data/Verlet.csv")
Yoshida4Data = pd.read_csv("../data/Yoshida4.csv")

fig, ax = plt.subplots()
ax.set_title("Orbits")
ax.set_xlabel("X position")
ax.set_ylabel("Y position")

ax.scatter([0], [0], marker="*", color="black", label="Star")
ax.plot(EulerCromerData["xpos"], EulerCromerData["ypos"],
        lw=0.5, label="EulerCromer")
ax.plot(VerletData["xpos"], VerletData["ypos"],
        lw=0.5, label="Verlet")
ax.plot(Yoshida4Data["xpos"], Yoshida4Data["ypos"],
        lw=0.5, label="Yoshida4")

ax.legend()
ax.set_aspect(1.0)
fig.tight_layout()

fig.savefig("../data/Orbits.jpeg")

fig, ax = plt.subplots()
ax.set_title("Energies")
ax.set_xlabel("Time")
ax.set_ylabel("Energy")

ax.plot(EulerCromerData["time"], EulerCromerData["energy"],
        lw=0.5, label="EulerCromer")
ax.plot(VerletData["time"], VerletData["energy"],
        lw=0.5, label="Verlet")
ax.plot(Yoshida4Data["time"], Yoshida4Data["energy"],
        lw=0.5, label="Yoshida4")

ax.legend()
fig.tight_layout()

fig.savefig("../data/Energies.jpeg")


#!/usr/bin/python

""" producer.py
Publish JSON data to the kafka broker every minute.
Three topics will be created
	1. sensors - all data
	2. sensorTrue - all true data
	3. sensorFalse - all false data

Multiple topics for UI debugging
"""
import json
import sys
import random
import time

from kafka import KafkaClient, KafkaProducer
from kafka.errors import KafkaUnavailableError

# Centeral point
AUSTIN_LAT_LONG = dict(lat=30.267153, lon=-97.743057)

# Build a bounding box for selection

def get_bounds(center, shift=1.0):
    """Take a dictionary with lat, lon and create the min/max for
            both lat and long

            @shift: float value to shift from center

            Returns dict of min,max lat and lon
            """
    assert(type(center) is dict)
    assert('lat' in center.keys())
    assert('lon' in center.keys())

    center_lat = center.get('lat')
    center_lon = center.get('lon')
    assert(center_lat is not None)
    assert(center_lon is not None)

    # Create the shifts
    min_lat = center_lat - shift
    max_lat = center_lat + shift
    min_lon = center_lon - shift
    max_lon = center_lon + shift
    return dict(min_lat=min_lat, max_lat=max_lat, min_lon=min_lon, max_lon=max_lon)

def get_multiple_points(bounds, num_points=10):
    assert(type(bounds) is dict)
    lons = (random.uniform(bounds.get('min_lon'), bounds.get('max_lon'))
            for _ in range(num_points))
    lats = (random.uniform(bounds.get('min_lat'), bounds.get('max_lat'))
            for _ in range(num_points))
    drone_detected = (random.choice([0, 1]) for _ in range(num_points))

    return (dict(lon=lon, lat=lat, isDrone=detect) for lon, lat, detect in zip(lons, lats, drone_detected))

if __name__ == '__main__':
	# URI to EC2
	uri = "ec2-54-236-152-162.compute-1.amazonaws.com:9092"

	# Create kafka client for topic generation
	print('Client connecting')
	try:
		client = KafkaClient(hosts=[uri])
	except KafkaUnavailableError as e:
		print('Kafka client unavailable')
		print(e)
		sys.exit()

	# Create publisher object
	print('Producer connecting')
	try:
		producer = KafkaProducer(bootstrap_servers=uri,
			acks=1,
			batch_size=10,
			retries=1,
			value_serializer=lambda m: json.dumps(m).encode('ascii'))
	except KafkaUnavailableError as e:
		print('Kafka producer unavailable')
		print(e)
		sys.exit()

	# Pull get the random data
	print('Getting data')
	bounds = get_bounds(AUSTIN_LAT_LONG, shift=0.01)

	client.ensure_topic_exists('sensors')
	client.ensure_topic_exists('drone')
	client.ensure_topic_exists('nodrone')

	# Run the service and push the data
	print('Pushing data')
	for _ in range(20000):
		# Get some random points
		points = get_multiple_points(bounds)
		for point in points:

			# Send all values
			producer.send('sensors', value=point)

			# Send only true values
			if point.get('isDrone'):
				producer.send('drone', value=point)

			# Send only false values
			if not point.get('isDrone'):
				producer.send('nodrone', value=point)

	producer.flush()
		#print('Sleeping')
		#time.sleep(5)

sys.exit()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
НЕ сделал
"""

import re
import sys
import requests

def reqsts(link):
    res = requests.get(link)
    if res.status_code == 200:
        return res.text
    else:
        return False

links = list()

for line in sys.stdin:
    links.append(line.strip())

for link in links:
    # req_link = re.findall(r"href=\"(.*)\"", reqsts(link))
    req_link = re.match(r"href=\"([\w\s:/\.]+?)\"", 'href="aaa" href="bbb"')
    print(req_link.groups())
    """
    res = requests.get(line)
    if res.status_code == 200:
        link = re.search(r"href=\"(.*)\"", res.text)
        print(link.group(1))
    """


import PySimpleGUI as sg
import os
from ..Components import config,game,stats
from ..Constants.constants import MENU_MUSIC_PATH

def jugar(window: sg.Window, event: str, nick: str, vlc_dict:dict):
    """Cierra el menu y abre la ventana de juego

    Args:
        window (sg.Window): La ventana donde ocurren los chequeos
        event (str): El boton que produce el evento
        nick (str): El nick del jugador
    """
    if event == "-PLAY-":
        window.close()
        game.start(nick, vlc_dict)

def configure(window: sg.Window, event: str, nick: str, vlc_dict:dict):
    """Cierra el menu y abre la ventana de configuracion

    Args:
        window (sg.Window): La ventana donde ocurren los chequeos
        event (str): El boton que produce el evento
        nick (str): El nick del jugador
    """
    if event == "-CONFIG-":
        window.close()
        config.start(nick, vlc_dict)

def statistics(window:sg.Window,event:str,nick:str,vlc_dict:dict,theme:str):
    "Cierra el menu y abre la ventana de Estadisticas"

    if event=="-STATS-":
        window.close()
        stats.start(nick,vlc_dict,theme)

def start_music(vlc_dict:dict):
    if vlc_dict["vlc"]:
        menu_song = os.path.join(os.getcwd(), MENU_MUSIC_PATH)
        background_music = vlc_dict["player_music"].get_instance().media_new(menu_song)
        vlc_dict["player_music"].set_media(background_music)
        vlc_dict["player_music"].play()


import argparse
import gzip
import struct
import sys
import os
import torch
from urllib.request import urlretrieve

parser = argparse.ArgumentParser(description='Preprocess MNIST')
parser.add_argument('--src-folder', type=str, default="data", metavar='Src', help='where to save raw mnist dataset')
parser.add_argument('--dst-folder', type=str, default="data", metavar='Dst', help='where to save tensor')
args = parser.parse_args()

TRAINING_SET_IMAGES = 'http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz'
TRAINING_SET_LABELS = 'http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz'

TEST_SET_IMAGES = 'http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz'
TEST_SET_LABELS = 'http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz'

file_to_download = [
	TRAINING_SET_IMAGES,
	TRAINING_SET_LABELS,

	TEST_SET_IMAGES,
	TEST_SET_LABELS
]

def reporthook(blocknum, blocksize, totalsize):
	readsofar = blocknum * blocksize
	if totalsize > 0:
		percent = readsofar * 1e2 / totalsize
		percent = 100 if percent > 100 else percent

		sys.stderr.write('\r{0:.2f}%'.format(percent))
		if readsofar >= totalsize: # near the end
			sys.stderr.write("\n")
	else: # total size is unknown
		sys.stderr.write("read %d\n" % (readsofar,))

def url_to_filename(url):
	return os.path.join(args.src_folder, url.split('/')[-1])

def label_to_tensor(src, dst):
	print('Parsing label to', dst)
	dst = os.path.join(args.dst_folder, dst)

	f = gzip.open(src, 'rb')
	content = f.read()
	f.close()

	magic, num_label, = struct.unpack(">II", content[:2*4])
	assert magic == 0x00000801, 'Magic mismatch.'

	labels = struct.unpack(">{}".format('B' * num_label), content[8:])
	labels = torch.LongTensor(labels)
	torch.save(labels, dst)

def image_to_tensor(src, dst):
	print('Parsing image to', dst)
	dst = os.path.join(args.dst_folder, dst)

	f = gzip.open(src, 'rb')
	content = f.read()
	f.close()

	magic, num_image, rows, cols, = struct.unpack(">IIII", content[:4*4])
	assert magic == 0x00000803, 'Magic mismatch.'

	images = struct.unpack(">{}".format('B' * num_image * rows * cols), content[4*4:])
	images = torch.Tensor(images)
	images = images.view(-1, 28, 28)
	torch.save(images, dst)

def download():
	if not os.path.exists('data'):
		os.mkdir('data')
	for url in file_to_download:
		save_to = url_to_filename(url)
		if not os.path.exists(save_to):
			print("Downloading:", url)
			urlretrieve(url, save_to, reporthook)
		else:
			print("Downloaded. skip", url)

if __name__ == '__main__':
	if not os.path.exists(args.src_folder):
		os.mkdir(args.src_folder)
	if not os.path.exists(args.dst_folder):
		os.mkdir(args.dst_folder)

	download()

	image_to_tensor(url_to_filename(TRAINING_SET_IMAGES), 'training_images.pth')
	label_to_tensor(url_to_filename(TRAINING_SET_LABELS), 'training_labels.pth')
	image_to_tensor(url_to_filename(TEST_SET_IMAGES), 'test_images.pth')
	label_to_tensor(url_to_filename(TEST_SET_LABELS), 'test_labels.pth')


import torch.nn as nn
import torch
from torch.autograd import Function

class Retain(nn.Module):
    def __init__(self, dict_len, embedding_dim,  transformer_hidden, attn_heads, transformer_dropout, transformer_layers):
        super().__init__()

        retain_embed = 2*embedding_dim
        self.softmax = torch.nn.Softmax(dim=-1)
        self.retain_embed = nn.Linear(dict_len, retain_embed)
        self.gru = nn.GRU(input_size = retain_embed, hidden_size = retain_embed, batch_first = True, bidirectional = False)
        self.retain_attn_a = nn.Linear(retain_embed, 1)
        self.gru_b = nn.GRU(input_size = retain_embed, hidden_size = retain_embed, batch_first = True, bidirectional = False)
        self.retain_attn_b = nn.Sequential(nn.Linear(retain_embed, retain_embed), nn.Tanh())
        self.pred_feat = nn.Sequential(nn.Linear(4*retain_embed, 2*retain_embed), nn.Tanh())
        self.classify_layer = nn.Linear(retain_embed, 1)

    def forward(self, batch_input, time_info, visit_length):
        batch_input = batch_input[:,0:50,:]
        input_embedding = self.retain_embed(batch_input)
        gru_output = self.gru(input_embedding)

        gru_embedding = gru_output[0]

        attn_a = self.retain_attn_a(gru_embedding).squeeze(-1)
        attn_a = self.softmax(attn_a).unsqueeze(-1).permute(0, 2, 1)

        gru_b_output = self.gru_b(input_embedding)
        gru_b_embedding = gru_b_output[0]
        attn_b = self.retain_attn_b(gru_b_embedding)

        attn_b_input = input_embedding * attn_b
        context = torch.matmul(attn_a, attn_b_input).squeeze(1)

        classify_score = self.classify_layer(context)
        binary_output = torch.sigmoid(classify_score)

        return binary_output


import sys
import numpy as np
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import cross_val_score
from numpy import mean
from numpy import std
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_validate
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import f1_score
from ClassificationMetrics import ClassificationMetrics

def SGDClass(data, labels, nof,file,filename ):
    params = {
        'loss' : ['hinge' , 'log', 'modified_huber', 'squared_hinge', 'perceptron'],
        'penalty' : ['l2', 'l1', 'elasticnet'],
        'max_iter' : [100 , 500 ,1000],
        'learning_rate': ['constant' , 'optimal' , 'invscaling' , 'adaptive'],
    }
    clf = GridSearchCV(SGDClassifier(), params, n_jobs=-1, cv=nof)
    clf.fit(data, labels)
    best_parameters = clf.best_params_
    ls = best_parameters['loss']
    pe = best_parameters['penalty']
    mi = best_parameters['max_iter']
    lr = best_parameters['learning_rate']

    model = SGDClassifier(loss=ls , penalty=pe ,max_iter=mi,learning_rate=lr)
    # cvs = cross_val_score(model, data, labels, cv=nof)
    # print(cvs)
    # print(cvs.mean() , "\t" , cvs.std())
    r = ClassificationMetrics()
    r.set_predictorName("SGDClassifier")
    r.set_contin(False)
    r.set_DatasetName(filename)

    bestparameters = 'loss= ' +str(ls) + ' penalty= '+str(pe)+ ' max_iter= '+str(mi)+ ' learning_rate= '+str(lr)
    r.set_best(bestparameters)
    r.PrintbestParameters()

    cv_results = cross_validate(model, data, labels, cv=nof, scoring=r.confusion_matrix_scorer, return_train_score = True)
    r.Print("SGDClassifier")
    r.Print("test")
    r.Results("acc" , cv_results['test_acc'])
    r.Results("MCR" , cv_results['test_MisclassificationRate'])
    r.Results("re" , cv_results['test_re'])
    r.Results("Sen" , cv_results['test_Sensitivity'])
    r.Results("pre" , cv_results['test_pre'])
    r.Results("F_S" , cv_results['test_f_sc'])
    # r.Results("Specificity" , cv_results['test_Specificity'])
    # r.Results("roc" , cv_results['test_roc'])
    # r.Results("fpr" , cv_results['test_fpr'])
    # r.Results("tpr" , cv_results['test_tpr'])
    # r.Results("auc" , cv_results['test_auc'])
    r.Print("train")
    r.Results("acc" , cv_results['train_acc'])
    r.Results("MCR" , cv_results['train_MisclassificationRate'])
    r.Results("re" , cv_results['train_re'])
    r.Results("Sen" , cv_results['train_Sensitivity'])
    r.Results("pre" , cv_results['train_pre'])
    r.Results("F_S" , cv_results['train_f_sc'])
    # r.Results("Specificity" , cv_results['train_Specificity'])
    # r.Results("roc" , cv_results['train_roc'])
    # r.Results("fpr" , cv_results['train_fpr'])
    # r.Results("tpr" , cv_results['train_tpr'])
    # r.Results("auc" , cv_results['train_auc'])


import sys
from collections import deque, Counter, defaultdict
sys.setrecursionlimit(5 * 10 ** 5)
# from pypyjit import set_param
# set_param('max_unroll_recursion=-1')
input = lambda: sys.stdin.readline().rstrip()
ii = lambda: int(input())
mi = lambda: map(int, input().split())
li = lambda: list(mi())
inf = 2 ** 63 - 1

def solve(N: int, x: "List[int]", y: "List[int]"):

    if N == 1:
        print(1)
    else:
        d = []
        for i in range(N):
            for j in range(N):
                if i == j:
                    continue
                dx = x[i] - x[j]
                dy = y[i] - y[j]
                d.append((dx,dy))
        c = Counter(d).most_common(1)
        max_c = c[0][1]

        print(N-max_c)

def main():
    N = ii()  # type: int
    x = [int()] * (N)  # type: "List[int]"
    y = [int()] * (N)  # type: "List[int]"
    for i in range(N):
        x[i], y[i] = mi()
    solve(N, x, y)
    return

main()


from typing import List

def quick_sort(array: List) -> List:
    """Quick Sorting algorithum
    pivot 을 기준으로 분류하고, 최소 크기로 쪼개어 정렬하는 방법

    Args:
        array (List): 정렬할 리스트

    Returns:
        List: 정렬된 리스트
    """
    if len(array) <= 1:
        return array

    else:
        pivot = array[0]

        greater_list = []
        less_list = []
        eq_list = []
        for element in array:
            if element < pivot:
                less_list.append(element)

            elif element == pivot:
                eq_list.append(element)

            else:
                greater_list.append(element)

        return quick_sort(less_list) + eq_list + quick_sort(greater_list)

if __name__ == "__main__":
    item_list = [2, 5, 1, 9, 2, 3, 2, 1]
    result = quick_sort(item_list)
    print(result)


#!/usr/bin/python
# -*- coding: UTF-8 -*-
# 性别年龄识别WebAPI 接口：即机器对说话者的年龄大小以及性别属性进行分析，可以通过收到的音频数据判定发音人的性别（男，女）及年龄范围（小孩，中年，老人），对语音内容和语种不做限制。
# 该语音能力是通过Websocket API的方式给开发者提供一个通用的接口。Websocket API具备流式传输能力，适用于需要流式数据传输的AI服务场景，比如边说话边识别。
import datetime
import hashlib
import base64
import hmac
import json
import time
from urllib.parse import urlencode
from wsgiref.handlers import format_date_time
from datetime import datetime
from time import mktime
import websocket
import ssl
import _thread as thread

STATUS_FIRST_FRAME = 0  # 第一帧的标识
STATUS_CONTINUE_FRAME = 1  # 中间帧标识
STATUS_LAST_FRAME = 2  # 最后一帧的标识

# 用于存储年龄和性别信息
age_data = []
gender_data = []

class Ws_Param(object):  # 定义类用于管理 websocket 参数
    # 初始化
    def __init__(self, APPID, APIKey, APISecret, AudioFile):
        self.APPID = APPID
        self.APIKey = APIKey
        self.APISecret = APISecret
        self.AudioFile = AudioFile  # 说话人的音频文件

        # 公共参数(common)
        self.CommonArgs = {"app_id": self.APPID}
        # 业务参数(business)
        self.BusinessArgs = {"ent": "igr",  # 引擎类型，目前仅支持igr
                             "aue": "raw",  # 音频格式 raw：支持pcm格式和wav格式
                             "rate": 16000  # 音频采样率 16000/8000
                             }

    # 生成url，通过在请求地址后面加上鉴权相关参数
    def create_url(self):
        # 参数1：生成RFC1123格式的时间戳
        now = datetime.now()
        date = format_date_time(mktime(now.timetuple()))

        # 参数2： 生成鉴权authorization参数（包括下面step1、2、3、4、5)
        # step1 拼接signature原始字段的字符串（signature原始字段包括host、date、authorization）
        signature_origin = "host: " + "ws-api.xfyun.cn" + "\n"
        signature_origin += "date: " + date + "\n"
        signature_origin += "GET " + "/v2/igr " + "HTTP/1.1"
        # step2 结合apiSecret对signature_origin，使用hmac-sha256进行加密（签名）
        signature_sha = hmac.new(self.APISecret.encode('utf-8'), signature_origin.encode('utf-8'),
                                 digestmod=hashlib.sha256).digest()
        # step3 使用base64编码对signature_sha进行编码获得最终的signature_sha
        signature_sha = base64.b64encode(signature_sha).decode(encoding='utf-8')
        # step4 根据以上信息拼接authorization base64编码前（authorization_origin）的字符串
        authorization_origin = "api_key=\"%s\", algorithm=\"%s\", headers=\"%s\", signature=\"%s\"" % (
            self.APIKey, "hmac-sha256", "host date request-line", signature_sha)
        # step5 对authorization_origin进行base64编码获得最终的authorization参数
        authorization = base64.b64encode(authorization_origin.encode('utf-8')).decode(encoding='utf-8')

        # 将请求的鉴权参数1、2及host组合为字典
        v = {
            "authorization": authorization,
            "date": date,
            "host": "ws-api.xfyun.cn"
        }
        # 请求地址
        url = 'wss://ws-api.xfyun.cn/v2/igr'
        # 在请求地址后面加上鉴权相关参数（拼接鉴权参数），生成url
        url = url + '?' + urlencode(v)
        return url

# 收到websocket错误的处理
def on_error(ws, error):  # websocket报错时，就会触发on_error事件
    print("错误信息:", error)

# 收到websocket关闭的处理
def on_close(ws):  # websocket关闭时，就会触发on_close事件
    print("websocket连接关闭")

def data_send(ws, file, commonargs, businessargs):
    frameSize = 5000  # 每一帧的音频大小
    intervel = 0.04  # 发送音频间隔(单位:s)
    status = STATUS_FIRST_FRAME  # 音频的状态信息，标识音频是第一帧，还是中间帧、最后一帧
    # 业务数据流参数
    data_format = "audio/L16;rate=16000"  # 音频的采样率支持16k和8k, 8k音频：audio/L16;rate=8000
    data_encoding = "raw"  # 音频数据格式, lame：mp3格式  raw：原生音频（支持单声道的pcm,wav）

    with open(file, "rb") as fp:
        while True:
            buf = fp.read(frameSize)
            # 文件结束
            if not buf:
                status = STATUS_LAST_FRAME
            # 第一帧处理
            # 发送第一帧音频，带business 参数
            # appid 必须带上，只需第一帧发送
            if status == STATUS_FIRST_FRAME:
                d = {"common": commonargs,  # 公共参数APPid
                     "business": businessargs,
                     "data": {"status": 0,  # 音频的状态 0 :第一帧音频
                              "format": data_format,
                              "audio": str(base64.b64encode(buf), 'utf-8'),  # 音频内容，采用base64编码
                              "encoding": data_encoding
                              }
                     }
                d = json.dumps(d)  # 将请求数据转化为字符串
                ws.send(d)  # 发送数据
                status = STATUS_CONTINUE_FRAME

            # 中间帧处理
            elif status == STATUS_CONTINUE_FRAME:
                d = {"data": {"status": 1,  # 音频的状态 1 :中间的音频
                              "format": data_format,
                              "audio": str(base64.b64encode(buf), 'utf-8'),
                              "encoding": data_encoding
                              }
                     }
                ws.send(json.dumps(d))
            # 最后一帧处理
            elif status == STATUS_LAST_FRAME:
                d = {"data": {"status": 2,  # 音频的状态 2 :最后一帧音频，最后一帧必须要发送
                              "format": data_format,
                              "audio": str(base64.b64encode(buf), 'utf-8'),
                              "encoding": data_encoding
                              }
                     }
                ws.send(json.dumps(d))
                time.sleep(1)
                break
            # 模拟音频采样间隔
            time.sleep(intervel)
    ws.close()  # 关闭websocket

def data_write(message, age_data, gender_data):
    try:
        code = json.loads(message)["code"]  # 返回码，0表示成功，其它表示异常
        sid = json.loads(message)["sid"]  # 本次会话的id，只在握手成功后第一帧请求时返回
        if code != 0:
            errMsg = json.loads(message)["message"]
            print("sid:%s call error:%s code is:%s" % (sid, errMsg, code))

        else:
            age_data.append(json.loads(message)["data"]["result"]["age"])
            gender_data.append(json.loads(message)["data"]["result"]["gender"])
    except Exception as e:
        print("接收消息，但解析异常:", e)

def age_gender_api_get_result(APPID, APIKey, APISecret, filename):
    # 实例化websocket参数类
    wsParam = Ws_Param(APPID=APPID,
                       APIKey=APIKey,
                       APISecret=APISecret,
                       AudioFile=filename)
    wsUrl = wsParam.create_url()  # 获取鉴权url

    # 收到websocket消息的处理：把服务端返回的json数据进行解析
    def on_message(ws, message):
        data_write(message, age_data, gender_data)

    # 建立websocket连接,客户端的数据发送给服务端
    def on_open(ws):  # 连接到服务器之后就会触发on_open事件
        def run(*args):
            data_send(ws, wsParam.AudioFile, wsParam.CommonArgs, wsParam.BusinessArgs)
        thread.start_new_thread(run, ())  # 产生新线程

    # 向服务器端发送Websocket协议握手请求
    # 握手成功后，客户端通过Websocket连接同时上传和接收数据。
    # 接收到服务器端的结果全部返回标识后断开Websocket连接
    websocket.enableTrace(False)  # 调试关闭
    ws = websocket.WebSocketApp(wsUrl,
                                on_open=on_open,
                                on_message=on_message,
                                on_error=on_error,
                                on_close=on_close)

    ws.run_forever(sslopt={"cert_reqs": ssl.CERT_NONE})  # CERT_NONE 禁用SSL证书验证

    return age_data, gender_data


import time

import pytest

from arango.database import AsyncDatabase
from arango.exceptions import (
    AQLQueryExecuteError,
    AsyncExecuteError,
    AsyncJobCancelError,
    AsyncJobClearError,
    AsyncJobListError,
    AsyncJobResultError,
    AsyncJobStatusError,
)
from arango.job import AsyncJob
from tests.helpers import extract

def wait_on_job(job):
    """Block until the async job is done."""
    while job.status() != "done":
        time.sleep(0.05)  # pragma: no cover
    return job

def wait_on_jobs(db):
    """Block until all async jobs are finished."""
    while len(db.async_jobs("pending")) > 0:
        time.sleep(0.05)  # pragma: no cover

def test_async_wrapper_attributes(db, col, username):
    async_db = db.begin_async_execution()
    assert isinstance(async_db, AsyncDatabase)
    assert async_db.username == username
    assert async_db.context == "async"
    assert async_db.db_name == db.name
    assert async_db.name == db.name
    assert repr(async_db) == f"<AsyncDatabase {db.name}>"

    async_col = async_db.collection(col.name)
    assert async_col.username == username
    assert async_col.context == "async"
    assert async_col.db_name == db.name
    assert async_col.name == col.name

    async_aql = async_db.aql
    assert async_aql.username == username
    assert async_aql.context == "async"
    assert async_aql.db_name == db.name

    job = async_aql.execute("INVALID QUERY")
    assert isinstance(job, AsyncJob)
    assert isinstance(job.id, str)
    assert repr(job) == f"<AsyncJob {job.id}>"

def test_async_execute_without_result(db, col, docs):
    # Insert test documents asynchronously with return_result set to False
    async_col = db.begin_async_execution(return_result=False).collection(col.name)

    # Ensure that no jobs were returned
    assert async_col.insert(docs[0]) is None
    assert async_col.insert(docs[1]) is None
    assert async_col.insert(docs[2]) is None

    # Ensure that the operations went through
    wait_on_jobs(db)
    assert extract("_key", col.all()) == ["1", "2", "3"]

def test_async_execute_error_in_result(db, col, docs):
    db.collection(col.name).import_bulk(docs)
    async_db = db.begin_async_execution(return_result=True)

    # Test async execution of a bad AQL query
    job = wait_on_job(async_db.aql.execute("INVALID QUERY"))
    with pytest.raises(AQLQueryExecuteError) as err:
        job.result()
    assert err.value.error_code == 1501

def test_async_get_job_status(db, bad_db):
    async_db = db.begin_async_execution(return_result=True)

    # Test get status of a pending job
    job = async_db.aql.execute("RETURN SLEEP(0.1)", count=True)
    assert job.status() == "pending"

    # Test get status of a finished job
    assert wait_on_job(job).status() == "done"
    assert job.result().count() == 1

    # Test get status of a missing job
    with pytest.raises(AsyncJobStatusError) as err:
        job.status()
    assert err.value.error_code == 404

    # Test get status from invalid job
    bad_job = wait_on_job(async_db.aql.execute("INVALID QUERY"))
    bad_job._conn = bad_db._conn
    with pytest.raises(AsyncJobStatusError) as err:
        bad_job.status()
    assert err.value.error_code in {11, 1228}

def test_async_get_job_result(db, bad_db):
    async_db = db.begin_async_execution(return_result=True)

    # Test get result from a pending job
    job = async_db.aql.execute("RETURN SLEEP(0.1)", count=True)
    with pytest.raises(AsyncJobResultError) as err:
        job.result()
    assert err.value.http_code == 204
    assert f"{job.id} not done" in str(err.value)

    # Test get result from a finished job
    assert wait_on_job(job).result().count() == 1

    # Test get result from a cleared job
    with pytest.raises(AsyncJobResultError) as err:
        job.result()
    assert err.value.error_code == 404

    # Test get result from an invalid job
    bad_job = async_db.aql.execute("INVALID QUERY")
    bad_job._conn = bad_db._conn
    with pytest.raises(AsyncJobResultError) as err:
        bad_job.result()
    assert err.value.error_code in {11, 1228}

def test_async_cancel_job(db, bad_db):
    async_db = db.begin_async_execution(return_result=True)

    # Start a long running request to ensure that job can be cancelled
    job = async_db.aql.execute("RETURN SLEEP(5)")

    # Test cancel a pending job
    assert job.cancel() is True

    # Test cancel a missing job
    job._id = "invalid_id"
    with pytest.raises(AsyncJobCancelError) as err:
        job.cancel(ignore_missing=False)
    assert err.value.error_code == 404
    assert job.cancel(ignore_missing=True) is False

    # Test cancel an invalid job
    job = async_db.aql.execute("RETURN SLEEP(5)")
    job._conn = bad_db._conn
    with pytest.raises(AsyncJobCancelError) as err:
        job.cancel()
    assert err.value.error_code in {11, 1228}

def test_async_clear_job(db, bad_db):
    async_db = db.begin_async_execution(return_result=True)

    job = async_db.aql.execute("RETURN 1")

    # Test clear finished job
    assert job.clear(ignore_missing=True) is True

    # Test clear missing job
    with pytest.raises(AsyncJobClearError) as err:
        job.clear(ignore_missing=False)
    assert err.value.error_code == 404
    assert job.clear(ignore_missing=True) is False

    # Test clear with an invalid job
    job._conn = bad_db._conn
    with pytest.raises(AsyncJobClearError) as err:
        job.clear()
    assert err.value.error_code in {11, 1228}

def test_async_execute_errors(bad_db):
    bad_async_db = bad_db.begin_async_execution(return_result=False)
    with pytest.raises(AsyncExecuteError) as err:
        bad_async_db.aql.execute("RETURN 1")
    assert err.value.error_code in {11, 1228}

    bad_async_db = bad_db.begin_async_execution(return_result=True)
    with pytest.raises(AsyncExecuteError) as err:
        bad_async_db.aql.execute("RETURN 1")
    assert err.value.error_code in {11, 1228}

def test_async_clear_jobs(db, bad_db, col, docs):
    async_db = db.begin_async_execution(return_result=True)
    async_col = async_db.collection(col.name)

    job1 = wait_on_job(async_col.insert(docs[0]))
    job2 = wait_on_job(async_col.insert(docs[1]))
    job3 = wait_on_job(async_col.insert(docs[2]))

    # Test clear all async jobs
    assert db.clear_async_jobs() is True
    for job in [job1, job2, job3]:
        with pytest.raises(AsyncJobStatusError) as err:
            job.status()
        assert err.value.error_code == 404

    # Set up test documents again
    job1 = wait_on_job(async_col.insert(docs[0]))
    job2 = wait_on_job(async_col.insert(docs[1]))
    job3 = wait_on_job(async_col.insert(docs[2]))

    # Test clear jobs that have expired
    past = int(time.time()) - 1000000
    assert db.clear_async_jobs(threshold=past) is True
    for job in [job1, job2, job3]:
        assert job.status() == "done"

    # Test clear jobs that have not expired yet
    future = int(time.time()) + 1000000
    assert db.clear_async_jobs(threshold=future) is True
    for job in [job1, job2, job3]:
        with pytest.raises(AsyncJobStatusError) as err:
            job.status()
        assert err.value.error_code == 404

    # Test clear job with bad database
    with pytest.raises(AsyncJobClearError) as err:
        bad_db.clear_async_jobs()
    assert err.value.error_code in {11, 1228}

def test_async_list_jobs(db, col, docs):
    async_db = db.begin_async_execution(return_result=True)
    async_col = async_db.collection(col.name)

    job1 = wait_on_job(async_col.insert(docs[0]))
    job2 = wait_on_job(async_col.insert(docs[1]))
    job3 = wait_on_job(async_col.insert(docs[2]))

    # Test list async jobs that are done
    job_ids = db.async_jobs(status="done")
    assert job1.id in job_ids
    assert job2.id in job_ids
    assert job3.id in job_ids

    # Test list async jobs that are pending
    job4 = async_db.aql.execute("RETURN SLEEP(0.3)")
    assert db.async_jobs(status="pending") == [job4.id]
    wait_on_job(job4)  # Make sure the job is done

    # Test list async jobs with invalid status
    with pytest.raises(AsyncJobListError) as err:
        db.async_jobs(status="bad_status")
    assert err.value.error_code == 400

    # Test list jobs with count
    job_ids = db.async_jobs(status="done", count=1)
    assert len(job_ids) == 1
    assert job_ids[0] in [job1.id, job2.id, job3.id, job4.id]


import requests
from bs4 import BeautifulSoup as bs
import json
import csv

def parc():
    headers={
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
        'user-agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Mobile Safari/537.36'
        }

    all_list=[]
    for i in range(1,9):

        ur=f'https://shop.casio.ru/catalog/?PAGEN_1={i}'
        req=requests.get(ur,headers=headers)
        src=req.content
        soup=bs(src,'lxml')
        item__link=soup.find_all(class_='product-item__link')
        item__articul=soup.find_all(class_='product-item__articul')
        item_price=soup.find_all(class_='product-item__price')

        for e,m,t in zip(item__articul,item_price,item__link):
            s='https://shop.casio.ru'+t.get('href')
            s=s.strip()
            e=e.text
            e=e.strip()
            m=m.text
            m=m.strip()
            all={
                'модель часов':e,
                'цена':m,
                'ссылка':s

            }

            all_list.append(all)

            with open('parc.csv','a',encoding='utf-8') as file:
                file_writer = csv.writer(file)
                file_writer.writerow((
                        e,
                        m,
                        s
                        ))
    with open('parc.json','a',encoding='utf-8') as file:
        json.dump(all_list,file,indent=4,ensure_ascii=False)

parc()


from flask import request, current_app as app
from flask_api import status
from flask_jwt import current_identity, jwt_required
from sqlalchemy.sql import exists, and_, expression

from receipt_split.models import Friend, User
from receipt_split.schemas import friend_schema, friends_schema, users_schema
from receipt_split.meta import db
from . import create_view, Call, View, get_model_view, accept_reject_view,\
    is_authorized, views, err

@views.route('/friend/<username>', methods=['POST'])
@jwt_required()
def friend_add(username):
    friend = User.query.filter_by(username=username).first()

    if friend is None:
        app.logger.info("/friend/%s does not exist - %s", username,
                        current_identity.username)
        return err("friend does not exist"), status.HTTP_400_BAD_REQUEST

    if friend == current_identity:
        app.logger.info("/friend/%s is self - %s", username,
                        current_identity.username)
        return err("cannot friend yourself"), status.HTTP_400_BAD_REQUEST

    if friend in current_identity.friends:
        return err("friend already added"), status.HTTP_400_BAD_REQUEST

    if db.session.query(expression.literal(True)).filter(
        db.session.query(Friend)
        .filter(
            Friend.from_user_id == current_identity.id,
            Friend.to_user_id == friend.id,
            Friend.accepted.is_(None)
        )
        .exists()
    ).scalar():
        return err("Friend request already sent"), status.HTTP_404_NOT_FOUND

    friend_request = Friend(from_user=current_identity, to_user=friend)

    db.session.add(friend_request)
    db.session.commit()

    friend_dump = friend_schema.dump(friend_request)

    app.logger.info("/friend/%s request for %s - ",
                    current_identity.username, friend_dump)

    return friend_dump, status.HTTP_200_OK

@views.route('/friends', methods=['GET', 'PUT'])
@jwt_required()
def friend_list():
    friends = users_schema.dump(current_identity.friends)
    friends_received = friends_schema.dump(
        Friend.get_received(current_identity)
    )
    friends_sent = friends_schema.dump(
        Friend.get_sent(current_identity)
    )
    app.logger.info("/friend list get - %s - %s", current_identity.username,
                    friends)

    friend_result = {
        "friends_received": friends_received,
        "friends_sent": friends_sent,
        "friends": friends
    }

    if request.method == 'PUT':
        Friend.archive_sent(current_identity)

    return friend_result, status.HTTP_200_OK

@views.route('/friends/<int:id>')
@views.route('/friends/<int:id>/<action>', methods=["GET", "POST"])
@jwt_required()
def friends(id, action=None):
    friend = Friend.query.get(id)

    if not friend:
        app.logger.info("/friend/%s NOT FOUND", id)
        return err("friend not found"), \
            status.HTTP_404_NOT_FOUND

    if friend.to_user_id != current_identity.id:
        app.logger.info("/friend/%s from_user incorrect", id)
        return err("you are not authorized to add friends from this user"), \
            status.HTTP_403_FORBIDDEN

    if friend.from_user_id == current_identity.id:
        app.logger.info("/friend/%s can't friend self", id)
        return err("Cannot friend self"), status.HTTP_403_FORBIDDEN

    if not db.session.query(exists().where(
                            User.id == friend.to_user_id)).scalar():
        return err("User to friend not found"), status.HTTP_404_NOT_FOUND

    if action is None and request.method == 'GET':
        friend_dump = friend_schema.dump(friend)
        return friend_dump

    # accept or reject bahavior after

    if (action == "accept" or action == "reject") and request.method == 'POST':
        # change accepted value

        if action == "accept":
            if friend.accept():
                db.session.commit()

        if action == "reject":
            if friend.reject():
                db.session.commit()

        friend_dump = friend_schema.dump(friend)
        return friend_dump

    return err("Should not get here"), status.HTTP_500_INTERNAL_SERVER_ERROR


from django.shortcuts import render, redirect

# Create your views here.

def view_bag(request):
    """ A view to return the bag content page """

    return render(request, 'bag/bag.html')

def add_to_bag(request, item_id):
    """ Add product to shopping bag """

    redirect_url = request.POST.get('redirect_url')
    bag = request.session.get('bag', {})

    bag[item_id] = 1

    request.session['bag'] = bag
    print(request.session['bag'])
    return redirect(redirect_url)

def remove_from_bag(request, item_id):
    if request.method == 'POST':
        # Perform the necessary logic to remove the item from the bag using the item_id
        bag = request.session.get('bag', {})
        if item_id in bag:
            del bag[item_id]
            request.session['bag'] = bag

        return redirect('view_bag')


import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
flights = sns.load_dataset('flights')
tips = sns.load_dataset('tips')
iris = sns.load_dataset("iris")
print(iris.head())
print(iris["species"].unique())
# g = sns.PairGrid(iris)
g = sns.FacetGrid(data=tips,col="time",row="smoker")
g.map(plt.scatter,"total_bill","tip")
plt.show()


import torch
import torch.nn as nn
import torchvision
import torch.backends.cudnn as cudnn
import torch.optim
import os
import sys
import argparse
import time
import image_data_loader
import lightdehazeNet
import numpy as np
from torchvision import transforms

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

def train(args):

	ld_net = lightdehazeNet.LightDehaze_Net().cuda()
	ld_net.apply(weights_init)

	training_data = image_data_loader.hazy_data_loader(args["train_original"],
											 args["train_hazy"])
	validation_data = image_data_loader.hazy_data_loader(args["train_original"],
											 args["train_hazy"], mode="val")
	training_data_loader = torch.utils.data.DataLoader(training_data, batch_size=8, shuffle=True, num_workers=4, pin_memory=True)
	validation_data_loader = torch.utils.data.DataLoader(validation_data, batch_size=8, shuffle=True, num_workers=4, pin_memory=True)

	criterion = nn.MSELoss().cuda()
	optimizer = torch.optim.Adam(ld_net.parameters(), lr=float(args["learning_rate"]), weight_decay=0.0001)

	ld_net.train()

	num_of_epochs = int(args["epochs"])
	for epoch in range(num_of_epochs):
		for iteration, (hazefree_image, hazy_image) in enumerate(training_data_loader):

			hazefree_image = hazefree_image.cuda()
			hazy_image = hazy_image.cuda()

			dehaze_image = ld_net(hazy_image)

			loss = criterion(dehaze_image, hazefree_image)

			optimizer.zero_grad()
			loss.backward()
			torch.nn.utils.clip_grad_norm(ld_net.parameters(),0.1)
			optimizer.step()

			if ((iteration+1) % 10) == 0:
				print("Loss at iteration", iteration+1, ":", loss.item())
			if ((iteration+1) % 200) == 0:

				torch.save(ld_net.state_dict(), "trained_weights/" + "Epoch" + str(epoch) + '.pth')

		# Validation Stage
		for iter_val, (hazefree_image, hazy_image) in enumerate(validation_data_loader):

			hazefree_image = hazefree_image.cuda()
			hazy_image = hazy_image.cuda()

			dehaze_image = ld_net(hazy_image)

			torchvision.utils.save_image(torch.cat((hazy_image, dehaze_image, hazefree_image),0), "training_data_captures/" +str(iter_val+1)+".jpg")

		torch.save(ld_net.state_dict(), "trained_weights/" + "trained_LDNet.pth")

if __name__ == "__main__":

	ap = argparse.ArgumentParser()
	ap.add_argument("-th", "--train_hazy", required=True, help="path to hazy training images")
	ap.add_argument("-to", "--train_original", required=True, help="path to original training images")
	ap.add_argument("-e", "--epochs", required=True, help="number of epochs for training")
	ap.add_argument("-lr", "--learning_rate", required=True, help="learning rate for training")

	args = vars(ap.parse_args())

	train(args)


import pygame
from pygame.sprite import Sprite

class Blob(Sprite):
    def __init__(self, settings, screen):
        """Инициализирум каплю и задает начальную позицию"""
        super(Blob, self).__init__()
        self.screen = screen
        self.settings = settings

        # Загрузка изображения капли и назначения аттрибута rect
        self.image = pygame.image.load('.\\images\\blob.png')
        self.rect = self.image.get_rect()

        # Каждая новая капля появляется в левом верхнем углу.
        self.rect.x = self.rect.width
        self.rect.y = self.rect.height
        # Сохранение точной позиции капли
        self.x = float(self.rect.x)

    def blitme(self):
        """ Вывод на экран в текущем положении"""
        self.screen.blit(self.image, self.rect)

    def update(self):
        """Падение капель."""
        self.rect.y += self.settings.drop_speed

    def check_edges(self):
        """Проверка достижения края экрана"""
        screen_rect = self.screen.get_rect()
        if self.rect.y >= screen_rect.bottom:
            return True


import json
import logging
import logging.config
import yaml
import click

from utils import setup_logging
from ml_project.data import read_dataset, split_train_val_data
from ml_project.entities import (
    TrainingPipelineParams,
    read_training_pipeline_params,
)
from ml_project.features import (
    build_transformer,
    transform_features,
    extract_target
)
from ml_project.models import (
    train_model,
    predict,
    evaluate_model,
    save_model,
)

logger = logging.getLogger(__name__)

def save_metrics(metric_path: str, metrics: dict):
    """
    Save metrics.
    """
    with open(metric_path, "w") as metric_file:
        json.dump(metrics, metric_file)

def train_pipeline(training_pipeline_params: TrainingPipelineParams):
    """
    Pipeline train model.
    """
    logger.info(f"Start train pipeline with params {training_pipeline_params}\n")
    data = read_dataset(training_pipeline_params.input_data_path)
    logger.info(f"Data shape is {data.shape}")
    data_train, data_val = split_train_val_data(
        data, training_pipeline_params.split_params
    )
    logger.info(f"Data_train shape is {data_train.shape}")
    logger.info(f"Data_val shape is {data_val.shape}")

    transformer = build_transformer(training_pipeline_params.feature_params)
    transformer.fit(data_train)
    train_features = transform_features(transformer, data_train)
    logger.debug(f"train_features shape is {train_features.shape}")
    train_target = extract_target(data_train,
                              training_pipeline_params.feature_params)

    model = train_model(
        train_features, train_target, training_pipeline_params.train_params
    )
    logger.info(f"{training_pipeline_params.train_params.model_type} trained")

    test_features = transform_features(transformer, data_val)
    test_target = extract_target(data_val, training_pipeline_params.feature_params)

    logger.info(f"test_features shape is {test_features.shape}")
    predicted = predict(
        model,
        test_features
    )

    metrics = evaluate_model(
        predicted,
        test_target
    )

    logger.info(f"Metrics on test: {metrics}")
    save_metrics(metric_path=training_pipeline_params.metric_path, metrics=metrics)
    logger.info(f"Metrics save in file")

    save_model(model, training_pipeline_params.output_model_path)

    return metrics

@click.command()
@click.argument("config_path")
def train_pipeline_command(config_path: str):
    params = read_training_pipeline_params(config_path)
    setup_logging(params)
    train_pipeline(params)

if __name__ == "__main__":
    train_pipeline_command()


#!/usr/bin/python3
# -*- coding: utf-8 -*-
import requests
import re
import json
import math
import qq_song

#获取每个歌手的全部歌曲的标识，并下载
def get_songmod(singermod):
    #获取url
    base_url='https://u.y.qq.com/cgi-bin/musicu.fcg?-=getUCGI6536838773372087&g_tk=930886421&loginUin=248819350&hostUin=0&format=json&inCharset=utf8&outCharset=utf-8&notice=0&platform=yqq.json&needNewCode=0&data=%7B%22comm%22%3A%7B%22ct%22%3A24%2C%22cv%22%3A0%7D%2C%22singer%22%3A%7B%22method%22%3A%22get_singer_detail_info%22%2C%22param%22%3A%7B%22sort%22%3A5%2C%22singermid%22%3A%22{0}%22%2C%22sin%22%3A{1}%2C%22num%22%3A{2}%7D%2C%22module%22%3A%22music.web_singer_info_svr%22%7D%7D'
    #构造url,获取每个歌手歌曲的总数
    total_url=base_url.format(singermod,0,10)
    total_response=requests.get(total_url).text
    total=json.loads(total_response)['singer']['data'].get('total_song')
    #songmods_list=[]
    #获取歌曲总共页数
    pager_num=math.ceil(total/10)
    for i in range(pager_num):
        #根据页码数和歌手标识构造多个url
        songs_url=base_url.format(singermod,i*10,10)
        songs_response=requests.get(songs_url).text
        #获取歌曲的标识
        songs_data=json.loads(songs_response)['singer']['data']['songlist']
        for song in songs_data:
            qq_song.load_music(song.get('mid'))
            #songmods_list.append(song.get('mid'))

    #return songmods_list
#
# if __name__ == '__main__':
#     singermod='001cnFzG2sq87c'
#     songmods=get_songmod(singermod)
#     print(songmods)


# %%
import os
import pymongo

# %%
myclient = pymongo.MongoClient("mongodb://localhost:27017/")

mydb = myclient["paper_reading_database"]

collist = mydb.list_collection_names()
mkdown_files_col = mydb["mkdown_files"]
image_files_col = mydb['image_files']
#if "mkdown_files" in collist:
#    mkdown_files_col.drop()
#if "image_files" in collist:
#    image_files_col.drop()

# %%
repo_doc_path = "/home/yxliu/Documents/papers_reading_sharing.github.io/docs/"

mkdown_files = []
image_files = []
for root, dirs, files in os.walk(repo_doc_path):
    for file in files:
        if file.endswith('.md') and not file.startswith('index.md'):
            mkdown_files.append(
                dict(root=root, file=file, all_path=os.path.join(root, file))
            )
            continue
        lower_name = file.lower()
        if lower_name.endswith('.png') or \
            lower_name.endswith('.jpg') or \
            lower_name.endswith('.jpeg') or \
            lower_name.endswith('.svg') or \
            lower_name.endswith('.gif'):
            image_files.append(
                dict(root=root, file=file, all_path=os.path.join(root, file), occurance=[]),
            )
            continue
for mkdown_file in mkdown_files:
    if mkdown_files_col.find_one({'file': mkdown_file['file']}):
        continue
    mkdown_files_col.insert_one(mkdown_file)
    print(mkdown_file)

for image_file in image_files:
    if image_files_col.find_one({'file': image_file['file']}):
        continue
    image_files_col.insert_one(image_file)
    print(image_file)
print(len(mkdown_files))
print(len(image_files))

# %%
mkdown_file

# %%
image_files_col.find_one({'all_path': image_file['all_path'], 'root': image_file['root']})

# %%
image_file

# %%
import re
from PIL import Image

for mkdown_file in mkdown_files_col.find():
    all_path = mkdown_file.get('all_path', '')
    if not (len(all_path) > 0):
        continue
    with open(mkdown_file['all_path'], 'r') as f:
        lines = f.readlines()
        for i, line in enumerate(lines):
            output = re.finditer("!\[.*\]\(.*\)", line)
            for match in output:
                sub_path = re.findall("\(.*\)", match.group())[0][1:-1]
                if sub_path.startswith('http'):
                    continue
                image_path = os.path.join(mkdown_file['root'], sub_path)
                image_path = os.path.abspath(image_path)
                query = {"all_path": image_path}
                for query_found in image_files_col.find(query):
                    occurance = query_found['occurance']
                    occurance.append(
                        dict(
                            file_place=mkdown_file['all_path'],
                            line_num=i, span=match.span(),
                            matched_sentence=match.group(),
                            sub_path=sub_path
                        )
                    )
                image_files_col.update_one(query, {'$set': {'occurance': occurance}})

# %%
image_files_col.find_one()

# %%
myquery = { "all_path": image_path}

# %%
mydoc = image_files_col.find(myquery)
mydoc[0]

# %%
sub_path

# %%
match.group()

# %%
# mkdown_file['root'].split('/')[-1]

# %%
for mkdown_file in mkdown_files_col.find():
    if 'file' not in mkdown_file:
        print(mkdown_file['title'], "not in paper reading database")
        continue
    query = {'file': mkdown_file['file']}
    file_data = {}
    all_path = mkdown_file.get('all_path', '')
    if not (len(all_path) > 0):
        continue
    with open(mkdown_file['all_path'], 'r') as f:
        lines = f.readlines()
        tag = [mkdown_file['root'].split('/')[-1]]
        file_data['tag'] = tag
        for i, line in enumerate(lines):
            if line.startswith('time: '):
                record_time = int(line.split(':')[1].strip())
                file_data['record_time'] = record_time
            if line.startswith('pdf_source: '):
                pdf_place = line[11:].strip()
                file_data['pdf'] = pdf_place
            if line.startswith('code_source: '):
                code_place = line[12:].strip()
                file_data['code'] = code_place
            if line.startswith("# "):
                if 'title' not in file_data:
                    title = line[2:].strip()
                    file_data['title'] = title
    with open(mkdown_file['all_path'], 'r') as f:
        file_data['content'] = f.read().lower()
    cls_1 = mkdown_file['root'].split('/')[-1]
    cls_2 = mkdown_file['root'].split('/')[-2]
    file_name = mkdown_file['file'][:-3].replace(' ', '%20').replace(':', '%3A')
    base_url = 'https://owen-liuyuxuan.github.io/papers_reading_sharing.github.io'
    if cls_2 == 'docs':
        url = f"{base_url}/{cls_1}/{file_name}"
    else:
        url = f"{base_url}/{cls_2}/{cls_1}/{file_name}"
    file_data['url'] = url
    for key in file_data:
        mkdown_files_col.update_one(query, {'$set': {key: file_data[key]}})

# %%


import scrapy
import requests,re
import json,pprint,codecs
from datetime import datetime
import time,os
header = {'User-Agent':'Mozilla/5.0'}

try:
    os.mkdir('./result')
except:
    pass

##For KUL-PVG
# origin = r"KUL"
# destination = r"PVG"
# airline = r"D7"
# fnumber = r"330"
# flightNumber = r"{}-{}".format(airline,fnumber)
# # month = '03'
# # day = ['26','28','30']
# month = '04'
# day = ['02','04','06','09','11','13','16','18','20','23','25','27','30']

####################################

#For KUL-TPE
origin = r"KUL"
destination = r"TPE"
month = '04'
airline = r'D7'
fnumber = r'378'
flightNumber = r"{}-{}".format(airline,fnumber)
day = ['01','02','03','04','05','06','07','08','09','10']
for i in range(11,32):
    day.append(str(i))
####################################

for d in day:

    baseurl = 'https://www.airasia.com/flights/search/?origin={}&destination={}&departDate={}/{}/2023&tripType=O&adult=1&child=0&infant=0&locale=en-gb&currency=MYR&airlineProfile=all&type=paired&cabinClass=economy&upsellWidget=true&upsellPremiumFlatbedWidget=true&isOC=false&isDC=false&uce=false'.format(origin,destination,d,month)
    r = requests.get(baseurl, headers= header, timeout=10)
    r.raise_for_status()
    file = codecs.open("./webPageText_airasia_.txt", 'w',encoding= "UTF-8")
    file.write(r.text)
    # print(r'"tripId":(.*?)"sharedFlight":"'+origin+'-'+ destination + '-' + flightNumber +'",')
    results = re.findall(r'"tripId":(.*?)"sharedFlight":"(.*?)"',r.text)
    for res in results:
        if str(re.findall(r'"marketingFlightNumber":(.*?),',res[0])[0]) == fnumber and res[1] == "{}-{}-{}".format(origin,destination,flightNumber):
            result = res[0]

    fare = re.findall(r'"fare":{"adults":(.*?),',result)[0]
    seatAvailability = re.findall(r'"seatAvailability":{"seats":(.*?)},',result)[0]
    flightName = "{}-{}-{}".format(origin,destination,flightNumber)
    departureDate = re.findall(r'"departureTime":"(.*?)T.*?"',result)[0]

    print(flightName,departureDate,datetime.now(),fare, seatAvailability)
    try:
        file = codecs.open("./result/{}_{}.txt".format(flightName,departureDate), 'a',encoding= "UTF-8")

        file.write("{},{},{}\n".format(datetime.now(),fare, seatAvailability))
    except:
        pass

    time.sleep(1)


import shutil

from scipy import misc
from pathlib import Path

from keras.utils import HDF5Matrix

def convert_from_hdf5():
    cur_dir = Path(__file__).parent.absolute()

    # examples/gtsrb/data/test_dataset_3_32_32.h5
    input_path = cur_dir.parent.joinpath(
        'examples', 'gtsrb', 'data', 'test_dataset_3_32_32.h5')

    # output of pngs and textfiles
    output_folder = cur_dir.joinpath('html', 'images')
    if output_folder.exists():
        shutil.rmtree(str(output_folder))

    output_folder.mkdir(exist_ok=True)

    print('Input: %s' % input_path)
    print('Output: %s' % output_folder)

    convert_images_from_hdf5(input_path, output_folder)
    print('Done')

def convert_images_from_hdf5(input_path, output_folder):
    images = HDF5Matrix(input_path, 'images')

    print('Converting images')
    for i, image in enumerate(images):
        img = image.transpose((1, 2, 0))
        misc.imsave(output_folder.joinpath('%d.jpg' % i), img)

if __name__ == '__main__':
    convert_from_hdf5()


from typing import Optional

from fastapi import HTTPException
from starlette.status import HTTP_401_UNAUTHORIZED, HTTP_400_BAD_REQUEST, HTTP_403_FORBIDDEN

class UnauthorizedException(HTTPException):
    def __init__(self, message: str = "Invalid credentials") -> None:
        super().__init__(
            status_code=HTTP_401_UNAUTHORIZED,
            detail=message,
        )

class ForbiddenException(HTTPException):
    def __init__(self, message: str) -> None:
        super().__init__(
            status_code=HTTP_403_FORBIDDEN,
            detail=message,
        )

class BadRequestException(HTTPException):
    def __init__(self, message: str) -> None:
        super().__init__(
            status_code=HTTP_400_BAD_REQUEST,
            detail=message
        )

class InvalidSettingsException(Exception):
    def __init__(self, message: Optional[str] = None) -> None:
        self.message = message


from tkinter import Y
import pandas as pd
import matplotlib.pyplot as plt
import math
import numpy as np

data = pd.read_excel('Trends in Prevalence Rates of Food Insecurity by Year.xlsx')
data

plt.figure(figsize=(7,7))
year = range(1995, 2021)

#Food Insecure People Plot
plt.plot(data['Food insecurity '], label = r'Food Insecure', c = '#f8c928')

#Very low food security plot
plt.plot(data['Very low food security'], label = r'Very Low Food Security', c = '#c35b7e')

year = range(1995,2020)

ind = np.arange(len(year))

#make the plot
#label tick marks
plt.xticks(ind, year, rotation = 90)
plt.locator_params(axis="x", nbins=8)

#mark important year line dates from 1995-2020
plt.axvline(x = 14, color = 'black', ls = ':', alpha = 0.75)
plt.annotate('2009 Great Recession', rotation = 90, xy = (14.2, 0.5), fontsize = 7)
plt.axvline(x = 6, color = 'black', ls = ':',alpha = 0.75)
plt.annotate('9/11 Terroists Attack', rotation = 90, xy = (6.2, 0.5), fontsize = 7)
plt.axvline(x = 24, color = 'black', ls = ':', alpha = 0.75)
plt.annotate('COVID-19 Pandemic', rotation = 90, xy = (24.2, 0.5), fontsize = 7)

#mark important percentages
plt.axhline(y = 12, color = 'red', ls = '--' ,alpha = 0.5 )
plt.annotate('Severe Food Insecurity', xy = (0, 12.2), fontsize = 7, color = 'red')
plt.axhline(y = 5, color = 'red',ls = '--', alpha = 0.5)
plt.annotate('Severe Very Food Insecure', xy = (0, 5.2), fontsize = 7, color = 'red')

#place legend
plt.legend(loc='upper right')

#plot x,y axis and title
plt.xlabel("Year")
plt.ylabel("Percent %")
plt.title('Trends in Prevalence Rates of Food Insecurity by Year')

#adjust the ylimit
plt.ylim(0, 18)
plt.show()

Trends in Prevalence Rates of Food Insecurity by Year.py


from typing import List

from dsu.dsu import DSU

class UnionFind(DSU):
    def __init__(self, max_size: int):
        self.max_rank = 0
        super().__init__(max_size)

    def add(self, id: int):
        self.insert(id)
        self.max_rank = max(self.max_rank, self.ranks[id])

    def union(self, p: int, q: int):
        p_root = self.find_root(p)
        q_root = self.find_root(q)
        if p_root == q_root:
            return

        self.join(p_root, q_root)
        new_rank = max(self.ranks[p_root], self.ranks[q_root])
        self.max_rank = max(self.max_rank, new_rank)

    def get_max_rank(self) -> int:
        return self.max_rank

class Solution:
    def longestConsecutive(self, nums: List[int]) -> int:
        size = len(nums)
        if size < 2:
            return size

        union_find = UnionFind(size)
        nums_dict = dict()
        for index, num in enumerate(nums):
            repeat = nums_dict.get(num, None)
            if repeat is not None:
                continue

            nums_dict[num] = index
            union_find.add(index)

            previous_index = nums_dict.get(num - 1, None)
            if previous_index is not None:
                union_find.union(previous_index, index)

            next_index = nums_dict.get(num + 1, None)
            if next_index is not None:
                union_find.union(next_index, index)

        return union_find.get_max_rank()


#!/usr/bin/env python3

'''
connect to: 84:28:D7:8F:54:70
notificationhandler installed
services:
Service <uuid=Generic Attribute handleStart=1 handleEnd=4>
Service <uuid=Generic Access handleStart=5 handleEnd=13>
Service <uuid=6e400001-b5a3-f393-e0a9-e50e24dcca9e handleStart=14 handleEnd=65535>

characteristics:
Characteristic <Service Changed> uuid: 00002a05-0000-1000-8000-00805f9b34fb handle: 3 properties: INDICATE
indicate enabled: 4
Characteristic <Device Name> uuid: 00002a00-0000-1000-8000-00805f9b34fb handle: 7 properties: READ
length: 7 value: b'libatt\x00'
Characteristic <Appearance> uuid: 00002a01-0000-1000-8000-00805f9b34fb handle: 9 properties: READ
length: 2 value: b'\x00\x00'
Characteristic <Peripheral Preferred Connection Parameters> uuid: 00002a04-0000-1000-8000-00805f9b34fb handle: 11 properties: READ
length: 8 value: b'\x00\x00\x00\x00\x00\x00\x00\x00'
Characteristic <Central Address Resolution> uuid: 00002aa6-0000-1000-8000-00805f9b34fb handle: 13 properties: READ
length: 1 value: b'\x00'
Characteristic <0002> uuid: 00000002-0000-1000-8000-00805f9b34fb handle: 16 properties: READ
length: 2 value: b'\x01\x02'
Characteristic <6e400002-b5a3-f393-e0a9-e50e24dcca9e> uuid: 6e400002-b5a3-f393-e0a9-e50e24dcca9e handle: 19 properties: WRITE NO RESPONSE WRITE
Characteristic <6e400003-b5a3-f393-e0a9-e50e24dcca9e> uuid: 6e400003-b5a3-f393-e0a9-e50e24dcca9e handle: 21 properties: NOTIFY
send notify: desc: [<bluepy.btle.Descriptor object at 0xb655ae30>] handle[0]: 22
Characteristic <0001> uuid: 00000001-0000-1000-8000-00805f9b34fb handle: 24 properties: INDICATE
indicate enabled: 25
'''
import sys
sys.path.append('..')
import os
os.environ['PATH'] = 'C:/msys64/mingw64/bin' + os.pathsep + os.environ['PATH']
import time
import atorlib
import threading
import logging

# read data from notification
class SupervoltBatteryBleak(atorlib.BleBleakBase):

    cellV = [None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]
    totalV = None
    soc = None
    workingState = None
    alarm = None
    chargingA = None;
    dischargingA = None;
    loadA = None
    tempC = [None, None, None, None]
    completeAh = None
    remainingAh = None
    designedAh = None

    def __init__(self,
                 name=None,
                 mac=None,
                 data=None,
                 verbose=False,
                 updatetimeS=1,
                 timeout=None,
                 callbackAfterData=None,
                 disconnectAfterData=False):
        super().__init__(mac=mac, name=name, mtuSize=246,
                         timeout=timeout,
                         data=data, verbose=verbose, updatetimeS=updatetimeS, callbackAfterData=callbackAfterData,
                         disconnectAfterData=disconnectAfterData);
        if data:
            self.parseData(data)

    async def enableNotifications(self):
        if False:
            data = b"\x02\x00"
            ret = await self.peripheral.writeCharacteristic(0x0004, data)
            if self.verbose:
                logging.info("0x0004: " + str(ret) + " " + str(data))
        data = b"\x01\x00"
        handle = 0x0015
        handle = "6e400003-b5a3-f393-e0a9-e50e24dcca9e"
        await self.device.start_notify(char_specifier=handle, callback=self.handleNotification)
        # ret = await self.peripheral.writeCharacteristic(0x0016, data)
        # if self.verbose:
        #    logging.info("0x0016: " + str(ret) + " " + str(data))
        if False:
            data = b"\x02\x00"
            ret = await self.peripheral.writeCharacteristic(0x0019, data)
            if self.verbose:
                logging.info("0x0019: " + str(ret) + " " + str(data))
        if self.verbose:
            logging.info("notifications enabled")

    def setData(self, data):
        if data:
            self.parseData(data)

    # send request to battery for Realtime-Data
    async def requestRealtimeData(self):
        data = bytes(":000250000E03~", "ascii")
        handle = 0x0013
        handle = 19
        handle = "6e400002-b5a3-f393-e0a9-e50e24dcca9e"
        # 0x0013 -> 19 -> 6e400002-b5a3-f393-e0a9-e50e24dcca9e
        ret = await self.device.write_gatt_char(char_specifier=handle, data=data)
        # ret = self.device.writeCharacteristic(0x0013, data)
        if self.verbose:
            logging.debug(":000250000E03~: " + str(ret) + " " + str(data))

    # send request to battery for Capacity-Data
    async def requestCapacity(self):
        data = bytes(":001031000E05~", "ascii")
        handle = 0x0013
        handle = 19
        handle = "6e400002-b5a3-f393-e0a9-e50e24dcca9e"
        # 0x0013 -> 19 -> 6e400002-b5a3-f393-e0a9-e50e24dcca9e
        ret = await self.device.write_gatt_char(char_specifier=handle, data=data)
        if self.verbose:
            logging.debug(":001031000E05~: " + str(ret) + " " + str(data))

    async def requestData(self):
        dis = self.disconnectAfterData
        try:
            await self.enableNotifications()
            # disable, cause we will need two answers
            self.disconnectAfterData = False
            self.notificationReceived = False
            await self.requestRealtimeData()
            await self.waitForNotification(10.0)

            self.notificationReceived = False
            self.disconnectAfterData = dis
            await self.requestCapacity()
            await self.waitForNotification(10.0)
        except:
            logging.error(sys.exc_info(), exc_info=True)
            await self.disconnect()
        self.disconnectAfterData = dis

    # try to read values from data
    def parseData(self, data):
        if self.verbose:
            logging.debug("parseData: {}".format(len(data)))
        try:
            if data:
                if len(data) == 128:
                    if self.verbose:
                        logging.info("parse Realtimedata: {}".format(type(data)))
                    if type(data) is bytearray:
                        data = bytes(data)
                    if type(data) is bytes:
                        # print("bytes")

                        start = 1
                        end = start + 2
                        self.address = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("address: " + str(self.address))

                        start = end
                        end = start + 2
                        self.command = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("command: " + str(self.command))

                        start = end
                        end = start + 2
                        self.version = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("version: " + str(self.version))

                        start = end
                        end = start + 4
                        self.length = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("length: " + str(self.length))

                        start = end
                        end = start + 14
                        bdate = data[start: end]
                        if self.verbose:
                            logging.debug("date: " + str(bdate))

                        start = end
                        end = start + 16 * 4
                        bvoltarray = data[start: end]
                        # print("voltarray: " + str(bvoltarray))
                        self.totalV = 0
                        for i in range(0, 11):
                            bvolt = data[(start + i * 4): (start + i * 4 + 4)]
                            self.cellV[i] = int(bvolt.decode(), 16) / 1000.0
                            self.totalV += self.cellV[i]
                            if self.verbose:
                                logging.debug("volt" + str(i) + ": " + str(bvolt) + " / " + str(self.cellV[i]) + "V")

                        if self.verbose:
                            logging.debug("totalVolt: " + str(self.totalV))

                        start = end
                        end = start + 4
                        bcharging = data[start: end]
                        self.chargingA = int(bcharging.decode(), 16) / 100.0
                        if self.verbose:
                            logging.debug("charching: " + str(bcharging) + " / " + str(self.chargingA) + "A")
                        if self.chargingA > 500:
                            # problem with supervolt
                            logging.info("charging too big: {}".format(self.chargingA))
                            self.chargingA = 0.0

                        start = end
                        end = start + 4
                        bdischarging = data[start: end]
                        self.dischargingA = int(bdischarging.decode(), 16) / 100.0
                        if self.verbose:
                            logging.debug("discharching: " + str(bdischarging) + " / " + str(self.dischargingA) + "A")
                        if self.dischargingA > 500:
                            # problem with supervolt
                            logging.info("discharging too big: {}".format(self.dischargingA))
                            self.dischargingA = 0.0

                        self.loadA = -self.chargingA + self.dischargingA

                        for i in range(0, 4):
                            start = end
                            end = start + 2
                            btemp = data[start: end]
                            self.tempC[i] = int(btemp.decode(), 16) - 40
                            if self.verbose:
                                logging.debug("temp" + str(i) + ": " + str(btemp) + " / " + str(self.tempC[i]) + "°C")

                        start = end
                        end = start + 4
                        self.workingState = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("workingstate: " + str(self.workingState) + " / " + str(data[start: end])
                              +" / " + self.getWorkingStateTextShort() + " / " + self.getWorkingStateText())

                        start = end
                        end = start + 2
                        self.alarm = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("alarm: " + str(self.alarm))

                        start = end
                        end = start + 4
                        self.balanceState = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("balanceState: " + str(self.balanceState))

                        start = end
                        end = start + 4
                        self.dischargeNumber = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("dischargeNumber: " + str(self.dischargeNumber))

                        start = end
                        end = start + 4
                        self.chargeNumber = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("chargeNumber: " + str(self.chargeNumber))

                        # State of Charge (%)
                        start = end
                        end = start + 2
                        self.soc = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("soc: " + str(self.soc))
                            logging.info("end of parse realtimedata")
                    else:
                        logging.warning("no bytes")
                elif len(data) == 30:
                    if self.verbose:
                        logging.debug("capacity")
                    if type(data) is bytearray:
                        data = bytes(data)
                    if type(data) is bytes:
                        start = 1
                        end = start + 2
                        self.address = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("address: " + str(self.address))

                        start = end
                        end = start + 2
                        self.command = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("command: " + str(self.command))

                        start = end
                        end = start + 2
                        self.version = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("version: " + str(self.version))

                        start = end
                        end = start + 4
                        self.length = int(data[start: end].decode(), 16)
                        if self.verbose:
                            logging.debug("length: " + str(self.length))

                        start = end
                        end = start + 4
                        breseved = data[start: end]
                        if self.verbose:
                            logging.debug("reseved: " + str(breseved))

                        start = end
                        end = start + 4
                        self.remainingAh = int(data[start: end].decode(), 16) / 10.0
                        if self.verbose:
                            logging.debug("remainingAh: " + str(self.remainingAh) + " / " + str(data[start: end]))

                        start = end
                        end = start + 4
                        self.completeAh = int(data[start: end].decode(), 16) / 10.0
                        if self.verbose:
                            logging.debug("completeAh: " + str(self.completeAh))

                        start = end
                        end = start + 4
                        self.designedAh = int(data[start: end].decode(), 16) / 10.0
                        if self.verbose:
                            logging.debug("designedAh: " + str(self.designedAh))
                            logging.info("end of parse capacity")

                else:
                    logging.warning("wrong length: " + str(len(data)))
            else:
                logging.debug("no data")
        except:
            logging.error(sys.exc_info(), exc_info=True)

    def resetValues(self):
        try:
            logging.info("reset")
            self.alarm = None
            self.balanceState = None
            for i in range(0, 11):
                self.cellV[i] = None
            self.chargeNumber = None
            self.chargingA = None
            self.completeAh = None
            self.designedAh = None
            self.dischargeNumber = None
            self.dischargingA = None
            self.loadA = None
            self.remainingAh = None
            self.soc = None
            for i in range(0, 4):
                self.tempC[i] = None
            self.totalV = None
            self.version = None
            self.workingState = None
        except:
            logging.error(sys.exc_info(), exc_info=True)

    def getWorkingStateTextShort(self):
        if self.workingState is None:
            return "nicht erreichbar"
        if self.workingState & 0xF003 >= 0xF000:
            return "Normal"
        if self.workingState & 0x000C > 0x0000:
            return "Schutzschaltung"
        if self.workingState & 0x0020 > 0:
            return "Kurzschluss"
        if self.workingState & 0x0500 > 0:
            return "Überhitzt"
        if self.workingState & 0x0A00 > 0:
            return "Unterkühlt"
        return "Unbekannt"

    def getWorkingStateText(self):
        text = ""
        if self.workingState is None:
            return "Unbekannt"
        if self.workingState & 0x0001 > 0:
            text = self.appendState(text, "Laden")
        if self.workingState & 0x0002 > 0:
            text = self.appendState(text , "Entladen")
        if self.workingState & 0x0004 > 0:
            text = self.appendState(text , "Überladungsschutz")
        if self.workingState & 0x0008 > 0:
            text = self.appendState(text , "Entladeschutz")
        if self.workingState & 0x0010 > 0:
            text = self.appendState(text , "Überladen")
        if self.workingState & 0x0020 > 0:
            text = self.appendState(text , "Kurzschluss")
        if self.workingState & 0x0040 > 0:
            text = self.appendState(text , "Entladeschutz 1")
        if self.workingState & 0x0080 > 0:
            text = self.appendState(text , "Entladeschutz 2")
        if self.workingState & 0x0100 > 0:
            text = self.appendState(text , "Überhitzt (Laden)")
        if self.workingState & 0x0200 > 0:
            text = self.appendState(text , "Unterkühlt (Laden)")
        if self.workingState & 0x0400 > 0:
            text = self.appendState(text , "Überhitzt (Entladen)")
        if self.workingState & 0x0800 > 0:
            text = self.appendState(text , "Unterkühlt (Entladen)")
        if self.workingState & 0x1000 > 0:
            text = self.appendState(text , "DFET an")
        if self.workingState & 0x2000 > 0:
            text = self.appendState(text , "CFET an")
        if self.workingState & 0x4000 > 0:
            text = self.appendState(text , "DFET Schalter an")
        if self.workingState & 0x8000 > 0:
            text = self.appendState(text , "CFET Schalter an")

        return text

    def appendState(self, text, append):
        if text is None  or len(text) == 0:
            return append
        return text + " | " + append

    def toJSON(self, prefix="battery"):
        self.checkAgeOfValues()
        json = ""
        prefixText = ""
        if prefix is not None:
            prefixText = prefix + "_"
        try:
            if self.tempC[0] is not None:
                json += "\"" + prefixText + "temperature\": {}".format(self.tempC[0]) + ",\n"
            if self.totalV is not None:
                json += "\"" + prefixText + "voltage\": {}".format(self.totalV) + ",\n"
            if self.cellV[0] is not None:
                json += "\"" + prefixText + "voltage_cell0\": {}".format(self.cellV[0]) + ",\n"
            if self.cellV[1] is not None:
                json += "\"" + prefixText + "voltage_cell1\": {}".format(self.cellV[1]) + ",\n"
            if self.cellV[2] is not None:
                json += "\"" + prefixText + "voltage_cell2\": {}".format(self.cellV[2]) + ",\n"
            if self.cellV[3] is not None:
                json += "\"" + prefixText + "voltage_cell3\": {}".format(self.cellV[3]) + ",\n"
            if self.soc is not None:
                json += "\"" + prefixText + "soc\": {}".format(self.soc) + ",\n"
            if self.chargingA is not None:
                json += "\"" + prefixText + "chargingA\": {}".format(self.chargingA) + ",\n"
            if self.dischargingA is not None:
                json += "\"" + prefixText + "dischargingA\": {}".format(self.dischargingA) + ",\n"
            if self.loadA is not None:
                json += "\"" + prefixText + "loadA\": {}".format(self.loadA) + ",\n"
            if self.alarm is not None:
                json += "\"" + prefixText + "alarm\": {}".format(self.alarm) + ",\n"
            if self.workingState is not None:
                json += "\"" + prefixText + "workingState\": {}".format(self.workingState) + ",\n"
                withoutUmlaute = self.getWorkingStateText().replace("Ü", "Ue").replace("ü", "ue")
                json += "\"" + prefixText + "workingStateText\": \"{}\"".format(withoutUmlaute) + ",\n"
                withoutUmlaute = self.getWorkingStateTextShort().replace("Ü", "Ue").replace("ü", "ue")
                json += "\"" + prefixText + "workingStateTextShort\": \"{}\"".format(withoutUmlaute) + ",\n"
            if self.completeAh is not None:
                json += "\"" + prefixText + "completeAh\": {}".format(self.completeAh) + ",\n"
            if self.remainingAh is not None:
                json += "\"" + prefixText + "remainingAh\": {}".format(self.remainingAh) + ",\n"
            if self.designedAh is not None:
                json += "\"" + prefixText + "designedAh\": {}".format(self.designedAh) + ",\n"

        except:
            logging.warning(sys.exc_info(), exc_info=True)
        return json

def main():
    try:
        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s')

        mac = "84:28:D7:8F:XX:XX"
        if len(sys.argv) > 1 and sys.argv[1] is not None:
            mac = sys.argv[1]
        else:
            logging.warning("usage: supervoltbatterybleak.py <BLE-Address>")
            return
        logging.info("connect to " + mac)
        battery = SupervoltBatteryBleak(mac=mac, name="SuperVoltTest", verbose=True, updatetimeS=10, disconnectAfterData=True)
        battery.startReading()
        while(True):
            time.sleep(10000)
    except:
        logging.error(sys.exc_info(), exc_info=True)

if __name__ == '__main__':
    main()