import os
import subprocess
import shutil
import requests
import json
from config import auth_headers, resource_type_mapping
from config import auth_json_body, query_json_body
from user import idEntProject, nameEntProject, ak, sk
from utils import Loader, signalHandler, Banner

Banner.show_banner()
signalHandler.register_signal_handler()
loader = Loader(3, 0.5)
jsondumps_folder = 'jsondumps'
class HuaweiCloudAPI:
    def __init__(self, auth_url, query_url, auth_headers, auth_json_body):
        self.auth_url = auth_url
        self.query_url = query_url
        self.auth_headers = auth_headers
        self.auth_json_body = auth_json_body
        self.x_subject_token = None

    def authenticate(self):
        auth_response = requests.post(self.auth_url, json=self.auth_json_body, headers=self.auth_headers)
        if auth_response.status_code == 201:
            self.x_subject_token = auth_response.headers.get('X-Subject-Token')
            return True
        return False

    def query_resources(self, query_json_body):
        if self.x_subject_token:
            query_headers = {
                "Content-Type": "application/json",
                "X-Auth-Token": self.x_subject_token
            }

            response = requests.post(self.query_url, json=query_json_body, headers=query_headers)
            return response

    def saveJson(self, json_data, file_path):
        file_path = os.path.join(jsondumps_folder, file_path)
        with open(file_path, "w") as outfile:
            json.dump(json_data, outfile, indent=4)
        loader.stop()
        print(f"\033[92mResources saved to {file_path}\033[0m")

class TerraformConfigGenerator:
    def __init__(self, json_data, resource_type_mapping):
        self.json_data = json_data
        self.resource_type_mapping = resource_type_mapping

    def generate_config(self, output_folder):
        if not os.path.exists(output_folder):
            os.makedirs(output_folder)

        dict = {}
        dictIdx = {}

        for resource in self.json_data["resources"]:
            resource_id = resource["resource_id"]
            resource_type = resource["resource_type"]
            project_name = resource["project_name"]

            if resource_type in self.resource_type_mapping:
                terraform_term = self.resource_type_mapping[resource_type]

                dict.setdefault(project_name, {})
                project_dict = dict[project_name]

                dict[project_name].setdefault(resource_type, [])
                project_resources = project_dict[resource_type]

                dict[project_name][resource_type].append({
                    "id": resource_id,
                    "term": terraform_term
                })

        for project_name, project_dict in dict.items():
            project_folder = os.path.join(output_folder, project_name)
            if not os.path.exists(project_folder):
                os.makedirs(project_folder)

            file_out = os.path.join(project_folder, "import.tf")

            static_terraform_config = (
                'terraform {\n'
                'required_providers {\n'
                'huaweicloud = {\n'
                'source  = "huaweicloud/huaweicloud"\n'
                'version = ">= 1.6.0"\n'
                '}\n'
                '}\n'
                '}\n\n'
                '# Configure the HuaweiCloud Provider\n'
                'provider "huaweicloud" {\n'
                'region     = "' + project_name + '"\n'
                f'access_key = "{ak}"\n'
                f'secret_key = "{sk}"\n'
                '}\n'
            )

            dynamic_terraform_config = ""

            for resource_type, resources in project_dict.items():
                for resource_info in resources:
                    resource_id = resource_info["id"]
                    terraform_term = resource_info["term"]

                    dictIdx.setdefault(resource_type, 0)
                    dictIdx[resource_type] += 1
                    idx = dictIdx[resource_type]

                    module_config = (
                        'import {\n'
                        'id = "' + resource_id + '"\n'
                        'to = ' + terraform_term + '.imported_' + resource_type + str(idx) + '\n'
                        '}\n'
                    )
                    dynamic_terraform_config += module_config

            terraform_config = static_terraform_config + dynamic_terraform_config

            with open(file_out, "w") as outfile:
                outfile.write(terraform_config)

            loader.stop()
            print(f"Terraform configuration for region '{project_name}' saved to {file_out}")

def main():

    loader.start(f"\nGetting resources from project \033[95m'{nameEntProject}'\033[0m")

    auth_url = "https://eps.myhuaweicloud.com/v3/auth/tokens"
    query_url = f"https://eps.myhuaweicloud.com/v1.0/enterprise-projects/{idEntProject}/resources/filter"

    huawei_cloud_api = HuaweiCloudAPI(auth_url, query_url, auth_headers, auth_json_body)
    if huawei_cloud_api.authenticate():
        response = huawei_cloud_api.query_resources(query_json_body)
        if response.status_code == 200:
            json_data = response.json()
            output_folder = "tf_configs"

            if os.path.exists(output_folder):
                shutil.rmtree(output_folder)

            os.makedirs(output_folder)

            if not os.path.exists(jsondumps_folder):
                os.makedirs(jsondumps_folder)

            huawei_cloud_api.saveJson(json_data, "output.json")

            loader.start("\n\033[93mGenerating Terraform configurations\033[0m")
            terraform_config_generator = TerraformConfigGenerator(json_data, resource_type_mapping)
            terraform_config_generator.generate_config(output_folder)
            loader.stop()

            loader.start("\n\033[93mGetting Subnet details\033[0m")
            loader.stop()

            subprocess.run(["python3", "subnet.py", json.dumps(json_data)])

            loader.start("\n\033[93mGetting ELB details\033[0m")
            loader.stop()

            subprocess.run(["python3", "elb.py", json.dumps(json_data)])

        else:
            loader.stop()
            print(f"API call failed with status code: {response.status_code}")
            print(response.text)
    else:
        loader.stop()
        print("Authentication failed.")

    user_input = input("\nDo you want to continue to Phase 2 (y/n)? ").lower()

    if user_input == 'y':
        print("\nContinuing to Phase 2...")
        loader.stop()
        subprocess.run(["python3", "tfcom.py"])
    elif user_input == 'n':
        print("Program terminated.")
    else:
        print("Invalid input. Program terminated.")

if __name__ == "__main__":
    main()


#Coding: UTF-8

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from transformers import BertTokenizer as BertTokenizer
import os
import torch
import numpy as np
import random
import nlp_data_utils as data_utils
from nlp_data_utils import ABSATokenizer
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
import math

# domains = ['Video_Games','Toys_and_Games','Tools_and_Home_Improvement','Sports_and_Outdoors','Pet_Supplies',
#            'Patio_Lawn_and_Garden','Office_Products','Musical_Instruments','Movies_and_TV',
#            'Kindle_Store','Home_and_Kitchen','Health_and_Personal_Care','Grocery_and_Gourmet_Food','Electronics',
#            'Digital_Music','Clothing_Shoes_and_Jewelry','Cell_Phones_and_Accessories','CDs_and_Vinyl',
#            'Books','Beauty','Baby','Automotive','Apps_for_Android','Amazon_Instant_Video']

domains = ['Video_Games','Toys_and_Games','Tools_and_Home_Improvement','Sports_and_Outdoors','Pet_Supplies',
           'Patio_Lawn_and_Garden','Office_Products','Musical_Instruments','Movies_and_TV',
           'Kindle_Store']

datasets = [ './dat/dsc/'+domain for domain in domains]

def get(logger=None,args=None):
    data={}
    taskcla=[]

    # Others
    f_name = 'dsc_random_10'

    with open(f_name,'r') as f_random_seq:
        random_sep = f_random_seq.readlines()[args.idrandom].split()

    print('random_sep: ',random_sep)
    print('domains: ',domains)

    for t in range(args.ntasks):
        dataset = datasets[domains.index(random_sep[t])]

        data[t]={}
        data[t]['name']=dataset
        data[t]['ncla']=2

        print('dataset: ',dataset)
        logger.info(dataset)

        processor = data_utils.DscProcessor()
        label_list = processor.get_labels()
        tokenizer = ABSATokenizer.from_pretrained(args.bert_model)
        train_examples = processor.get_train_examples(dataset)

        if args.train_data_size > 0: #TODO: for replicated results, better do outside (in prep_dsc.py), so that can save as a file
            random.Random(args.data_seed).shuffle(train_examples) #more robust
            train_examples = train_examples[:args.train_data_size]

        #TODO: to be conssitent, sometimes we want to convert it back and save
        # with open('./dat/dsc/'+random_sep[t]+'/train_'+str(args.train_data_size),'w') as fw:
        #     logger.info('train_examples: '+str(len(train_examples)))
        #     for train_example in train_examples:
        #         fw.writelines(train_example.text_a + '\t' + train_example.label + '\n')
        #     logger.info('train_example.text_a: '+str(train_example.text_a))

        # Doble Saved:  So that no change on data even if we change the seed
        # if args.train_data_size > 0: # in some case, I might not cutting anything
        #     #So that no change on data even if we change the seed
        #     if not os.path.exists("./dat/dsc/"+str(random_sep[t])+"/train_data_"+str(args.train_data_size)+'_'+str(args.data_seed)):
        #         torch.save(train_examples,"./dat/dsc/"+str(random_sep[t])+"/train_data_"+str(args.train_data_size)+'_'+str(args.data_seed))
        #         logger.info("  save")
        #     else: #in that case, change seed will not change data
        #         train_examples = torch.load("./dat/dsc/"+str(random_sep[t])+"/train_data_"+str(args.train_data_size)+'_'+str(0)) #always use 0
        #         logger.info("  load")

        num_train_steps = int(math.ceil(len(train_examples) / args.train_batch_size)) * args.num_train_epochs
        # num_train_steps = int(len(train_examples) / args.train_batch_size) * args.num_train_epochs

        train_features = data_utils.convert_examples_to_features_dsc(
            train_examples, label_list, args.max_seq_length, tokenizer, "dsc")
        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_examples))
        logger.info("  Batch size = %d", args.train_batch_size)
        logger.info("  Num steps = %d", num_train_steps)

        all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)
        all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)
        all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)
        all_tasks = torch.tensor([t for f in train_features], dtype=torch.long)

        train_data = TensorDataset(all_input_ids, all_segment_ids, all_input_mask, all_label_ids, all_tasks)

        data[t]['train'] = train_data
        data[t]['num_train_steps']=num_train_steps

        valid_examples = processor.get_dev_examples(dataset)
        # No need to change valid for DSC
        # if args.dev_data_size > 0:
        #     random.Random(args.data_seed).shuffle(valid_examples) #more robust
        #     valid_examples = valid_examples[:args.dev_data_size]

        valid_features=data_utils.convert_examples_to_features_dsc(
            valid_examples, label_list, args.max_seq_length, tokenizer, "dsc")
        valid_all_input_ids = torch.tensor([f.input_ids for f in valid_features], dtype=torch.long)
        valid_all_segment_ids = torch.tensor([f.segment_ids for f in valid_features], dtype=torch.long)
        valid_all_input_mask = torch.tensor([f.input_mask for f in valid_features], dtype=torch.long)
        valid_all_label_ids = torch.tensor([f.label_id for f in valid_features], dtype=torch.long)
        valid_all_tasks = torch.tensor([t for f in valid_features], dtype=torch.long)

        valid_data = TensorDataset(valid_all_input_ids, valid_all_segment_ids, valid_all_input_mask, valid_all_label_ids, valid_all_tasks)

        logger.info("***** Running validations *****")
        logger.info("  Num orig examples = %d", len(valid_examples))
        logger.info("  Num split examples = %d", len(valid_features))
        logger.info("  Batch size = %d", args.train_batch_size)

        data[t]['valid']=valid_data

        processor = data_utils.DscProcessor()
        label_list = processor.get_labels()
        tokenizer = BertTokenizer.from_pretrained(args.bert_model)
        eval_examples = processor.get_test_examples(dataset)

        eval_features = data_utils.convert_examples_to_features_dsc(eval_examples, label_list, args.max_seq_length, tokenizer, "dsc")

        logger.info("***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Batch size = %d", args.eval_batch_size)
        all_input_ids = torch.tensor([f.input_ids for f in eval_features], dtype=torch.long)
        all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
        all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
        all_label_ids = torch.tensor([f.label_id for f in eval_features], dtype=torch.long)
        all_tasks = torch.tensor([t for f in eval_features], dtype=torch.long)

        eval_data = TensorDataset(all_input_ids, all_segment_ids, all_input_mask, all_label_ids, all_tasks)
        # Run prediction for full data

        data[t]['test']=eval_data

        taskcla.append((t,int(data[t]['ncla'])))

    # Others
    n=0
    for t in data.keys():
        n+=data[t]['ncla']
    data['ncla']=n

    return data,taskcla


# www.naver.com/robots.txt
import requests
from bs5 import BeautifulSoup
import time

# url = 'https://finance.naver.com/marketindex/'
#
# recvd = requests.get(url)
# # print(recvd)
# # print(recvd.text)
#
# dom = BeautifulSoup(recvd.text, 'lxml')
# span = dom.find('span', class_='value')
# print(span.text)

# print('-'*30)
# with open('data\\webtoon.csv', 'w', encoding='utf-8') as f:
#     url = 'https://comic.naver.com/webtoon/list.nhn?titleId=733766&weekday=mon'
#     recvd = requests.get(url)
#     # print(recvd)
#
#     dom = BeautifulSoup(recvd.text, 'lxml')
#     table = dom.find('table', class_='viewList')
#     # print(table)
#     trs = table.find_all('tr')
#     # print(len(trs))
#
#     for i in range(2, len(trs)):
#         td1 = trs[i].find('td', class_='title')
#         title = td1.find('a').text
#         div = trs[i].find('div', class_='rating_type')
#         rating = div.strong.text
#         # print(rating)
#         regdate = trs[i].find('td', class_='num').text
#         print('{},{},{}'.format(title, rating, regdate))
#         f.write('{},{},{}\n'.format(title, rating, regdate))

print('-'*30)

def saveImg(imgurl):
    print('저장중')

with open('data\\webtoon.csv', 'w', encoding='utf-8') as f:
    for page in range(1, 6+1):
        pageurl = 'https://comic.naver.com/webtoon/list.nhn?titleId=733766&weekday=mon&page={}'.format(page)

        recvd = requests.get(pageurl)

        dom = BeautifulSoup(recvd.text, 'lxml')
        table = dom.find('table', class_='viewList')
        trs = table.find_all('tr')

        for i in range(2, len(trs)):
            img = trs[i].find('td').find('img')['src']
            saveImg(img)
            td1 = trs[i].find('td', class_='title')
            title = td1.find('a').text
            div = trs[i].find('div', class_='rating_type')
            rating = div.strong.text
            regdate = trs[i].find('td', class_='num').text
            f.write('{},{},{},{}\n'.format(img, title, rating, regdate))


"""create tags table

Revision ID: 1bd006d42d5d
Revises: fd3dc97009bb
Create Date: 2022-10-18 14:59:09.926966

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '1bd006d42d5d'
down_revision = 'fd3dc97009bb'
branch_labels = None
depends_on = None

def upgrade() -> None:
    op.create_table(
        'tags',
        sa.Column('id', sa.Integer, primary_key=True, autoincrement=True),
        sa.Column('name', sa.String, nullable=False),
    )

def downgrade() -> None:
    op.drop_table('tags')


#!/usr/bin/python3
#
# background.py - Background generators
#
# Frank Blankenburg, May 2017
#

import argparse
import random
import math
import numpy as np
import os
import cProfile
import pstats
import sys

import common.utils as utils

import skimage.color
import skimage.exposure
import skimage.filters
import skimage.io
import skimage.transform
import skimage.util

from common.geometry import Point2d, Size2d, Rect2d
from generator.generator import Generator

#----------------------------------------------------------------------------------------------------------------------
# CLASS BackgroundGenerator
#
# Abstract base class for background generators
#
class BackgroundGenerator (Generator):

    def __init__ (self, args):
        super ().__init__ (args)

    #--------------------------------------------------------------------------
    # Return if this generator creates an active layer which must be detected as a separate image segmentation class
    #
    def is_active_layer (self):
        return False

    #--------------------------------------------------------------------------
    # Add args for background generator configuration to argument parser
    #
    @staticmethod
    def add_to_args_definition (parser):
        parser.add_argument ('-m', '--background_mode', action='store', choices=[NoisyRectBackgroundGenerator.TYPE,
                                                                                ImageBackgroundGenerator.TYPE,
                                                                                EmptyBackgroundGenerator.TYPE],
                             default=NoisyRectBackgroundGenerator.TYPE, help='Background creation mode')
        parser.add_argument ('-d', '--background_directory', type=str, default=None, help='Directory for database based background generation')

    #--------------------------------------------------------------------------
    # Create background generator matching the command line arguments
    #
    @staticmethod
    def create (args):

        if args.background_mode == EmptyBackgroundGenerator.TYPE:
            generator = EmptyBackgroundGenerator (args)

        elif args.background_mode == NoisyRectBackgroundGenerator.TYPE:
            generator = NoisyRectBackgroundGenerator (args)

        elif args.background_mode == ImageBackgroundGenerator.TYPE:
            generator = ImageBackgroundGenerator (args)

        else:
            raise RuntimeError ('Unknown background generator name \'{name}\''.arg (name=args.background_mode))

        return generator

#----------------------------------------------------------------------------------------------------------------------
# CLASS EmptyBackgroundGenerator
#
# Generate empty background
#
class EmptyBackgroundGenerator (BackgroundGenerator):

    TYPE = 'empty'

    #
    # Constructor
    #
    # @param args Command line arguments
    #
    def __init__ (self, args):
        super ().__init__ (args)

    #
    # Generate single image
    #
    def generate (self):
        return np.zeros ((self.height, self.width, 3), dtype=np.float64), None

#----------------------------------------------------------------------------------------------------------------------
# CLASS NoisyRectBackgroundGenerator
#
# Generate pattern based background
#
# This class will generate an image background consisting of random, blurred, noisy and variated rectangles
#
class NoisyRectBackgroundGenerator (BackgroundGenerator):

    TYPE = 'rects'

    #
    # Constructor
    #
    # @param args Command line arguments
    #
    def __init__ (self, args):
        super ().__init__ (args)

    #
    # Generate single image
    #
    def generate (self):

        shape = (int (self.height), int (self.width), 3)

        image = np.zeros (shape, dtype=np.float64)
        image = skimage.util.random_noise (image, mode='gaussian', seed=None, clip=True, mean=0.2, var=0.0001)

        for _ in range (random.randint (30, 80)):

            rect_image = np.zeros (shape, dtype=image.dtype)

            rect = Rect2d (Point2d (.2 * self.width, .45 * self.height),
                           Point2d (.8 * self.width, .55 * self.height))

            rect = rect.move_to (Point2d (random.randint (int (-.1 * self.width), int (.9 * self.width)),
                                          random.randint (int (-.1 * self.height), int (.9 * self.height))))

            color = (random.uniform (0.1, 0.7), random.uniform (0.1, 0.7), random.uniform (0.1, 0.7))
            rect.draw (rect_image, color, fill=True)

            if random.randint (0, 3) == 0:
                rect_image = skimage.util.random_noise (rect_image, mode='gaussian', seed=None, clip=True, mean=color, var=0.005)

            rect_image = utils.transform (rect_image, Size2d (self.width, self.height),
                                          scale=(random.uniform (0.3, 1.4), random.uniform (0.8, 1.2)),
                                          shear=random.uniform (0.0, 0.3),
                                          rotation=random.uniform (0.0, 2 * math.pi))

            mask = rect_image[:,:,0] >= color[0]
            mask |= rect_image[:,:,1] >= color[1]
            mask |= rect_image[:,:,2] >= color[2]
            image[mask] = rect_image[mask]

        image = skimage.filters.gaussian (image, sigma=3, multichannel=True)
        return image, None

#----------------------------------------------------------------------------------------------------------------------
# CLASS ImageBackgroundGenerator
#
# Background generator using a file based image database
#
# This generator class reads images from a database directory, selects appropriate parts matching the desired target
# resolution and transforms the results to get a larger base for the training set.
#
class ImageBackgroundGenerator (BackgroundGenerator):

    #
    # Type id for command line arguments
    #
    TYPE = 'imagedb'

    #
    # Constructor
    #
    # @param args Command line arguments
    #
    def __init__ (self, args):
        super ().__init__ (args)

        if args.background_directory is None:
            raise RuntimeError ('Directory must be specified when using mode \'imagedb\' (option -d)')

        path = os.path.abspath (args.background_directory)

        self.files = [os.path.join (path, file) for file in os.listdir (path) if utils.is_image (os.path.join (path, file))]

    #
    # Generate single image matching the given command line argument parameters
    #
    def generate (self):

        #
        # Read image
        #
        image = skimage.io.imread (random.choice (self.files))

        #
        # Randomly rotate image in 90 degree steps
        #
        rotation = random.choice ([0, 90, 180, 270])
        if rotation > 0:
            image = skimage.transform.rotate (image, angle=rotation, resize=True)

        #
        # Randomly flip image around the horizontal/vertical axis
        #
        flip = random.choice (['none', 'horizontal', 'vertical'])
        if flip == 'horizontal':
            image = np.fliplr (image)
        elif flip == 'vertical':
            image = np.flipud (image)

        #
        # Compute scale factor needed to squeeze / enlarge the source image to the target image size
        #
        scale = min (image.shape[0] / float (self.height), image.shape[1] / float (self.width))

        #
        # If the source image is larger than the target image, scale it down randomly so that we do not get small
        # image fragments only.
        #
        if scale > 1.0:
            scale = random.uniform (max (1.0, scale / 2), scale)

        #
        # Compute crop area and extract image part
        #
        crop_size = (int (self.height * scale), int (self.width * scale))

        crop_offset_y = random.randint (0, image.shape[0] - crop_size[0])
        crop_offset_x = random.randint (0, image.shape[1] - crop_size[1])

        image = image[crop_offset_y:crop_offset_y + crop_size[0], crop_offset_x:crop_offset_x + crop_size[1], :]

        #
        # The cropped image part can still be smaller that the target image and has to be scaled up accordingly
        #
        image = skimage.transform.resize (image, (self.height, self.width, image.shape[2]), mode='reflect')
        image = np.reshape (image, (image.shape[0], image.shape[1], image.shape[2]))

        return image, None

#----------------------------------------------------------------------------------------------------------------------
# MAIN
#
if __name__ == '__main__':

    random.seed ()

    #
    # Parse command line arguments
    #
    parser = argparse.ArgumentParser ()

    parser.add_argument ('-x', '--width',   type=int,            default=512,   help='Width of the generated image')
    parser.add_argument ('-y', '--height',  type=int,            default=512,   help='Height of the generated image')
    parser.add_argument ('-p', '--profile', action='store_true', default=False, help='Profile run')

    BackgroundGenerator.add_to_args_definition (parser)

    args = parser.parse_args ()

    #
    # Instantiate generator class from command line arguments
    #
    generator = BackgroundGenerator.create (args)

    if args.profile:
        pr = cProfile.Profile ()
        pr.enable ()

    #
    # Generate image probe
    #
    image, mask = generator.generate ()

    assert mask is None
    assert image.shape[0] == args.height
    assert image.shape[1] == args.width
    assert image.shape[2] == 3
    assert image.dtype == np.float64

    if args.profile:
        pr.disable ()

        stats = pstats.Stats (pr, stream=sys.stdout).sort_stats ('cumulative')
        stats.print_stats ()

    utils.show_image ([image, 'Generated background'])


import time
import openpyxl
from aiogram import Bot, Dispatcher, executor, types
from config import tg_token, file_link

bot = Bot(token=tg_token)
dp = Dispatcher(bot)

@dp.message_handler(commands=['start'])
async def start_command(message: types.Message):
    await message.answer("Приветствую!")
    await message.answer("Давай начнем поиск отделений Сбербанка, где можно приобрести валюту в кассе")
    await message.answer("Введи название города на русском языке")

@dp.message_handler()
async def get_data(message: types.Message):
    await message.answer("Пожалуйста, подожди. Идет сбор данных... \U0001F609")
    time.sleep(1)

    try:
        book = openpyxl.open("exchange.xlsx", read_only=True)
        sheet = book.active
        address = sheet['F3:F999']

        for row in address:
            for cell in row:
                address_list = cell.value
                if f'г.{message.text}'.lower().replace('ё', 'е') in address_list.lower().replace('ё', 'е'):
                    await message.answer(address_list)

    except:
        await message.answer('\U0001F635 Упс... что-то пошло не так. Попробуй снова')

    time.sleep(1)
    await message.answer("Можно приступать к поиску ближайшего к вам адреса. Удачи! \U0001F601")

if __name__ == '__main__':
    executor.start_polling(dp)


from environment import Environment
import os
import matplotlib.pyplot as plt
import numpy as np
import random
import torch
import torch.nn as nn
import torch.optim as optim
from torch.nn.utils import clip_grad_norm_

class ReplayBuffer:
    def __init__(self, column, max_size, batch_size):
        self.current_state = np.zeros((max_size, column), dtype=np.float32)
        self.next_state = np.zeros((max_size, column), dtype=np.float32)
        self.action = np.zeros(max_size, dtype=np.float32)
        self.reward = np.zeros(max_size, dtype=np.float32)
        self.done = np.zeros(max_size,dtype=np.float32)
        self.max_size, self.batch_size = max_size, batch_size
        self.size, self.current_index = 0, 0

    def store(self, current_state, action, next_state, reward, done):
        self.current_state[self.current_index] = current_state
        self.action[self.current_index] = action
        self.next_state[self.current_index] = next_state
        self.reward[self.current_index] = reward
        self.done[self.current_index] = done
        self.current_index = (self.current_index + 1) % self.max_size
        self.size = min(self.size + 1, self.max_size)

    def sample_batch(self):
        ptr = np.random.choice(self.size, self.batch_size)
        return dict(current_state=self.current_state[ptr],
                    next_state=self.next_state[ptr],
                    action=self.action[ptr],
                    reward=self.reward[ptr],
                    done=self.done[ptr]
        )

    def __len__(self):
        return self.size

class Network(nn.Module):
    def __init__(self, in_dim, out_dim):
        super(Network, self).__init__()

        self.layers = nn.Sequential(
            nn.Linear(in_dim, 128),
            nn.ReLU(),
            nn.Linear(128, 128),
            nn.ReLU(),
            nn.Linear(128, out_dim)
        )
    def forward(self,x):
        return self.layers(x)

min_epsilon = 0.05
max_epsilon = 1
epsilon_decay = 80
epsilon_episode = lambda episode : min_epsilon + np.exp(-episode / epsilon_decay)*0.95

env = Environment("test_you")
state_space = 3
action_space = 3

batch_size = 32
max_size = 1000
memory = ReplayBuffer(state_space, max_size, batch_size)

device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")

network = Network(state_space,action_space).to(device)
target_network = Network(state_space,action_space).to(device)
target_network.load_state_dict(network.state_dict())
target_network.eval()

optimizer = optim.Adam(network.parameters())

gamma = 0.99
target_update = 200

def select_action(episode, state):
    if np.random.random_sample() > epsilon_episode(episode):
        selected_action = network(torch.FloatTensor(state).to(device)).argmax().detach().item()
    else:
        selected_action = random.randint(0,2)
    return selected_action

def train():
    samples = memory.sample_batch()
    state = torch.FloatTensor(samples["current_state"]).to(device).to(device)
    next_state = torch.FloatTensor(samples["next_state"]).to(device)
    action = torch.LongTensor(samples["action"].reshape(-1, 1)).to(device)
    reward = torch.FloatTensor(samples["reward"].reshape(-1, 1)).to(device)
    done = torch.FloatTensor(samples["done"].reshape(-1, 1)).to(device)

    current_Q_value = network(state).gather(1, action)
    next_Q_value = target_network(next_state).max(dim=1,keepdim=True)[0].detach()
    target = (reward + gamma*next_Q_value*(1 - done)).to(device)
    loss = ((target - current_Q_value).pow(2)).mean()
    optimizer.zero_grad()
    loss.backward()
    clip_grad_norm_(network.parameters(),1.0,norm_type=1) # Gradient clipping(增加稳定性)
    optimizer.step()
    return loss.item()

def plot_and_save(frame_idx, rewards, losses):
    rewards_factor = 10
    losses_smooth_x = np.arange(len(losses))
    losses_smooth = [np.mean(losses[i:i+rewards_factor]) if i > rewards_factor else np.mean(losses[0:i+1])
                     for i in range(len(losses))]
    rewards_smooth_x = np.arange(len(rewards))
    rewards_smooth = [np.mean(rewards[i:i+rewards_factor]) if i > rewards_factor else np.mean(rewards[0:i+1])
                      for i in range(len(rewards))]

    for i in range(len(losses)//3000):
        losses_smooth = losses_smooth[::2]
        losses_smooth_x = losses_smooth_x[::2]
    for i in range(len(rewards)//200):
        rewards_smooth = rewards_smooth[::2]
        rewards_smooth_x = rewards_smooth_x[::2]

    plt.figure(figsize=(18,10))
    plt.subplot(211)
    plt.xlabel("episode")
    plt.ylabel("episode_rewards")
    plt.title('episode %s. rewards: %s' % (frame_idx, rewards[-1]))
    plt.plot(rewards, label="Rewards",color='lightsteelblue',linewidth='1')
    plt.plot(rewards_smooth_x, rewards_smooth,
             label="Smoothed_Rewards",color='darkorange',linewidth='3')
    plt.legend(loc='best')

    plt.subplot(212)
    plt.title('loss')
    plt.plot(losses,label="Losses",color='lightsteelblue',linewidth='1')
    plt.plot(losses_smooth_x, losses_smooth,
             label="Smoothed_Losses",color='darkorange',linewidth='3')
    plt.legend(loc='best')

    plt.savefig("dqn_test.png")

all_rewards = []
losses = []
update_count = 0

env.create_sockets_server()
state = env.reset()
for episode in range(200):
    rewards = 0
    for i in range(100):
        action = select_action(episode, state)
        next_state, reward, done, _ = env.step(action)
        print("episode:",episode,"state:",next_state,"reward:", reward,"action:",action )
        memory.store(state, action, next_state, reward, done)
        state = next_state
        rewards += reward
        #if done:
            #break
        if len(memory) > batch_size:
            loss = train()
            update_count += 1
            losses.append(loss)
            if update_count % target_update == 0:
                target_network.load_state_dict(network.state_dict())
    all_rewards.append(rewards)
    plot_and_save(episode , [round(all_rewards[i], 3) for i in range(len(all_rewards))], losses)


"""
Clickhouse-control classes module
"""

import os
import shutil
from contextlib import contextmanager, suppress
from hashlib import md5
from pathlib import Path
from tarfile import BLOCKSIZE  # type: ignore
from typing import Any, Dict, List, Optional, Sequence, Union

from pkg_resources import parse_version

from ch_backup import logging
from ch_backup.backup.metadata import TableMetadata
from ch_backup.backup.restore_context import RestoreContext
from ch_backup.calculators import calc_aligned_files_size
from ch_backup.clickhouse.client import ClickhouseClient
from ch_backup.clickhouse.models import Database, Disk, FrozenPart, Table
from ch_backup.clickhouse.schema import is_replicated
from ch_backup.exceptions import ClickhouseBackupError
from ch_backup.util import (
    chown_dir_contents,
    escape,
    list_dir_files,
    retry,
    strip_query,
)

ACCESS_ENTITY_CHAR = {
    "users": "U",
    "roles": "R",
    "quotas": "Q",
    "row_policies": "P",
    "settings_profiles": "S",
}

GET_TABLES_SQL = strip_query(
    """
    SELECT
        database,
        name,
        engine,
        engine_full,
        create_table_query,
        data_paths,
        metadata_path,
        uuid
    FROM system.tables
    WHERE (empty('{db_name}') OR database = '{db_name}')
      AND (empty({table_names}) OR has(cast({table_names}, 'Array(String)'), name))
    ORDER BY metadata_modification_time
    FORMAT JSON
"""
)

GET_TABLES_SHORT_SQL = strip_query(
    """
    SELECT
        database,
        name,
        create_table_query
    FROM system.tables
    WHERE (empty('{db_name}') OR database = '{db_name}')
      AND (empty({table_names}) OR has(cast({table_names}, 'Array(String)'), name))
    ORDER BY metadata_modification_time
    FORMAT JSON
"""
)

GET_TABLE_SQL = strip_query(
    """
    SELECT
        database,
        name,
        engine,
        engine_full,
        create_table_query,
        data_paths,
        metadata_path,
        uuid
    FROM system.tables
    WHERE database = '{db_name}' AND name = '{table_name}'
    FORMAT JSON
"""
)

CHECK_TABLE_SQL = strip_query(
    """
    SELECT countIf(database = '{db_name}' AND name = '{table_name}')
    FROM system.tables
    FORMAT TSVRaw
"""
)

DATABASE_ATTACH_SQL = strip_query(
    """
    ATTACH DATABASE `{db_name}`
"""
)

PART_ATTACH_SQL = strip_query(
    """
    ALTER TABLE `{db_name}`.`{table_name}`
    ATTACH PART '{part_name}'
"""
)

TABLE_ATTACH_SQL = strip_query(
    """
    ATTACH TABLE `{db_name}`.`{table_name}`
"""
)

FREEZE_TABLE_SQL = strip_query(
    """
    ALTER TABLE `{db_name}`.`{table_name}`
    FREEZE WITH NAME '{backup_name}'
"""
)

SYSTEM_UNFREEZE_SQL = strip_query(
    """
    SYSTEM UNFREEZE WITH NAME '{backup_name}'
"""
)

DROP_TABLE_IF_EXISTS_SQL = strip_query(
    """
    DROP TABLE IF EXISTS `{db_name}`.`{table_name}` NO DELAY
"""
)

DROP_DICTIONARY_IF_EXISTS_SQL = strip_query(
    """
    DROP DICTIONARY IF EXISTS `{db_name}`.`{table_name}` NO DELAY
"""
)

DROP_DATABASE_IF_EXISTS_SQL = strip_query(
    """
    DROP DATABASE IF EXISTS `{db_name}` NO DELAY
"""
)

DROP_UDF_SQL = strip_query(
    """
    DROP FUNCTION `{udf_name}`
"""
)

DROP_NAMED_COLLECTION_SQL = strip_query(
    """
    DROP NAMED COLLECTION `{nc_name}`
"""
)

RESTORE_REPLICA_SQL = strip_query(
    """
    SYSTEM RESTORE REPLICA `{db_name}`.`{table_name}`
"""
)

GET_DATABASES_SQL = strip_query(
    """
    SELECT
        name,
        engine,
        metadata_path
    FROM system.databases
    WHERE name NOT IN ('system', '_temporary_and_external_tables', 'information_schema', 'INFORMATION_SCHEMA')
    FORMAT JSON
"""
)

SHOW_CREATE_DATABASE_SQL = strip_query(
    """
    SHOW CREATE DATABASE `{db_name}`
    FORMAT TSVRaw
"""
)

GET_DATABASE_ENGINE = strip_query(
    """
    SELECT engine FROM system.databases WHERE name='{db_name}'
    FORMAT TSVRaw
"""
)

GET_DATABASE_METADATA_PATH = strip_query(
    """
    SELECT metadata_path FROM system.databases WHERE name='{db_name}'
    FORMAT JSON
"""
)

GET_VERSION_SQL = strip_query(
    """
    SELECT version()
    FORMAT TSVRaw
"""
)

GET_MACROS_SQL = strip_query(
    """
    SELECT macro, substitution FROM system.macros
    FORMAT JSON
"""
)

GET_ACCESS_CONTROL_OBJECTS_SQL = strip_query(
    """
    SELECT id, name
    FROM system.{type}
    WHERE storage IN ('disk', 'local directory', 'local_directory', 'replicated')
    FORMAT JSON
"""
)

GET_DISK_SQL = strip_query(
    """
    SELECT name, path, type, cache_path FROM system.disks
    WHERE name = '{disk_name}'
    FORMAT JSON
"""
)

GET_DISKS_SQL = strip_query(
    """
    SELECT name, path, type FROM system.disks
    ORDER BY length(path) DESC
    FORMAT JSON
"""
)

GET_DISKS_SQL_22_8 = strip_query(
    """
    SELECT name, path, type, cache_path FROM system.disks
    ORDER BY length(path) DESC
    FORMAT JSON
"""
)

GET_UDF_QUERY_SQL = strip_query(
    """
    SELECT name, create_query FROM system.functions WHERE origin == 'SQLUserDefined'
    FORMAT JSON
"""
)

GET_NAMED_COLLECTIONS_QUERY_SQL = strip_query(
    """
    SELECT name FROM system.named_collections
    FORMAT JSON
"""
)

RELOAD_CONFIG_SQL = strip_query(
    """
    SYSTEM RELOAD CONFIG
"""
)

GET_ZOOKEEPER_ADMIN_USER = strip_query(
    """
    SELECT
        name,
        value
    FROM system.zookeeper
    WHERE (path = '/clickhouse/access/U') AND (name = 'admin')
    FORMAT JSON
"""
)

GET_ZOOKEEPER_ADMIN_UUID = strip_query(
    """
    SELECT
        name,
        value
    FROM system.zookeeper
    WHERE (path = '/clickhouse/access/uuid') AND (value LIKE 'ATTACH USER admin%')
    FORMAT JSON
"""
)

# pylint: disable=too-many-public-methods
class ClickhouseCTL:
    """
    ClickHouse control tool.
    """

    # pylint: disable=too-many-instance-attributes

    def __init__(self, ch_ctl_config: dict, main_config: dict) -> None:
        self._ch_ctl_config = ch_ctl_config
        self._main_config = main_config
        self._root_data_path = self._ch_ctl_config["data_path"]
        self._shadow_data_path = os.path.join(self._root_data_path, "shadow")
        self._timeout = self._ch_ctl_config["timeout"]
        self._freeze_timeout = self._ch_ctl_config["freeze_timeout"]
        self._unfreeze_timeout = self._ch_ctl_config["unfreeze_timeout"]
        self._restore_replica_timeout = self._ch_ctl_config["restore_replica_timeout"]
        self._ch_client = ClickhouseClient(self._ch_ctl_config)
        self._ch_version = self._ch_client.query(GET_VERSION_SQL)
        self._disks = self.get_disks()
        settings = {
            "allow_experimental_database_materialized_postgresql": 1,
            "allow_experimental_database_materialized_mysql": 1,
            "allow_experimental_database_replicated": 1,
            "allow_experimental_funnel_functions": 1,
            "allow_experimental_live_view": 1,
            "allow_experimental_window_view": 1,
            "allow_suspicious_codecs": 1,
            "allow_suspicious_low_cardinality_types": 1,
        }
        if self.ch_version_ge("22.3"):
            settings.update(
                {
                    "allow_experimental_object_type": 1,
                }
            )
        if self.ch_version_ge("22.6"):
            settings.update(
                {
                    "allow_experimental_hash_functions": 1,
                }
            )
        if self.ch_version_ge("22.7"):
            settings.update(
                {
                    "allow_deprecated_database_ordinary": 1,
                    "allow_deprecated_syntax_for_merge_tree": 1,
                }
            )
        if self.ch_version_ge("22.8.5"):
            settings.update(
                {
                    "check_table_dependencies": 0,
                    "kafka_disable_num_consumers_limit": 1,
                }
            )
        if self.ch_version_ge("22.9"):
            settings.update(
                {
                    "allow_experimental_annoy_index": 1,
                    "allow_suspicious_fixed_string_types": 1,
                }
            )
        if self.ch_version_ge("23.1"):
            settings.update(
                {
                    "allow_experimental_inverted_index": 1,
                }
            )
        self._ch_client.settings.update(settings)

    def chown_detached_table_parts(self, table: Table, context: RestoreContext) -> None:
        """
        Change permissions (owner and group) of detached data parts for the
        specified table. New values for permissions are taken from the config.
        """
        for _, disk in table.paths_with_disks:
            detached_path = self._get_table_detached_path(table, disk.name)
            try:
                self.chown_dir(detached_path)
            except FileNotFoundError:
                logging.warning(
                    f"table {table.database}.{table.name} path {detached_path} not found"
                )
                context.add_failed_chown(table.database, table.database, detached_path)

    def attach_part(self, table: Table, part_name: str) -> None:
        """
        Attach data part to the specified table.
        """
        query_sql = PART_ATTACH_SQL.format(
            db_name=escape(table.database),
            table_name=escape(table.name),
            part_name=part_name,
        )

        self._ch_client.query(query_sql)

    def attach_table(self, table: Union[TableMetadata, Table]) -> None:
        """
        Attach data part to the specified table.
        """
        query_sql = TABLE_ATTACH_SQL.format(
            db_name=escape(table.database), table_name=escape(table.name)
        )

        self._ch_client.query(query_sql)

    def freeze_table(self, backup_name: str, table: Table) -> None:
        """
        Make snapshot of the specified table.
        """
        query_sql = FREEZE_TABLE_SQL.format(
            db_name=escape(table.database),
            table_name=escape(table.name),
            backup_name=backup_name,
        )
        self._ch_client.query(query_sql, timeout=self._freeze_timeout)

    def system_unfreeze(self, backup_name: str) -> None:
        """
        Unfreeze all partitions from all disks.
        """
        if self.ch_version_ge("22.6"):
            query_sql = SYSTEM_UNFREEZE_SQL.format(backup_name=backup_name)
            self._ch_client.query(query_sql, timeout=self._unfreeze_timeout)

    def remove_freezed_data(self) -> None:
        """
        Remove all freezed partitions from all local disks.
        """
        for disk in self._disks.values():
            if disk.type == "local":
                shadow_path = os.path.join(disk.path, "shadow")
                logging.debug("Removing shadow data: {}", shadow_path)
                self._remove_shadow_data(shadow_path)

    def remove_freezed_part(self, part: FrozenPart) -> None:
        """
        Remove the freezed part.
        """
        logging.debug("Removing freezed part: {}", part.path)
        self._remove_shadow_data(part.path)

    def get_databases(
        self, exclude_dbs: Optional[Sequence[str]] = None
    ) -> Sequence[Database]:
        """
        Get list of all databases.
        """
        if not exclude_dbs:
            exclude_dbs = []

        result: List[Database] = []
        ch_resp = self._ch_client.query(GET_DATABASES_SQL)
        if "data" in ch_resp:
            result = [
                Database(row["name"], row["engine"], row["metadata_path"])
                for row in ch_resp["data"]
                if row["name"] not in exclude_dbs
            ]

        return result

    def get_database_schema(self, db_name: str) -> str:
        """
        Return database schema (CREATE DATABASE query).
        """
        query_sql = SHOW_CREATE_DATABASE_SQL.format(db_name=escape(db_name))
        return self._ch_client.query(query_sql)

    def get_database_engine(self, db_name: str) -> str:
        """
        Return database engine.
        """
        query_sql = GET_DATABASE_ENGINE.format(db_name=escape(db_name))
        return self._ch_client.query(query_sql)

    def get_tables(
        self,
        db_name: str = None,
        tables: Optional[Sequence[str]] = None,
        short_query: bool = False,
    ) -> Sequence[Table]:
        """
        Get database tables.

        A short query does not access the source of table if it was built from an external source.
        Example: CREATE ... AS postgresql() or CREATE ... AS s3().
        """
        base_query_sql = GET_TABLES_SHORT_SQL if short_query else GET_TABLES_SQL
        query_sql = base_query_sql.format(
            db_name=escape(db_name) if db_name is not None else "",
            table_names=list(map(escape, tables)) if tables is not None else [],
        )  # type: ignore
        result: List[Table] = []
        for row in self._ch_client.query(query_sql)["data"]:
            result.append(self._make_table(row))

        return result

    def get_table(self, db_name: str, table_name: str) -> Optional[Table]:
        """
        Get table by name, returns None if no table has found.
        """
        query_sql = GET_TABLE_SQL.format(
            db_name=escape(db_name), table_name=escape(table_name)
        )

        result = self._ch_client.query(query_sql)["data"]
        if result:
            return self._make_table(result[0])

        return None

    def does_table_exist(self, db_name: str, table_name: str) -> bool:
        """
        Return True if the specified table exists.
        """
        query_sql = CHECK_TABLE_SQL.format(
            db_name=escape(db_name), table_name=escape(table_name)
        )
        return bool(int(self._ch_client.query(query_sql)))

    def attach_database(self, db: Database) -> None:
        """
        Restore database.
        """
        self._ch_client.query(DATABASE_ATTACH_SQL.format(db_name=escape(db.name)))

    def restore_database(self, database_schema: str) -> None:
        """
        Restore database.
        """
        self._ch_client.query(database_schema)

    def restore_udf(self, udf_statement):
        """
        Restore user defined function.
        """
        self._ch_client.query(udf_statement)

    def restore_named_collection(self, nc_statement):
        """
        Restore named collection.
        """
        self._ch_client.query(nc_statement)

    def create_table(self, table: Table) -> None:
        """
        Restore table.
        """
        self._ch_client.query(table.create_statement)

    def restore_replica(self, table: Table) -> None:
        """
        Call SYSTEM RESTORE REPLICA for table.
        """
        assert is_replicated(table.create_statement)
        self._ch_client.query(
            RESTORE_REPLICA_SQL.format(
                db_name=escape(table.database), table_name=escape(table.name)
            ),
            timeout=self._restore_replica_timeout,
        )

    def drop_table_if_exists(self, table: Table) -> None:
        """
        Drop table. If the specified table doesn't exist, do nothing.
        """
        with self._force_drop_table():
            self._ch_client.query(
                DROP_TABLE_IF_EXISTS_SQL.format(
                    db_name=escape(table.database), table_name=escape(table.name)
                )
            )

    def drop_dictionary_if_exists(self, table: Table) -> None:
        """
        Drop dictionary. If the specified dictionary doesn't exist, do nothing.
        """
        self._ch_client.query(
            DROP_DICTIONARY_IF_EXISTS_SQL.format(
                db_name=escape(table.database), table_name=escape(table.name)
            )
        )

    def drop_database_if_exists(self, db_name: str) -> None:
        """
        Drop database. If the specified database doesn't exist, do nothing.
        """
        with self._force_drop_table():
            self._ch_client.query(
                DROP_DATABASE_IF_EXISTS_SQL.format(db_name=escape(db_name))
            )

    def drop_udf(self, udf_name: str) -> None:
        """
        Drop user defined function.
        """
        self._ch_client.query(DROP_UDF_SQL.format(udf_name=escape(udf_name)))

    def drop_named_collection(self, nc_name: str) -> None:
        """
        Drop named collection.
        """
        self._ch_client.query(DROP_NAMED_COLLECTION_SQL.format(nc_name=escape(nc_name)))

    def get_database_metadata_path(self, database: str) -> str:
        """
        Get filesystem absolute path to database metadata.
        """
        data = self._ch_client.query(
            GET_DATABASE_METADATA_PATH.format(db_name=database)
        )["data"]
        assert len(data) == 1
        return data[0]["metadata_path"]

    def get_detached_part_path(
        self, table: Table, disk_name: str, part_name: str
    ) -> str:
        """
        Get filesystem absolute path to detached data part.
        """
        return os.path.join(self._get_table_detached_path(table, disk_name), part_name)

    def get_version(self) -> str:
        """
        Get ClickHouse version.
        """
        return self._ch_version

    def get_access_control_objects(self) -> Sequence[Dict[str, Any]]:
        """
        Returns all access control objects.
        """
        result: List[Dict[str, Any]] = []

        for obj_type, obj_char in ACCESS_ENTITY_CHAR.items():
            ch_resp = self._ch_client.query(
                GET_ACCESS_CONTROL_OBJECTS_SQL.format(type=obj_type)
            )
            obj_result = ch_resp.get("data", [])
            for row in obj_result:
                row.update({"char": obj_char})
            result.extend(obj_result)

        return result

    def get_zookeeper_admin_id(self) -> str:
        """
        Returns linked admin's UUID from zookeeper.
        """
        result = self._ch_client.query(GET_ZOOKEEPER_ADMIN_USER).get("data", [])
        assert len(result) == 1
        return result[0]["value"]

    def get_zookeeper_admin_uuid(self) -> Dict[str, str]:
        """
        Returns all UUIDs associated with admin user.
        """
        result = self._ch_client.query(GET_ZOOKEEPER_ADMIN_UUID).get("data", [])
        return {item["name"]: item["value"] for item in result}

    @staticmethod
    def list_frozen_parts(
        table: Table, disk: Disk, data_path: str, backup_name: str
    ) -> Sequence[FrozenPart]:
        """
        List frozen parts from specific disk and path.
        """
        table_relative_path = os.path.relpath(data_path, disk.path)
        path = os.path.join(disk.path, "shadow", backup_name, table_relative_path)

        if not os.path.exists(path):
            logging.debug("Shadow path {} is empty", path)
            return []

        freezed_parts: List[FrozenPart] = []
        for part in os.listdir(path):
            part_path = os.path.join(path, part)
            checksum = _get_part_checksum(part_path)
            rel_paths = list_dir_files(part_path)
            abs_paths = [Path(part_path) / file for file in rel_paths]

            size = calc_aligned_files_size(abs_paths, alignment=BLOCKSIZE)
            logging.debug(
                f"list_freezed_parts: {table.name} -> {escape(table.name)} \n {part}"
            )
            freezed_parts.append(
                FrozenPart(
                    table.database,
                    table.name,
                    part,
                    disk.name,
                    part_path,
                    checksum,
                    size,
                    rel_paths,
                )
            )

        return freezed_parts

    @staticmethod
    def _get_table_detached_path(table: Table, disk_name: str) -> str:
        for data_path, disk in table.paths_with_disks:
            if disk.name == disk_name:
                return os.path.join(data_path, "detached")
        raise RuntimeError(f"Disk not found: {disk_name}")

    def chown_dir(self, dir_path: str) -> None:
        """
        Change owner and group for all files in folder.
        """
        need_recursion = not self._main_config["drop_privileges"]
        chown_dir_contents(
            self._ch_ctl_config["user"],
            self._ch_ctl_config["group"],
            dir_path,
            need_recursion,
        )

    @retry(OSError)
    def _remove_shadow_data(self, path: str) -> None:
        if path.find("/shadow") == -1:
            raise ClickhouseBackupError(
                f"Path '{path}' is incompatible for shadow data"
            )

        if os.path.exists(path):
            logging.debug(f"Path {path} exists. Trying to remove it.")

            shutil.rmtree(path)

            msg = "shutil.rmtree has done working"
            if os.path.exists(path):
                msg += ", but path still exists"
            logging.debug(msg)
        else:
            logging.debug(f"Path {path} does not exist. There is nothing to remove.")

    def ch_version_ge(self, comparing_version: str) -> bool:
        """
        Returns True if ClickHouse version >= comparing_version.
        """
        return parse_version(self.get_version()) >= parse_version(comparing_version)  # type: ignore

    def get_macros(self) -> Dict:
        """
        Get ClickHouse macros.
        """
        ch_resp = self._ch_client.query(GET_MACROS_SQL)
        return {row["macro"]: row["substitution"] for row in ch_resp.get("data", [])}

    def get_udf_query(self) -> Dict[str, str]:
        """
        Get udf query from system table.
        """
        resp = self._ch_client.query(GET_UDF_QUERY_SQL)
        return {row["name"]: row["create_query"] for row in resp.get("data", [])}

    def get_named_collections_query(self) -> List[str]:
        """
        Get named collections query from system table.
        """
        resp = self._ch_client.query(GET_NAMED_COLLECTIONS_QUERY_SQL)
        return [row["name"] for row in resp.get("data", [])]

    def get_disk(self, disk_name: str) -> Disk:
        """
        Get disk by name.
        """
        resp = self._ch_client.query(GET_DISK_SQL.format(disk_name=disk_name)).get(
            "data"
        )
        assert resp, f"disk '{disk_name}' not found"
        resp = resp[0]
        return Disk(resp["name"], resp["path"], resp["type"], resp["cache_path"])

    def get_disks(self) -> Dict[str, Disk]:
        """
        Get all configured disks.
        """
        if self.ch_version_ge("22.8"):
            disks_resp = self._ch_client.query(GET_DISKS_SQL_22_8)
            return {
                row["name"]: Disk(
                    row["name"], row["path"], row["type"], row["cache_path"]
                )
                for row in disks_resp.get("data", [])
            }
        disks_resp = self._ch_client.query(GET_DISKS_SQL)
        return {
            row["name"]: Disk(row["name"], row["path"], row["type"])
            for row in disks_resp.get("data", [])
        }

    def _make_table(self, record: dict) -> Table:
        return Table(
            database=record["database"],
            name=record["name"],
            engine=record.get("engine", None),
            disks=list(self._disks.values()),
            data_paths=record.get("data_paths", None)
            if "MergeTree" in record.get("engine", "")
            else [],
            metadata_path=record.get("metadata_path", None),
            create_statement=record["create_table_query"],
            uuid=record.get("uuid", None),
        )

    def read_s3_disk_revision(self, disk_name: str, backup_name: str) -> Optional[int]:
        """
        Reads S3 disk revision counter.
        """
        file_path = os.path.join(
            self._disks[disk_name].path, "shadow", backup_name, "revision.txt"
        )
        if not os.path.exists(file_path):
            return None

        with open(file_path, "r", encoding="utf-8") as file:
            return int(file.read().strip())

    def reload_config(self):
        """
        Reload ClickHouse configuration query.
        """
        self._ch_client.query(RELOAD_CONFIG_SQL, timeout=self._timeout)

    @staticmethod
    @contextmanager
    def _force_drop_table():
        """
        Set and clear on exit force_drop_table flag.

        If it has been set before then don't touch it.
        This flag allows to overcome TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT ClickHouse error.
        """
        flag_path = Path("/var/lib/clickhouse/flags/force_drop_table")

        flag_path.touch()
        try:
            flag_path.chmod(0o666)
            yield
        finally:
            with suppress(FileNotFoundError):
                flag_path.unlink()

def _get_part_checksum(part_path: str) -> str:
    with open(os.path.join(part_path, "checksums.txt"), "rb") as f:
        return md5(f.read()).hexdigest()  # nosec


import zipfile

def zip_files(path, file_names):
    zfile = zipfile.ZipFile(path, 'w')
    for path in file_names:
        zfile.write(path)

    zfile.close()

QUERY_FILES = ['credentials/trader_bot0', 'backtester.py', 'indicators.py', 'lambda_function.py', 'sm_util.py', 'strategy.py']
zip_files('query.zip', QUERY_FILES)
AUTOUPDATER_FILES = ['autoupdater.py', 'credentials/alphavantage_key0', 'credentials/admin_credentials', 'backtester.py', 'indicators.py','sm_util.py', 'strategy.py', 'alphavantage.py']
zip_files('autoupdater.zip', AUTOUPDATER_FILES)


import asyncio
import itertools
import networkx as nx
import os
import re
from math import inf
from imdb_helper_functions import create_visited_pairs_checkpoint,\
    get_all_pages, get_correct_url, get_graph, get_results_csv,\
    get_soup, get_pages_for_description, read_graph_from_checkpoint,\
    read_labels_from_checkpoint, read_visited_pairs_from_checkpoint,\
    set_is_full_movie_is_realised

MAX_DEPTH_SEARCH = 3
ACTORS_SEARCH_LIMIT = 5
MOVIES_SEARCH_LIMIT = 5
DISTANCE_STEP = 0.5
HEADERS = {'Accept-Language': 'en', 'X-FORWARDED-FOR': '2.21.184.0'}

def get_actors_by_movie_soup(
        cast_page_soup,
        num_of_actors_limit=None
):
    all_actors = cast_page_soup.find_all(
        'td',
        attrs={'class': 'primary_photo'},
        limit=num_of_actors_limit
    )
    actors_list = []
    for actor in all_actors:
        link = actor.find('a')['href']
        actors_list.append(
            (actor.find('img')['title'],
             f'https://www.imdb.com{link}'))
    return actors_list

def get_movies_by_actor_soup(
        actor_page_soup,
        num_of_movies_limit=None
):
    all_movies = actor_page_soup.find_all(
        'div',
        attrs={'id': re.compile(r'(act(or|ress)-tt\d+)')}
    )
    movie_list = []
    for movie in all_movies:
        if set_is_full_movie_is_realised(movie):
            link = movie.find('a')['href']
            movie_list.append(
                (movie.b.text,
                 f'https://www.imdb.com{link}')
            )
        if num_of_movies_limit and len(movie_list) == num_of_movies_limit:
            break
    return movie_list

def get_movie_distance(actor_start_url,
                       actor_end_url,
                       num_of_actors_limit=None,
                       num_of_movies_limit=None,
                       num_of_depth_level_limit=None):
    actor_start_url = get_correct_url(actor_start_url)
    actor_end_url = get_correct_url(actor_end_url)

    if os.path.exists('visited_pairs.dump'):
        visited_pairs = read_visited_pairs_from_checkpoint()
        if (actor_start_url, actor_end_url) in visited_pairs:
            G_all_actors = read_graph_from_checkpoint()
            if actor_start_url in G_all_actors.nodes() and \
                    actor_end_url in G_all_actors.nodes() and \
                    nx.has_path(G_all_actors, actor_start_url, actor_end_url):
                return int(nx.shortest_path_length(G_all_actors,
                                                   actor_start_url,
                                                   actor_end_url)
                           * DISTANCE_STEP)
            return inf

    get_graph(actor_start_url,
              actor_end_url,
              get_actors_by_movie_soup,
              get_movies_by_actor_soup,
              num_of_actors_limit,
              num_of_movies_limit,
              num_of_depth_level_limit)

    G_all_actors = read_graph_from_checkpoint()
    if os.path.exists('visited_pairs.dump'):
        visited_pairs = read_visited_pairs_from_checkpoint()
    else:
        visited_pairs = []
    visited_pairs.append((actor_start_url, actor_end_url))
    create_visited_pairs_checkpoint(visited_pairs)

    if actor_start_url in G_all_actors.nodes() and\
            actor_end_url in G_all_actors.nodes() and\
            nx.has_path(G_all_actors, actor_start_url, actor_end_url):
        return int(nx.shortest_path_length(G_all_actors,
                                           actor_start_url,
                                           actor_end_url) * DISTANCE_STEP)

    return inf

def get_movie_descriptions_by_actor_soup(actor_page_soup):
    descriptions = []
    movies = get_movies_by_actor_soup(actor_page_soup)
    movies_urls = list(map(lambda x: x[1], movies))
    loop = asyncio.get_event_loop()
    movies_html = loop.run_until_complete(
        get_pages_for_description(movies_urls)
    )
    for movie in movies_html:
        soup = get_soup(movie[1])
        description = soup.find(
            'div',
            attrs={'class': 'ipc-html-content ipc-html-content--base'}
        )
        if description:
            descriptions.append(description.text)
    return descriptions

def main(urls,
         num_of_actors_limit=None,
         num_of_movies_limit=None,
         num_of_depth_level_limit=None):
    all_pairs = itertools.permutations(urls, 2)
    for pair in all_pairs:
        distance = get_movie_distance(pair[0],
                                      pair[1],
                                      num_of_actors_limit,
                                      num_of_movies_limit,
                                      num_of_depth_level_limit)
        print(f'from: {pair[0]}, '
              f'to: {pair[1]}, '
              f'distance = {distance}')

    get_results_csv(urls)

    labels = read_labels_from_checkpoint()

    loop = asyncio.get_event_loop()
    urls_actors_pages = loop.run_until_complete(get_all_pages(urls))
    for actor in urls_actors_pages:
        actor_name = labels[actor[0]]
        soup = get_soup(actor[1])
        descriptions = get_movie_descriptions_by_actor_soup(soup)
        with open(f'{actor_name}.txt', 'w', encoding='utf-8') as file:
            for el in descriptions:
                file.write(f'{el}\n')

if __name__ == "__main__":
    urls = ['https://www.imdb.com/name/nm0425005/',
            'https://www.imdb.com/name/nm1165110/',
            'https://www.imdb.com/name/nm0000375/',
            'https://www.imdb.com/name/nm0474774/',
            'https://www.imdb.com/name/nm0000329/',
            'https://www.imdb.com/name/nm0177896/',
            'https://www.imdb.com/name/nm0001191/',
            'https://www.imdb.com/name/nm0424060/',
            'https://www.imdb.com/name/nm0005527/',
            'https://www.imdb.com/name/nm0262635/']

    main(urls, ACTORS_SEARCH_LIMIT, MOVIES_SEARCH_LIMIT, MAX_DEPTH_SEARCH)


# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# In[2]:

fig_size = plt.rcParams["figure.figsize"]
fig_size[0] = 10
fig_size[1] = 5
plt.style.use('fivethirtyeight')

# In[3]:

df = pd.read_csv('911 calls- Mongomery county/911.csv')
df = df[df['zip'] > 10000]

# In[4]:

# lat: Latitude
# lng: Longitude
# desc: Description of Emergency
# zip: ZIP Code
# title: Title of Emergency
# time: StampDate and time of the call
# twp: Town
# addr: Address

# In[5]:

print (df.head())

# # Aspects to explore
# 1. How are emergencies ranked by frequency?

#     1.1. What are the top few most frequent emergencies in each type?

#     1.2. Subcategories by count.
#
# 2. Location based exploration

#     2.1. Do specific emergencies happen at specific places more than the others?
#         - a. Plot emergency categories by 1. town,
#                                           2. zip,
#                                           3. Latitude, longitude
#         - b. Under categories, plot each sub category by 1. town,
#                                                          2. zip,
#                                                          3. Latitude, longitude.
#
# 3. Time based exploration

#     3.1. Is there a specific time window when an emergency occurs?
#         - a. By time of day
#             - Plot emergency categories
#                 - Plot emergency subcategories
#         - b. By month
#             - Plot emergency categories
#                 - Plot emergency subcategories
#
# 4. Geo-temporal exploration

#     4.1. Do emergencies show any pattern?
#         - a. At a given time, which place needs most attention? What type of attention is it?

# In[6]:

df.columns = ['latitude','longitude','description','zip','type','time','town','address','if_emergency']

# In[7]:

df.dtypes

# In[8]:

df['time'] = pd.to_datetime(df['time'])
print (df['time'].max())
print (df['time'].min())
print (df['time'].max() - df['time'].min())

# In[9]:

df.count()

# In[10]:

df.head()

# ## 1. General Exploration (1/4)

# ### 1.1. Ranking emergency categories by frequency

# In[11]:

df2= df.groupby('type')

# In[12]:

grouped = df.groupby('type').agg({'type':'count'}).sort_values('type')[::-1][:30]/len(df)*100

# In[13]:

grouped

# In[14]:

xi = grouped.values[:,0]
yi = grouped.index
plt.subplots(figsize=(20,15))
sns.barplot(x=xi,y=yi,color='r')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=2)
plt.xlim=(0,25)
plt.ylabel("Emergency")
plt.xlabel("Percentage of all 911 calls")
plt.title('30 most common reasons behind 911 calls')
plt.show()

# In[15]:

## Categorizing data by emergency type
df_title_cat = pd.DataFrame(df.type.str.split(':',1).tolist(),columns = ['emergency_type','emergency_subtype'])
df_cat = df.copy()
del df_cat['type']
df_cat.insert(4,'emergency_type',df_title_cat['emergency_type'].values)
df_cat.insert(5,'emergency_subtype',df_title_cat['emergency_subtype'].values)

# In[16]:

df_cat.head()

# In[17]:

grouped_by_type = df_cat.groupby('emergency_type').agg(
    {'emergency_type':'count'}).sort_values('emergency_type')[::-1][:10]/len(df_cat)*100

# In[18]:

plt.style.use('fivethirtyeight')

grouped = grouped_by_type
yi = grouped.values[:,0]
xi = grouped.index
plt.subplots(figsize=(3,7))
plt.style.use('fivethirtyeight')

sns.barplot(x=xi,y=yi,color='b')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=1)
plt.xlim=(0,100)
plt.xlabel("")
plt.ylabel('Percentage of Total Number of Calls')
plt.show()
# plt.title('All 911 calls Grouped by Emergency Types')

# In[19]:

grouped_by_emstype = df_cat[df_cat['emergency_type']=='EMS'].groupby('emergency_subtype').agg({
    'emergency_type':'count'}).sort_values('emergency_type')[::-1][:10]/len(df_cat)*100

# In[20]:

grouped = grouped_by_emstype
plt.style.use('fivethirtyeight')

xi = grouped.values[:,0]/sum(grouped.values[:,0])*100
yi = grouped.index
plt.subplots(figsize=(20,7))
sns.barplot(x=xi,y=yi,color='r')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=1.5)
plt.xlim=(0,100)
plt.ylabel("")
plt.xlabel("Percentage of All EMS Calls")
plt.title('EMS Calls Grouped by Emergency Sub-type')
plt.show()

# In[21]:

grouped_by_traffictype = df_cat[df_cat['emergency_type']=='Traffic'].groupby('emergency_subtype').agg({
    'emergency_type':'count'}).sort_values('emergency_type')[::-1][:10]/len(df_cat)*100

# In[22]:

grouped = grouped_by_traffictype
plt.style.use('fivethirtyeight')

xi = grouped.values[:,0]/sum(grouped.values[:,0])*100
yi = grouped.index
plt.subplots(figsize=(20,7))
sns.barplot(x=xi,y=yi,color='r')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=1.5)
plt.xlim=(0,100)
plt.ylabel("")
plt.xlabel("Percentage of All Traffic calls")
plt.title('Traffic Calls Grouped by Emergency Sub-type')
plt.show()

# In[23]:

grouped_by_firetype = df_cat[df_cat['emergency_type']=='Fire'].groupby('emergency_subtype').agg({
    'emergency_type':'count'}).sort_values('emergency_type')[::-1][:10]/len(df_cat)*100

# In[24]:

grouped = grouped_by_firetype
plt.style.use('fivethirtyeight')

xi = grouped.values[:,0]/sum(grouped.values[:,0])*100
yi = grouped.index
plt.subplots(figsize=(20,7))
sns.barplot(x=xi,y=yi,color='r')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=1.5)
plt.xlim=(0,100)
plt.ylabel("")
plt.xlabel("Percentage of All Fire calls")
plt.title('Fire Calls Grouped by Emergency Sub-type')
plt.show()

# ## 2. Location-based Exploration

# ### 2.1. Emergency categories by town

# In[25]:

count_by_town = df_cat[df_cat['emergency_type'] == 'EMS'].groupby('town').agg({'town':'count'}).sort_values('town')[::-1][:20]

# In[26]:

grouped = count_by_town
plt.style.use('fivethirtyeight')

xi = grouped.values[:,0]/1000
yi = grouped.index
plt.subplots(figsize=(20,7))
sns.barplot(x=xi,y=yi,color='r')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=1.5)
plt.xlim=(0,100)
plt.ylabel("")
plt.xlabel("1000 Calls")
plt.title('EMS Calls Grouped by Town')
plt.show()

# In[27]:

count_by_zip = df_cat[df_cat['emergency_type'] == 'EMS'].groupby('zip').agg({'zip':'count'}).sort_values('zip')[::-1].iloc[:20]
count_by_zip.columns = ['count']

# In[28]:

grouped = count_by_zip
plt.style.use('fivethirtyeight')

yi = count_by_zip['count']
xi = count_by_zip.index
plt.subplots(figsize=(20,7))
sns.barplot(x=xi,y=yi,color='r')

sns.set(style="whitegrid")
sns.set_color_codes("muted")
sns.set(font_scale=2)
sns.despine(left=True, bottom=True)

sns.set(font_scale=1.5)
plt.xlim=(0,100)
plt.ylabel("1000 Calls")
plt.title('EMS Calls Grouped by zip')
plt.show()

# In[29]:

count_by_zip = df_cat.groupby('zip').agg({'zip':'count'}).sort_values('zip')[::-1]
count_by_zip['cumsum_percent'] = count_by_zip.cumsum()/len(df)*100
count_by_zip.columns = ['count','cumsum_percent']

# In[30]:

count_by_zip[count_by_zip['cumsum_percent']<81].count()/len(count_by_zip)
# 80% of all EMS calls originated only from about 25% of zip codes
# 99% of all EMS calls originated only from about 45% of zip codes

# In[31]:

plt.style.use('fivethirtyeight')

plt.plot(range(len(count_by_zip)),count_by_zip['cumsum_percent'])
plt.xlabel('Number of Zip Codes')
plt.ylabel('Percent of all EMS Calls')
plt.axhline(y = 80,xmin = 0,xmax =0.22,color = 'r')
plt.axvline(x = 26,ymax = 0.75,color='r')

plt.show()

# In[32]:

print (str(len(df_cat['zip'].unique()))+' unique zip codes.')
print (str(len(df_cat['town'].unique()))+ ' unique towns.')

# ## Plot all calls on map

#

# ## Plot EMS calls on map

# In[33]:

only_EMS_view = df_cat[df_cat['emergency_type']=='EMS']

# In[34]:

only_EMS_view.to_csv('segregated_data.csv')

# In[35]:

only_EMS_view

# In[36]:

df_cat['zip'].unique()

# In[37]:

us_pop = pd.read_csv('us-population-by-zip-code/population_by_zip_2010.csv')

# In[38]:

us_pop.head()

# In[39]:

us_pop_by_zip = us_pop[us_pop['zipcode'].isin(df_cat['zip'].unique())].groupby('zipcode').agg({'population':'sum'})

# In[40]:

result = pd.concat([us_pop_by_zip, count_by_zip], axis=1, sort=False)

# In[41]:

result.dropna(inplace=True)

# In[42]:

result.sort_values('population').corr()
# As seen, population and EMS calls are very poorly correlated

# ## 3. Analyze EMS Calls according to time

# In[43]:

plt.style.use('fivethirtyeight')
sns.distplot(df_cat[df_cat['emergency_type'] == 'EMS']['time'].dt.hour,bins=24)
plt.ylabel('Inverse of calls per hour')
plt.xlabel('Time of Day')

plt.show()

# In[44]:

plt.style.use('fivethirtyeight')
sns.distplot(df_cat[df_cat['emergency_type'] == 'EMS']['time'].dt.hour[df_cat['time'].dt.month==6],bins=24,label='June')
sns.distplot(df_cat[df_cat['emergency_type'] == 'EMS']['time'].dt.hour[df_cat['time'].dt.month==12],bins=24,label='December')
plt.ylabel('Inverse of calls per hour')
plt.xlabel('Time of Day')

plt.legend()
plt.show()

# In[45]:

plt.style.use('fivethirtyeight')
sns.distplot(df_cat[df_cat['emergency_subtype'] == ' RESPIRATORY EMERGENCY']['time'].dt.hour[df_cat['time'].dt.month==6],bins=24,label='June')
sns.distplot(df_cat[df_cat['emergency_subtype'] == ' RESPIRATORY EMERGENCY']['time'].dt.hour[df_cat['time'].dt.month==12],bins=24,label='December')
plt.ylabel('Inverse of calls per hour')
plt.xlabel('Time of Day')

plt.legend()
plt.show()

# In[46]:

df_cat['date'] = df_cat['time'].dt.date
df_cat.head()

# In[47]:

daily_count = df_cat[df_cat['zip']==19401.0].groupby('date').agg({'if_emergency':'count'}).sort_values('date')
daily_count.head()#['year'] =

# In[48]:

daily_count.describe()

# In[49]:

plt.style.use('fivethirtyeight')
sns.distplot(daily_count,label='Distribution of daily emergencies in 19401')
plt.xlabel('Number of calls')
plt.legend()
plt.show()

# In[50]:

plt.style.use('seaborn')
daily_count.plot()


import http.cookiejar
import urllib.request
# 保存Cookie
filename = 'saved_cookies.txt'
# FileCookieJar、MozillaCookieJar、LWPCookieJar约为保存cookie信息，
# 只是保存格式不同，读者可自行尝试
cookie = http.cookiejar.MozillaCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)


import utils
import gensio
import gensios_enabled

class Logger:
    def gensio_log(self, level, log):
        print("***%s log: %s" % (level, log))

def test_sync_gensio(o):
    print("Testing basic sync I/O")

    gensios_enabled.check_iostr_gensios("echo")
    g = gensio.gensio(o, "echo(readbuf=10)", None)
    g.set_sync()
    g.open_s()

    # Basic write then read
    (count, time) = g.write_s("Hello", 1000)
    if count != 5 or time < 500:
        raise Exception("Invalid write return: %d %d\n" % (count, time))
    (buf, time) = g.read_s(10, 1000)
    buf = buf.decode(encoding='utf8')
    if buf != "Hello" or time < 500:
        raise Exception("Invalid read return: '%s' %d\n" % (buf, time))

    # This should time out, no data
    (buf, time) = g.read_s(10, 250)
    buf = buf.decode(encoding='utf8')
    if buf != "" or time != 0:
        raise Exception("Invalid read timeout return: '%s' %d\n" % (buf, time))

    # This should time out, buffer is only 10 bytes per readbuf above
    (count, time) = g.write_s("HelloHelloHello", 250)
    if count != 10 or time != 0:
        raise Exception("Invalid write timeout return: %d %d\n" % (count, time))

    # Read out what should have been written
    time = 1000
    buf = ""
    while len(buf) < 10 and time >= 500:
        (tbuf, time) = g.read_s(10, time)
        buf = buf + tbuf.decode(encoding='utf8')

    if buf != "HelloHello" or time < 500:
        raise Exception("Invalid read return(2): '%s' %d\n" % (buf, time))

    # This should time out, no data
    (buf, time) = g.read_s(10, 250)
    buf = buf.decode(encoding='utf8')
    if buf != "" or time != 0:
        raise Exception("Invalid read timeout return: '%s' %d\n" % (buf, time))

    g.close_s()
    return

class SyncEvent:
    def __init__(self):
        self.opened = False
        return

    def read_callback(self, io, err, data, auxdata):
        return len(data)

    def write_callback(self, io):
        io.write_callback_enable(false);
        return

    def open_done(self, io, err):
        if err:
            raise Exception("accept_s_timeout open error: " + err);
        self.opened = True
        return

def test_sync_gensio_accepter(o):
    print("Testing sync accept")

    gensios_enabled.check_iostr_gensios("tcp")
    a = gensio.gensio_accepter(o, "tcp,localhost,0", None)
    a.set_sync()
    a.startup()
    port = a.control(gensio.GENSIO_CONTROL_DEPTH_FIRST,
                     gensio.GENSIO_CONTROL_GET,
                     gensio.GENSIO_ACC_CONTROL_LPORT, "0")

    sa = SyncEvent()
    (io, time) = a.accept_s_timeout(o, sa, 1)
    if io != None or time != 0:
        raise Exception("accept_s_timeout didn't time out");

    sg = SyncEvent()
    gensios_enabled.check_iostr_gensios("tcp")
    g = gensio.gensio(o, "tcp,localhost," + port, sg)
    g.open(sg)

    (io, time) = a.accept_s_timeout(o, sa, 1000)
    if io == None or time == 0:
        raise Exception("accept_s_timeout timed out");

    io.close_s()
    g.close_s()
    a.shutdown_s()
    return

import utils

test_sync_gensio(utils.o)
test_sync_gensio_accepter(utils.o)
utils.test_shutdown()


from __future__ import annotations

from unittest import mock

from airflow.providers.amazon.aws.hooks.quicksight import QuickSightHook
from airflow.providers.amazon.aws.operators.quicksight import QuickSightCreateIngestionOperator

DATA_SET_ID = "DemoDataSet"
INGESTION_ID = "DemoDataSet_Ingestion"
AWS_ACCOUNT_ID = "123456789012"
INGESTION_TYPE = "FULL_REFRESH"

MOCK_RESPONSE = {
    "Status": 201,
    "Arn": "arn:aws:quicksight:us-east-1:123456789012:dataset/DemoDataSet/ingestion/DemoDataSet_Ingestion",
    "IngestionId": "DemoDataSet_Ingestion",
    "IngestionStatus": "INITIALIZED",
    "RequestId": "fc1f7eea-1327-41d6-9af7-c12f097ed343",
}

class TestQuickSightCreateIngestionOperator:
    def setup_method(self):
        self.quicksight = QuickSightCreateIngestionOperator(
            task_id="test_quicksight_operator",
            data_set_id=DATA_SET_ID,
            ingestion_id=INGESTION_ID,
        )

    @mock.patch.object(QuickSightHook, "get_conn")
    @mock.patch.object(QuickSightHook, "create_ingestion")
    def test_execute(self, mock_create_ingestion, mock_client):
        mock_create_ingestion.return_value = MOCK_RESPONSE
        self.quicksight.execute(None)
        mock_create_ingestion.assert_called_once_with(
            data_set_id=DATA_SET_ID,
            ingestion_id=INGESTION_ID,
            ingestion_type="FULL_REFRESH",
            wait_for_completion=True,
            check_interval=30,
        )


#TP Proba
import random
from math import *
from matplotlib import pyplot
import numpy

def miseajourMABA(p,A,k):
    l=k
    if (l!=A) and (l!=0):
        pas=2*(random.random()<p)-1
        l=k+pas
    return l

def trajMABA(p,A,k,n):
    traj=[k]
    for i in range(n):
        traj.append(miseajourMABA(p,A,traj[-1]))
    pyplot.plot(range(n+1),traj,color='blue')
    pyplot.show()

def tempsabsMABA(p,A,k):
    traj=[k]
    compteur=0
    while traj[-1] not in [0,A]:
        traj.append(miseajourMABA(p,A,traj[-1]))
        compteur=compteur+1
    absorbeur=traj[-1]
    return absorbeur,compteur

def tempsabsMABA1(p,A,k):
    traj=[k]
    compteur=0
    while traj[-1] not in [0,A]:
        traj.append(miseajourMABA(p,A,traj[-1]))
        compteur=compteur+1
    absorbeur=traj[-1]
    return compteur

def estimMABA(p,A,k,n):#où n est la taille de l'échantillon
    absorb=[]
    temps=[]
    for i in range(n):
        [abs,tps]=tempsabsMABA(p,A,k)
        absorb.append(abs)
        temps.append(tps)
    tpsmoy=sum(temps)/n
    pAapprox=sum(absorb)/(A*n)
    return(pAapprox,tpsmoy)

#ex2
def miseajourMA2BR(p,A,k):
    l=k
    if l==0:
        if(random.random()<p):
            pas=1
        else:
            pas=0
    elif l==A:
        if (random.random()>p):
            pas=-1
        else:
            pas=0
    else:
        pas=2*(random.random()<p)-1
    l=k+pas
    return l


#
# A collection of useful functions when dealing with emulators
#

from __future__ import absolute_import, division
from __future__ import print_function, unicode_literals

import numpy as np
from GPy import kern

def fix_parameters(bounds):
    """
    Returns lists consisting of (parameter index, value) tuples.
    Use for visualisations in situations where there are more than 2
    inputs to a model
    """
    n_parameters = bounds.n_parameters()
    fixed_parameters = []
    for i in range(n_parameters):
        for j in range(n_parameters):
            if i == j:
                continue
            mid_vals = enumerate((bounds.lower()+bounds.upper())/2)
            mid_vals = list(mid_vals)
            mid_vals = [(idx, val) for (idx, val) in mid_vals
                        if idx not in [i, j]]

            fixed_parameters.append(mid_vals)

    return fixed_parameters

def generate_grid(lower, upper, splits, fixed=[]):
    """
    Generates a grid of evenly spaced out points for testing
    returns grid of the first paramater, grid of second parameter,
    and their values stacked with any fixed values provided
    Generated values are convenient for plotting surface
    or contour plots.
    fixed -- contains position and argument to keep fixed
    """

    if not fixed:
        p1_low, p2_low = lower
        p1_high, p2_high = upper
    else:
        # find out which parameters are not fixed and get their bounds
        n_params = len(lower)

        p1_idx, p2_idx = [i for i in range(n_params)
                          if i not in [j for (j, _) in fixed]]

        p1_low, p2_low = lower[p1_idx], lower[p2_idx]
        p1_high, p2_high = upper[p1_idx], upper[p2_idx]

    p1_range = np.linspace(p1_low, p1_high, splits)
    p2_range = np.linspace(p2_low, p2_high, splits)
    p1_grid, p2_grid = np.meshgrid(p1_range, p2_range)

    if fixed:
        # create a grid for every parameter and insert in
        # corresponding position in the grids array
        grids = [None] * n_params
        grids[p1_idx] = p1_grid
        grids[p2_idx] = p2_grid
        for (i, val) in fixed:
            fixed_grid = np.zeros((splits, splits)) + val
            grids[i] = fixed_grid
    else:
        grids = [p1_grid, p2_grid]

    grid = np.dstack(tuple(grids))

    return p1_grid, p2_grid, grid

def predict_grid(model, grid, dims=None):
    """
    Given a PDF and a grid of inputs calculates probability for
    each index in the grid
    """
    rows, cols, n_params = grid.shape
    flatten_grid = grid.reshape((rows * cols, n_params))
    pred = np.apply_along_axis(model, 1, flatten_grid)
    return pred.reshape(rows, cols)

# Functions to deal with composite kernels
def is_prod_kernel(kernel):
    """
    True when a given kernel is a GPy Prod kernel
    """
    return type(kernel) == kern.src.prod.Prod

def is_add_kernel(kernel):
    """
    True when a given kernel is a GPy Add kernel
    """
    return type(kernel) == kern.src.add.Add

def kernel_to_string(kernel, ident=0, decimal_places=4):
    """
    Converts complex GPy kernels to strings
    TODO: rewrite with .format
    """
    if kernel is None:
        return ""
    s = ""
    formatting = "{:." + str(decimal_places) + "f}"
    tab = ident * " "
    if is_prod_kernel(kernel) or is_add_kernel(kernel):
        op = "*" if is_prod_kernel(kernel) else "+"
        sub_kernels = []
        for sub_kernel in kernel.parameters:
            sub_kernels.append(kernel_to_string(sub_kernel, ident=ident + 1))
        s = "(" + op + "\n" + "\n".join(sub_kernels) + "\n" + tab + ")"
    else:
        # get name of kernel without "'>" characters
        name = str(type(kernel)).split(".")[-1]
        name = name[:-2]

        values = ",".join([formatting.format(x) for x in kernel])
        s = name + "(" + values + ")"
    return " " * ident + s

def get_total_variance(kernel):
    ans = 0
    if is_prod_kernel(kernel) or is_add_kernel(kernel):
        for sub_kernel in kernel.parameters:
            ans += get_total_variance(sub_kernel)
    else:
        if hasattr(kernel, "variance"):
            ans += kernel.variance
        elif hasattr(kernel, "variances"):
            ans += kernel.variances
        else:
            ans += 0

    return ans

def has_high_variance(kernel, threshold=10):
    variance = 0
    if is_prod_kernel(kernel) or is_add_kernel(kernel):
        for sub_kernel in kernel.parameters:
            if has_high_variance(sub_kernel):
                return True
    else:
        if hasattr(kernel, "variances"):
            variance = kernel.variances
        elif hasattr(kernel, "variance"):
            variance = kernel.variance
        else:
            # some kernels don't have variance as a parameter
            return False

    return variance > threshold

def simulate(
    model,
    parameters=None,
    times=None,
    noise_range_percent=0.05,
    n_splits=None
):
    """
    Simulates model for specified time interval with specified noise.
    Noise is normal with standart deviation as: range * noise_range_percent.
    Pass noise_range_percent=None if no noise wanted
    Returns values, times, noise_stds
    If n_splits is provided divide time interval into n_splits uniform parts.
    """

    if parameters is None:
        parameters = np.array(model.suggested_parameters())

    if times is None:
        times = model.suggested_times()

    # take times and calculate values
    if n_splits:
        min_time, max_time = min(times), max(times)
        times = np.linspace(min_time, max_time, n_splits)

    # simulate
    values = model.simulate(parameters, times)

    # noise
    # by default set 5% of range as the standard deviation
    if noise_range_percent is not None:
        noise_stds = np.abs(values.max(axis=0) - values.min(axis=0)) * noise_range_percent

        # final values
        values = values + np.random.normal(0, noise_stds, values.shape)

        return values, times, noise_stds

    return values, times


""" docstring """
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.status import *
from rest_framework.views import APIView

from AudioApp.models import *
from .serializers import *

# Create your views here.
# @csrf_exempt
class AboutUsAPI(APIView):
    """
    API for Retreiving AboutUs of company
    """
    permission_classes = (IsAuthenticated,)

    def post(self, request):
        """ docstring """
        user = request.user.email
        try:
            employee = Employess.objects.get(employee_email=user)
            about_us = AboutUs.objects.get(company=employee.company)
            serializer = AboutUsSerializer(about_us)
            return Response({"message": "AboutUs Retrieved Successfully!!", "data": serializer.data},
                            status=HTTP_200_OK)
        except:
            return Response({"message": "AboutUs Not Found!!"}, status=HTTP_404_NOT_FOUND)

class TermsServicesAPI(APIView):
    """
    API for Retreiving TermsServices of company
    """
    permission_classes = (IsAuthenticated,)

    def post(self, request):
        """ docstring """
        user = request.user.email
        try:
            employee = Employess.objects.get(employee_email=user)
            term_services = TermServices.objects.get(company=employee.company)
            serializer = TermServicesSerializer(term_services)
            return Response({"message": "Terms and Services Retrieved Successfully!!", "data": serializer.data},
                            status=HTTP_200_OK)
        except:
            return Response({"message": "Terms and Services Not Found!!"}, status=HTTP_404_NOT_FOUND)

class PrivacyPolicyAPI(APIView):
    """
    API for Retreiving PrivacyPolicy of company
    """
    permission_classes = (IsAuthenticated,)

    def post(self, request):
        """ docstring """
        user = request.user.email
        try:
            employee = Employess.objects.get(employee_email=user)
            privacy_policy = PrivacyPolicy.objects.get(company=employee.company)
            serializer = PrivacyPolicySerializer(privacy_policy)
            return Response({"message": "Privacy Policy Retrieved Successfully!!", "data": serializer.data},
                            status=HTTP_200_OK)
        except:
            return Response({"message": "Privacy Policy not Found!!"}, status=HTTP_404_NOT_FOUND)