import numpy as np

import sys
import os

sys.path.append(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))

import utils.utils as utils
from apkit.apkit.doa import load_pts_horizontal

# class Angle:
#     def __init__(self, degree, depth, fs=24000):
#         self.fs = fs
#         self.depth = depth
#         self.steer_angle = degree   # Degree: 0 ~ 360
#         self.steer_range = []  # Consists of two end points
#         self.child_indice = []
#         self.TDOA_range = []
#
#         angle_window = 90
#         for idx_depth in range(depth):
#             angle_window = angle_window / 2.0
#         half_angle_window = angle_window / 2.0
#         min_steer_range = self.steer_angle - half_angle_window if self.steer_angle - half_angle_window > 0 else 0
#         max_steer_range = self.steer_angle + half_angle_window if self.steer_angle + half_angle_window < 360 else 360
#         self.steer_range.append(min_steer_range)
#         self.steer_range.append(max_steer_range)
#
#     def set_child(self, input_idx):
#         if input_idx not in self.child_indice:
#             self.child_indice.append(input_idx)
#
#     def get_coord(self):
#         return self.deg2coord(self.steer_angle)
#
#     def deg2rad(self, degree):
#         return degree * np.pi / 180.0
#
#     def deg2coord(self, degree):
#         rad = self.deg2rad(degree)
#         return np.array([np.cos(rad), np.sin(rad), 0])
#
#     def compute_TDOA_range(self, mic_pair_pos, half_delay_len):
#         # Extract neighbor vertices
#         range_angle_list = [self.deg2coord(range_angle) for range_angle in self.steer_range]
#
#         # Compute ranges of TDOAs of neighbors
#         range_angle_diff_list = utils.utils.compute_diff_TDOA(range_angle_list, np.array([0, 0, 0]),
#                                                                         mic_pair_pos, half_delay_len,
#                                                               sampling_rate=self.fs)
#
#         self.TDOA_range = np.stack((np.min(range_angle_diff_list, axis=0), np.max(range_angle_diff_list, axis=0)),
#                                    axis=-1).tolist()
#         # self.TDOA_range = np.stack((np.min(range_angle_diff_list, axis=0)-1, np.max(range_angle_diff_list, axis=0)+1),
#         #                            axis=-1).tolist()

class CircleGrid:
    # def __init__(self, subdiv, fs=24000, DOA_likelihood_threshold=0.6):
    def __init__(self, fs=24000, z_angle=0):
        self.fs = fs
        # self.DOA_likelihood_threshold = DOA_likelihood_threshold

        #######################################
        # Version 1
        # init_steer_angles = [45, 135, 225, 315] # Depth 0
        #
        # init_angles = []
        # for tmp_angle in init_steer_angles:
        #     init_angles.append(Angle(tmp_angle, 0, fs=self.fs))
        #
        # # Initialize the empty lists of angles for each depth
        # self.steer_angle_list = [[] for i in range(subdiv + 1)]
        # self.steer_angle_list[0] = init_angles
        #
        # # Perform dividing each angle ranges into two angle ranges
        # self.initialize_recursive_circle(subdiv)

        #######################################
        # Version 2
        self.pts_horizontal = load_pts_horizontal()

        self.pts_horizontal[:, 2] = np.sqrt(self.pts_horizontal[:, 0]*self.pts_horizontal[:, 0] + self.pts_horizontal[:, 1]*self.pts_horizontal[:, 1]) * np.tan(z_angle*np.pi/180)
        self.pts_horizontal[:, 0] = self.pts_horizontal[:, 0] / np.linalg.norm(self.pts_horizontal, axis=-1)
        self.pts_horizontal[:, 1] = self.pts_horizontal[:, 1] / np.linalg.norm(self.pts_horizontal, axis=-1)
        self.pts_horizontal[:, 2] = self.pts_horizontal[:, 2] / np.linalg.norm(self.pts_horizontal, axis=-1)

        self.sigma_gauss_dist = 1.5
        self.TDOA_filter = None
        # print("Debugging")

    # def estimate_DOA(self, tdoa_pred):
    # def estimate_DOA(self, tdoa_pred, gt_doas): # for testing
    #     # E.g., [128, 6, 10, 21] --> The shape of GT DOAs: [128, 10, 9 (DOAs) + 1]
    #     batch_size, num_mic_pairs, num_frames, delay_len = tdoa_pred.shape
    #
    #     # The output: DOA predictions (Shape: 128, 10, 9 + 1)
    #     doas_pred = np.zeros((batch_size, num_frames, 10))
    #
    #     # For all batches
    #     for batch_idx in range(batch_size):
    #         # For all frames
    #         for frame_idx in range(num_frames):
    #             # Empty list of estimated DOAs
    #             estimated_DOAs = []
    #             computed_DOAs_list = []
    #             # estimated_DOAs_coord = []
    #
    #             tmp_tdoa_pred = tdoa_pred[batch_idx, :, frame_idx, :]
    #
    #             # For the deepest depth
    #             steer_angles = self.steer_angle_list[-1]
    #
    #             # For all angles
    #             for idx_angle, tmp_angle in enumerate(steer_angles):
    #                 # Extract the TDOA range of the steering angle
    #                 angle_TDOA_range = tmp_angle.TDOA_range
    #
    #                 # Compute the DOA likelihood
    #                 DOA_likelihood_list = []
    #                 for idx_pair, (range_min, range_max) in enumerate(angle_TDOA_range):
    #                     tmp_TDOA_likelihoold = np.max(tmp_tdoa_pred[idx_pair, range_min: range_max + 1], axis=-1)
    #                     DOA_likelihood_list.append(tmp_TDOA_likelihoold)
    #                 # Compute the mean of the DOA likelihood of all mic's pairs
    #                 avg_DOA_likelihood = np.mean(np.array(DOA_likelihood_list), axis=0)
    #
    #                 computed_DOAs_list.append((avg_DOA_likelihood, idx_angle, tmp_angle))
    #                 # If the DOA likelihood is larger than the DOA likelihood,
    #                 if avg_DOA_likelihood > self.DOA_likelihood_threshold:
    #                     estimated_DOAs.append((avg_DOA_likelihood, idx_angle, tmp_angle))
    #                     # estimated_DOAs_coord.append(tmp_angle.get_coord())
    #
    #             # Consider a single source, 20220608 IK
    #             max_DOAs = max(computed_DOAs_list, key= lambda DOAs: DOAs[0])
    #             if max_DOAs[0] > self.DOA_likelihood_threshold:
    #                 tmp_angle = max_DOAs[2]
    #                 estimated_DOA = tmp_angle.get_coord()
    #                 idx_DOA = 0
    #                 doas_pred[batch_idx, frame_idx, 0] = 1
    #                 doas_pred[batch_idx, frame_idx, 1 + idx_DOA * 3:1 + idx_DOA * 3 + 3] = estimated_DOA
    #
    #             # Consider multiple sources, 20220608 IK
    #             # # If there exist esimated DOAs,
    #             # if len(estimated_DOAs) > 0:
    #             #
    #             #     # Check the continuous between the zero and last angle
    #             #     if estimated_DOAs[0][1] == 0:
    #             #         TDOA_likelihood, idx_angle, tmp_angle = estimated_DOAs.pop(0)
    #             #         estimated_DOAs.append((TDOA_likelihood, idx_angle, tmp_angle))
    #             #         while idx_angle + 1 == estimated_DOAs[0][1]:
    #             #             TDOA_likelihood, idx_angle, tmp_angle = estimated_DOAs.pop(0)
    #             #             estimated_DOAs.append((TDOA_likelihood, idx_angle, tmp_angle))
    #             #
    #             #     # Grouping the neighbors of estimated DOAs
    #             #     # Grouped DOAs become the final DOAs
    #             #     starting_flg = True
    #             #     prev_idx = 0
    #             #     estimated_DOAs_group_list = []  # It contains the grouped DOAs
    #             #     estimated_DOAs_group = []
    #             #     DOA_likelihood_list = []
    #             #     for idx_DOA, (DOA_likelihood, idx_angle, tmp_angle) in enumerate(estimated_DOAs):
    #             #         if starting_flg:
    #             #             prev_idx = idx_angle
    #             #             estimated_DOAs_group.clear()
    #             #             estimated_DOAs_group.append(tmp_angle.get_coord())
    #             #             DOA_likelihood_list.append(DOA_likelihood)
    #             #             starting_flg = False
    #             #         else:
    #             #             if prev_idx + 1 == idx_angle:
    #             #                 prev_idx = idx_angle
    #             #                 estimated_DOAs_group.append(tmp_angle.get_coord())
    #             #                 DOA_likelihood_list.append(DOA_likelihood)
    #             #             elif prev_idx == len(steer_angles) - 1 and idx_angle == 0:
    #             #                 prev_idx = idx_angle
    #             #                 estimated_DOAs_group.append(tmp_angle.get_coord())
    #             #                 DOA_likelihood_list.append(DOA_likelihood)
    #             #             else:
    #             #                 starting_flg = True
    #             #                 estimated_DOAs_group_list.append((
    #             #                     np.mean(np.array(estimated_DOAs_group), axis=0),
    #             #                     np.mean(np.array(DOA_likelihood_list))
    #             #                 ))
    #             #
    #             #         if idx_DOA == len(estimated_DOAs) - 1 and len(estimated_DOAs_group) > 0:
    #             #             # estimated_DOAs_group_list.append(np.mean(np.array(estimated_DOAs_group), axis=0))
    #             #             estimated_DOAs_group_list.append((
    #             #                 np.mean(np.array(estimated_DOAs_group), axis=0),
    #             #                 np.mean(np.array(DOA_likelihood_list))
    #             #             ))
    #             #
    #             #     # Set the final output ...
    #             #     if len(estimated_DOAs_group_list) > 0:
    #             #         # if len(estimated_DOAs_group_list) > 1:
    #             #         #     print("Check point")
    #             #
    #             #         estimated_DOAs_group_list.sort(key=lambda tmp_DOA_info: tmp_DOA_info[1], reverse=True)
    #             #
    #             #         doas_pred[batch_idx, frame_idx, 0] = len(estimated_DOAs_group_list)
    #             #         for idx_DOA, (estimated_DOA, avg_DOA_likelihood) in enumerate(estimated_DOAs_group_list):
    #             #             if idx_DOA < 3:
    #             #                 doas_pred[batch_idx, frame_idx, 1+idx_DOA*3:1+idx_DOA*3+3] = estimated_DOA
    #     return doas_pred

    # 20220527, backup
    # def estimate_DOA(self, tdoa_pred):
    #     # E.g., [128, 6, 10, 21] --> The shape of GT DOAs: [128, 9 (DOAs), 10]
    #     batch_size, num_mic_pairs, num_frames, delay_len = tdoa_pred.shape
    #
    #     for batch_idx in range(batch_size):
    #         for frame_idx in range(num_frames):
    #             estimated_DOAs = []
    #
    #             tmp_tdoa_pred = tdoa_pred[batch_idx, :, frame_idx, :]
    #             # For depth 0
    #             steer_angles = self.steer_angle_list[0]
    #
    #             # avg_TDOA_likelihood_list = []
    #             for tmp_angle in steer_angles:
    #                 angle_TDOA_range = tmp_angle.TDOA_range
    #
    #                 TDOA_likelihood_list = []
    #                 for idx_pair, (range_min, range_max) in enumerate(angle_TDOA_range):
    #                     tmp_TDOA_likelihoold = np.max(tmp_tdoa_pred[idx_pair, range_min: range_max + 1], axis=-1)
    #                     TDOA_likelihood_list.append(tmp_TDOA_likelihoold)
    #                 avg_TDOA_likelihood = np.mean(np.array(TDOA_likelihood_list), axis=0)
    #                 # avg_TDOA_likelihood_list.append(avg_TDOA_likelihood)
    #
    #                 if avg_TDOA_likelihood > self.DOA_likelihood_threshold:
    #                     child_angle_list = []
    #                     for child_index in tmp_angle.child_indice:
    #                         child_angle_list.append(self.steer_angle_list[tmp_angle.depth+1][child_index])
    #
    #                     while len(child_angle_list) > 0:
    #                         tmp_child_angle = child_angle_list.pop()
    #
    #                         child_angle_TDOA_range = tmp_child_angle.TDOA_range
    #
    #                         child_TDOA_likelihood_list = []
    #                         for idx_pair, (range_min, range_max) in enumerate(child_angle_TDOA_range):
    #                             tmp_child_TDOA_likelihoold = np.max(tmp_tdoa_pred[idx_pair, range_min: range_max + 1],
    #                                                           axis=-1)
    #                             child_TDOA_likelihood_list.append(tmp_child_TDOA_likelihoold)
    #                         avg_child_TDOA_likelihood = np.mean(np.array(child_TDOA_likelihood_list), axis=0)
    #
    #
    #                         if avg_child_TDOA_likelihood <= self.DOA_likelihood_threshold or tmp_child_angle.depth + 1 == len(
    #                                 self.steer_angle_list):
    #                             estimated_DOAs.append((tmp_child_angle, avg_child_TDOA_likelihood))
    #                         else:
    #                             for child_index in tmp_child_angle.child_indice:
    #                                 child_angle_list.append(self.steer_angle_list[tmp_child_angle.depth+1][child_index])
    #
    #
    #             print("Debugging")

        # # For all depths
        # for steer_angles in self.steer_angle_list:
        #
        #     avg_TDOA_list = []
        #     for tmp_angle in steer_angles:
        #         angle_TDOA_range = tmp_angle.TDOA_range
        #
        #         TDOA_list = []
        #         for idx_pair, (range_min, range_max) in enumerate(angle_TDOA_range):
        #             tmp_TDOA = np.max(tdoa_pred[:, idx_pair, :, range_min: range_max + 1], axis=-1)
        #             TDOA_list.append(tmp_TDOA)
        #         avg_TDOA = np.mean(np.array(TDOA_list), axis=0)
        #         avg_TDOA_list.append(avg_TDOA)
        #
        #     print("Debugging")

        ############ TODO: Implement the estimating part of DOAs; IK, 20220524

    def estimate_DOA_v2(self, tdoa_pred): # for testing
        # E.g., [128, 6, 10, 21] --> The shape of GT DOAs: [128, 10, 9 (DOAs) + 1]
        batch_size, num_mic_pairs, num_frames, delay_len = tdoa_pred.shape

        num_angles = len(self.pts_horizontal)

        # The output: DOA predictions (Shape: 128, 10, 9 + 1)
        # doas_pred = np.zeros((batch_size, num_frames, 10))
        DOA_pred_function = np.zeros((batch_size, num_frames, num_angles))

        # For all batches
        for batch_idx in range(batch_size):
            # For all frames
            for frame_idx in range(num_frames):
                # Empty list of estimated DOAs
                estimated_DOAs = []
                computed_DOAs_list = []
                # estimated_DOAs_coord = []

                tmp_tdoa_pred = tdoa_pred[batch_idx, :, frame_idx, :]

                # tmp_DOA_likelihood_function = np.zeros((num_angles))
                # For all angles
                for angle_idx in range(num_angles):
                    steer_angle = self.pts_horizontal[angle_idx]
                    steer_TDOA_filter = self.TDOA_filter[angle_idx]

                    DOA_pred_function[batch_idx, frame_idx, angle_idx] = np.sum(steer_TDOA_filter * tmp_tdoa_pred) / num_mic_pairs
                    # tmp_DOA_likelihood_function[angle_idx] = np.sum(steer_TDOA_filter * tmp_tdoa_pred) / num_mic_pairs
                # print("Debugging")

        return DOA_pred_function

    # def initialize_TDOA_ranges(self, mic_pair_pos, half_delay_len):
    #     # For each depth
    #     for angles in self.steer_angle_list:
    #         for angle in angles:
    #             angle.compute_TDOA_range(mic_pair_pos, half_delay_len)

    def initialize_TDOA_filter(self, mic_pair_pos, half_delay_len):
        delay_len = 2 * half_delay_len + 1
        num_mic_pairs = len(mic_pair_pos)
        num_angles = len(self.pts_horizontal)

        TDOA_filter = np.zeros((num_angles, num_mic_pairs, delay_len))
        TDOA_delay_dummy = np.zeros((num_mic_pairs, delay_len))
        for idx_delay in range(delay_len):
            TDOA_delay_dummy[:, idx_delay] = idx_delay

        for angle_idx in range(num_angles):
            steer_angle = self.pts_horizontal[angle_idx]
            diff_TDOA_pair = utils.compute_diff_TDOA_float(np.array([steer_angle]), np.array([0, 0, 0]),
                                                                 mic_pair_pos, half_delay_len,
                                                                sampling_rate=self.fs)

            diff_TDOA_pair_dummy = np.tile(np.transpose(diff_TDOA_pair), (1, delay_len))
            diff_TDOA_pair_dummy_dist = diff_TDOA_pair_dummy - TDOA_delay_dummy

            gauss_filter = np.exp(-diff_TDOA_pair_dummy_dist * diff_TDOA_pair_dummy_dist / self.sigma_gauss_dist)

            TDOA_filter[angle_idx] = gauss_filter

        self.TDOA_filter = TDOA_filter

    # def initialize_recursive_circle(self, subdiv):
    #     # For all depth ...
    #     for i in range(subdiv):
    #         # Initialize vertice for the next depth
    #         angles = []
    #         chil_idx = 0
    #         for tmp_angle in self.steer_angle_list[i]:
    #             steer_angle = tmp_angle.steer_angle
    #             for range_angle in tmp_angle.steer_range:
    #                 middle_angle = (steer_angle + range_angle) / 2.0    # Between Steering angle and Range angle
    #                 angles.append(Angle(middle_angle, i + 1, fs=self.fs))
    #                 tmp_angle.set_child(chil_idx)
    #                 chil_idx = chil_idx + 1
    #
    #         # Set the computed angles to the next depth
    #         self.steer_angle_list[i+1] = angles


import calendar
import time

from django.db.models import Q
import requests

from models import Song
import spotifyviews

__author__ = 'Ian'

class Spotify:
    """
    Class to handle maintaining the Spotify Client Credentials code to increase query request speed
    """

    def __init__(self):
        self.TOKEN_EXPIRES = calendar.timegm(time.gmtime())
        self.TOKEN = None

    def spotify_auth(self):
        """
        Gets the Client Credentials for Steazy
        :return: Spotify access token
        """
        header = spotifyviews.HEADERS
        body = {'grant_type': 'client_credentials'}

        client_request = requests.post(spotifyviews.SPOTIFYURL, data=body, headers=header)

        response = client_request.json()

        self.TOKEN_EXPIRES = self.TOKEN_EXPIRES + response['expires_in']

        return response['access_token']

    def search_spotify(self, argument):
        """
        Searches Spotify with Client Credentials
        :param argument: String to search Spotify
        :return: JSON response of tracks from Spotify. Described at https://developer.spotify.com/web-api/search-item/
        """
        if calendar.timegm(time.gmtime()) > self.TOKEN_EXPIRES:
            self.TOKEN = self.spotify_auth()
        header = {'Authorization': 'Bearer ' + self.TOKEN}
        argument = argument.replace(' ', '+')
        data = requests.get('https://api.spotify.com/v1/search?q=' + argument + '%20&type=track', headers=header)
        return data.json()

    @staticmethod
    def parse_spotify(data):
        """
        Parse JSON of list of Spotify track search results and return a list of Steazy songs associated with them.
        Doesn't create new songs if they already exist in the database
        :param data: JSON list of Spotify track results. Described at https://developer.spotify.com/web-api/search-item/
        :return:
        """
        songs = []
        for track in data:
            song = check_if_exists({'source': 'Spotify', 'tag': track['id']})
            if song is not None:
                songs.append(song)
            else:
                artists = ''
                for art in track['artists']:
                    artists += unicode(art['name']) + ', '
                artists = artists[:len(artists) - 2]
                song = Song(name=unicode(track['name']), artist=artists, album=unicode(track['album']['name']),
                            source='Spotify', tag=track['id'], inherited_popularity=track['popularity'])
                song.save()
                songs.append(song)
        return songs

SPOTIFY = Spotify()

def check_if_exists(song_data):
    """
    Returns a song if it already exists in the database
    :param song_data: Dictionary containing song's source and tag
    :return: Song object if song exists, None otherwise
    """
    try:
        return Song.objects.get(source=song_data['source'], tag=song_data['tag'])
    except Song.DoesNotExist:
        return None

def sort_songs(songs):
    """
    Returns songs sorted in order of inherited popularity.
    To be improved once more data has been collected
    :param songs: List of Song objects
    :return: songs in order of descending popularity
    """
    return sorted(songs, key=lambda song: song.inherited_popularity, reverse=True)

def search_songs(query):
    """
    Returns a sorted list of songs from Spotify and Soundcloud
    :param query: String keyword(s) to search
    :return: List of Songs matching query
    """
    return sort_songs(Spotify.parse_spotify(SPOTIFY.search_spotify(query)['tracks']['items']) +
                      parse_soundcloud(search_soundcloud(query)))

def search_database(query):
    """
    Queries the local database to find any songs that match query
    :param query: String keyword(s) to search
    :return: List of Songs matching query
    """

    # TODO: Improve the search (ability to search multiple fields at once)
    return sort_songs(Song.objects.filter(Q(name__icontains=query) |
                                          Q(artist__icontains=query) | Q(album__icontains=query)))[0:20]

def search_soundcloud(argument):
    """
    Searches Soundcloud and returns a JSON result of songs
    :param argument: String keyword(s) to search
    :return: JSON result of search
    """
    # create a client object with your app credentials
    params = {'client_id': '81ca87317b91e4051f6d8797e5cce358',
              'q': argument, 'filter': 'public'}
    tracks = requests.get("https://api.soundcloud.com/tracks", params=params)
    return tracks.json()[:10]

def parse_soundcloud(data):
    """
    Turns a JSON response from Soundcloud into a list of Songs
    :param data: JSON response of tracks from Soundcloud
    :return: List of Songs from search
    """
    songs = []
    for track in data:
        if track['streamable']:
            song = check_if_exists({'source': 'Soundcloud', 'tag': track['id']})
            if song is not None:
                songs.append(song)
            else:
                song = Song(name=unicode(track['title']), artist=unicode(track['user']['username']), album='',
                            source='Soundcloud', tag=track['id'],
                            inherited_popularity=track['playback_count'] / float(200000))
                song.save()
                songs.append(song)
    return songs


import os
import bs4 as bs
from urllib.request import urlopen
from flask import Flask
from models import *

app = Flask(__name__)

# Configure the sqlalchemy url to get the url
# from heroku if exsit or get the currency.db
DATABASE_URL = os.environ.get('DATABASE_URL', 'sqlite:///currency.db')
app.config['SQLALCHEMY_DATABASE_URI'] = DATABASE_URL
# Disable flask_sqlalchemy event system to save memory
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

# Initialize the app
db.init_app(app)

# Using BeautifulSoup parse the xml file after open and read the url
# Get all the 'cube' tags and load time, currency and rate to DB
def scrap():
    """Function to get the data from the site and loads it to DB"""
    source = urlopen('https://www.ecb.europa.eu/stats/eurofxref/eurofxref-hist-90d.xml').read()
    soup = bs.BeautifulSoup(source, 'lxml')
    cube = soup.find('cube').find_all('cube')

    for i in cube:
        if i.has_attr('time'):
            time = i['time']
        if i.has_attr('currency'):
            newCurrency = CurrencyRate(time=time,
                                       currency=i['currency'],
                                       rate=i['rate']
                                       )
            db.session.add(newCurrency)
            db.session.commit()

# create DB and table then load the data into it
def main():
    db.create_all()
    scrap()

if __name__ == "__main__":
    with app.app_context():
        main()


from django.shortcuts import render

from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django.core import serializers
import json

kcuser = None
kcroles = None
kctoken = None

def load_user_roles(user, roles):
    global kcuser
    kcuser = user
    global kcroles
    kcroles = roles

def update_user_data(user, token):
    global kctoken
    kctoken = token

def index(request):
    out = 'You\'re at the index. Secure'
    return HttpResponse(out)

@login_required
def secure(request):
    global kcuser
    global kcroles
    out='This Page is Secured
'
    if kcuser is not None:
        out = out + 'username: ' + kcuser.username + '
'
        if kcroles is not None:
            out = out + 'roles: '+",".join(kcroles) + '
'
    if kctoken is not None:
        out = out + 'token data: '+json.dumps(kctoken) + '
'
    out = out + 'Logout '
    return HttpResponse(out)


#!/usr/bin/env python3

import pytest
from importlib import import_module
from utils import msa_from_str, str_from_msa

from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import IUPAC

@pytest.mark.parametrize(
        'input, expected', [
            ([], []),

            ([0], []),
            ([1], [(0, 1)]),

            ([0, 0], []),
            ([0, 1], [(1, 2)]),
            ([1, 0], [(0, 1)]),
            ([1, 1], [(0, 2)]),

            ([0, 0, 0, 0], []),
            ([1, 1, 1, 1], [(0, 4)]),
            ([1, 1, 0, 0], [(0, 2)]),
            ([0, 1, 1, 0], [(1, 3)]),
            ([0, 0, 1, 1], [(2, 4)]),
            ([1, 0, 0, 1], [(0, 1), (3, 4)]),
    ]
)
def test_04_find_runs(input, expected):
    mod = import_module('minp.commands.04_choose_deletions_via_thresholds')
    assert list(mod.find_runs(input)) == expected

@pytest.mark.parametrize(
        'seqs_in, seqs_out', [(
            [],
            [],
        ), (
            [('1', 'MA')],
            [('1', 'MA')],
        ), (
            [('1', 'MA'), ('1', 'MA')],
            [('1', 'MA')],
        ), (
            [('1', 'MA'), ('2', 'MC')],
            [('1', 'MA'), ('2', 'MC')],
        )]
)
def test_03_remove_duplicates(seqs_in, seqs_out):
    mod = import_module('minp.commands.03_build_msa')

    def records_from_seqs(seqs):
        return [
                SeqRecord(
                    Seq(seq, IUPAC.protein),
                    id=id,
                )
                for id, seq in seqs
        ]

    def seqs_from_records(records):
        return [
                (x.id, str(x.seq))
                for x in records
        ]

    records_in = records_from_seqs(seqs_in)
    records_uniq = mod.remove_duplicates(records_in)
    seqs_uniq = seqs_from_records(records_uniq)

    assert seqs_uniq == seqs_out

@pytest.mark.parametrize(
        'ref_id,msa_in_str,msa_out_str', [(
            '1', "MNT", "MNT",
        ), (
            '1', "MNT/MNT", "MNT",
        ), (
            '1', "MNT/M-T", "MNT/M-T",
        ), (
            '1', "M-NT/MA-T", "MNT/M-T",
        )]
)
def test_03_prune_gaps(ref_id, msa_in_str, msa_out_str):
    mod = import_module('minp.commands.03_build_msa')

    msa_in = msa_from_str(msa_in_str)
    msa_pruned = mod.prune_gaps(msa_in, ref_id)
    msa_pruned_str = str_from_msa(msa_pruned)

    assert msa_pruned_str == msa_out_str


import pytest
import os
import sympy as sp

import rolldecay.paper_writing as paper_writing
import rolldecay
import rolldecayestimators.equations
import rolldecayestimators.symbols

paper_path = rolldecay.paper_path

def test_find_tex_files():
    file_paths = paper_writing._find_tex_files(paper_path=paper_path)
    assert len(file_paths) > 0
    file_path = file_paths[0]
    assert '.tex' in file_path

def test_find_eq_label():
    s=r"\label{eq:roll_decay_equation_cubic}"
    eq_labels = paper_writing._find_eq_labels(s=s)
    assert len(eq_labels) > 0
    eq_label=eq_labels[0]
    assert eq_label=='roll_decay_equation_cubic'

def test_find_eq_label_multiline():
    s=r"""

\begin{equation} \label{eq:roll_decay_equation_cubic}
A_{44} \ddot{\phi} + \left(B_{1} + B_{2} \left|{\dot{\phi}}\right| + B_{3} \dot{\phi}^{2}\right) \dot{\phi} + \left(C_{1} + C_{3} \phi^{2} + C_{5} \phi^{4}\right) \phi = 0
\end{equation}
    """
    eq_labels = paper_writing._find_eq_labels(s=s)
    assert len(eq_labels) > 0
    eq_label=eq_labels[0]
    assert eq_label=='roll_decay_equation_cubic'

def test_find_eq_labels():
    s=r"""
    \label{eq:roll_decay_equation_cubic}
    \label{eq:roll_decay_equation_linear}
    """
    eq_labels = paper_writing._find_eq_labels(s=s)
    assert len(eq_labels) ==2
    eq_label=eq_labels[0]
    assert eq_label=='roll_decay_equation_cubic'

    eq_label = eq_labels[1]
    assert eq_label == 'roll_decay_equation_linear'

def test_match_sympy_equations():
    eq_labels = [
        'roll_decay_equation_cubic',
        'roll_decay_equation_linear',
    ]

    equation_dict = paper_writing._match_sympy_equations(eq_labels=eq_labels)
    assert 'roll_decay_equation_cubic' in equation_dict
    assert 'roll_decay_equation_linear' in equation_dict
    assert isinstance(equation_dict['roll_decay_equation_cubic'], sp.Eq)

def test_get_symbols():
    equation_dict={
        'roll_decay_equation_cubic':rolldecayestimators.equations.roll_decay_equation_cubic,
        'roll_decay_equation_linear':rolldecayestimators.equations.roll_decay_equation_linear,
    }
    symbols = paper_writing._get_symbols(equation_dict=equation_dict)

    assert symbols['B_1'] == rolldecayestimators.symbols.B_1
    assert symbols['B_3'] == rolldecayestimators.symbols.B_3

def test_generate_latex_nomenclature():

    symbols = {
        'B_1':rolldecayestimators.symbols.B_1,
        'B_3': rolldecayestimators.symbols.B_3,
    }

    latex_nomenclature = paper_writing._generate_latex_nomenclature(symbols=symbols)
    a=1

def test_generate_nomenclature():

    latex_nomenclature = paper_writing.generate_nomenclature(paper_path=paper_path)
    a = 1


import torch
import json
from torch import nn
from pytorch_pretrained_bert import BertModel
from pytorch_pretrained_bert import BertTokenizer
from keras.preprocessing.sequence import pad_sequences
from pytrends.request import TrendReq
import pandas as pd
import time

class BERT_Arch2(nn.Module):
    def __init__(self, dropout=0.1):
        super(BERT_Arch2, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-uncased')
        self.linear = nn.Linear(768,41)
        self.softmax = nn.LogSoftmax(dim=1)

    #define the forward pass
    def forward(self, sent_id, mask=None):
        #pass the inputs to the model
        _, cls_hs = self.bert(sent_id, attention_mask=mask)
        x = self.linear(cls_hs)
        x = self.softmax(x)
        return x

class Run_BERT:
    def __init__(self):
        self.tmp = ['bts', 'blackpink', 'itzy', 'nct', 'netflix', 'squid_game', 'korean_food', 'korean_street_food', 'north_korea']
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
        self.device = torch.device("cpu")
        self.model = torch.load('./model_v1.pt', map_location=self.device)

        with open("num_to_label.json", 'r') as json_file:
            self.num_to_label = json.load(json_file)

    def text_pipeline(self, text):
        tmp = tuple(['[CLS]'] + self.tokenizer.tokenize(text)[:510] + ['[SEP]'])
        return pad_sequences(list(map(self.tokenizer.convert_tokens_to_ids,[tmp])), maxlen=512, truncating="post", padding="post", dtype="int")

    def text_masks(self, texts):
        return [[float(i > 0) for i in ii] for ii in texts]

    def predict(self, text):
        origin_text = text
        piped = self.text_pipeline(text)
        text = torch.tensor(piped).to(self.device)
        masks = torch.tensor(self.text_masks(piped)).to(self.device)
        output = self.model(text, masks)
        output = output.cpu().detach().numpy()[0]
        for i in self.num_to_label:
            word = self.num_to_label[i]
            if word in origin_text.lower() and word in self.tmp:
                output[int(i)] += 5
            if word not in self.tmp:
                output[int(i)] -= 100
        output[7] -= 3
        return output

    def google_trend(self, topic):
        topic = topic.replace("_", " ")
        print(f"topic is : {topic}")
        kw_list = [topic]
        pytrends = TrendReq(hl="ko", tz=540)
        pytrends.build_payload(kw_list, cat=0, timeframe='today 5-y', geo='KR', gprop='')
        related_topics = pytrends.related_topics()
        # print(related_topics)
        # print("-------------")
        df = pytrends.related_queries()
        df = pd.DataFrame(df[topic]["rising"])
        related_queries = df["query"].values
        print(f"google realted queries : {related_queries}")
        return related_queries

    def run(self, text, num_result=5):
        start = time.time()
        result = self.predict(text)
        result = [self.num_to_label[str(i)] for i in result.argsort()[::-1][:num_result]]
        print(f"BERT_Arch result: {result}")
        result = result[:3]
        try:
            google_trends = list(self.google_trend(result[0]))[:1]
            print(f"google_trend : {google_trends}")
            result[0] = google_trends[0]
        except Exception as e:
            print("Exception message: ", e)
            pass
        print(f"BERT + google: {result}")
        print(f"take time : {time.time() - start}")
        return result


import config
import numpy as np

class Memory:
    vector = [0 for i in range(config.kMaxVectorAddresses)]
    scalar = [0 for i in range(config.kMaxScalarAddresses)]

    @classmethod
    def oneIfPredictIsAccurateElseZero(cls):
        return (
            1
            if np.abs(
                cls.scalar[config.kLabelsScalarAddress]
                - cls.scalar[config.kPredictionsScalarAddress]
            )
            < 0.5
            else 0
        )

    @classmethod
    def wipe(cls):
        cls.vector = [0 for i in range(config.kMaxVectorAddresses)]
        cls.scalar = [0 for i in range(config.kMaxScalarAddresses)]


from __future__ import annotations

import pathlib
import os

import AppKit

from twisted.internet.interfaces import IReactorTime
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure

from quickmacapp import mainpoint, Status, ask, choose, answer, quit

resultTemplate = """
You need to go to the store to get more eggs in {eggDays} days
You need to go to the store to get more milk in {milkDays} days
You will need to bring around ${eggPrice} to get the same amount of eggs
You will need to bring around ${milkPrice} to get the same amount of milk
Thank you
"""

# Prices from reference implementation, possibly accurate circa 2002
eggUnitPrice = 0.09083
milkOuncePrice = 0.05

def alwaysFloat(value: str | None) -> float:
    try:
        return float(value or "nan")
    except:
        return float("nan")

async def eggsAndMilkMinder() -> None:
    eggCount = alwaysFloat(await ask("Enter the number of eggs you have"))
    eatEggCount = alwaysFloat(await ask("Enter the number of eggs you eat per day"))
    milkCount = alwaysFloat(await ask("Enter the number of ounces of milk you have"))
    drinkMilkCount = alwaysFloat(
        await ask("Enter in the amount of milk you drink each day in ounces")
    )
    await answer(
        resultTemplate.format(
            eggDays=(eggCount / eatEggCount),
            milkDays=(milkCount / drinkMilkCount),
            eggPrice=eggCount * eggUnitPrice,
            milkPrice=milkCount * milkOuncePrice,
        )
    )
    hashBrowns = await choose(
        [(True, "y"), (False, "n")],
        "Would you like to also make some delicious hashbrowns?",
    )
    await answer(
        (
            "Then you will need to get some potatoes and grate them,"
            " also some onions and cook it all so it's delicious"
        )
        if hashBrowns
        else (
            "Suit yourself, but hashbrowns are delicious,"
            " you should definitely have them sometime"
        ),
    )

@mainpoint()
def app(reactor: IReactorTime) -> None:
    app = AppKit.NSApplication.sharedApplication()
    app.setActivationPolicy_(AppKit.NSApplicationActivationPolicyAccessory)
    status = Status("ðŸ¥šðŸ¥›")
    status.menu(
        [
            (
                "About",
                lambda: Deferred.fromCoroutine(
                    answer(
                        "Eggs And Milk Minder 1.0c",
                        "With apologies to Roast Beef Kazenzakis",
                    )
                ),
            ),
            (
                "Calculate Eggs And Milk",
                lambda: Deferred.fromCoroutine(eggsAndMilkMinder()),
            ),
            ("Quit", quit),
        ]
    )

if __name__ == "__main__":
    app.runMain()


import cgi
import json
import logging
import operator
import re
import urlparse
logging.basicConfig(format='%(process)-6d %(levelname)-8s %(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)

__author__ = 'Stephen McGregor <s.mcgregor@griffith.edu.au>'
__description__ = """Modular, self documenting API server using Python"""
__version__ = '0.0.31'
__license__ = 'MIT'

def filter_by_list(fdict, flist):
	def inner(key):
		return key[0] in flist
	return dict(filter(inner, fdict.items()))

class __Route(object):
	def __init__(self, path, version="v1", notAPI=False):
		if notAPI:
			self.path = path
		else:
			self.path = '/' + '/'.join(['api', version, path.lstrip('/')])
		self.priority = 10

	def __eq__(self, *args):
		return False

	def __lt__(self, other_route):
		return self.priority < other_route.priority

	def __gt__(self, other_route):
		return self.priority > other_route.priority

class StringRoute(__Route):
	def __init__(self, path, **kwargs):
		super(StringRoute, self).__init__(path, **kwargs)
		self.priority = 1

	def __eq__(self, wsgi_path):
		return self.path == wsgi_path

class RegexRoute(__Route):
	def __init__(self, path, **kwargs):
		super(RegexRoute, self).__init__(path, **kwargs)
		self.path = "^" + self.path
		self.regex = re.compile(self.path)
		self.priority = 14

	def __eq__(self, wsgi_path):
		return self.regex.match(wsgi_path)

class Request(object):
	def __init__(self, environ):
		self.headers = {'content-type': 'text/html'}
		self.status = '200 OK'
		self.method = environ['REQUEST_METHOD']
		self._ENV = environ
		self._GET = self.__split_get()
		self._POST = self.__split_post()

	def set_status(self, status):
		self.status = status

	def set_header(self, header, value):
		self.headers.update({header: value})

	def __split_get(self):
		return urlparse.parse_qs(self._ENV.get('QUERY_STRING', ''), keep_blank_values=True)

	def __split_post(self):
		safe_env = filter_by_list(self._ENV, ['CONTENT_LENGTH', 'CONTENT_TYPE', 'REQUEST_METHOD'])
		return cgi.FieldStorage(fp=self._ENV['wsgi.input'], environ=safe_env, keep_blank_values=1)

class Engine(object):
	_instance = None
	def __new__(cls, *args, **kwargs):
		if not cls._instance:
			cls._instance = super(Engine, cls).__new__(cls, *args, **kwargs)
		return cls._instance

	def __init__(self):
		if not hasattr(self, 'routes'):
			self.routes = []

	def register(self, path_handler, handler=None):
		'''Route decorator, @application_instance.register(..)'''

		def wrapper(handler):
			func_handler = path_handler

			if isinstance(path_handler, str):
				func_handler = StringRoute(path_handler)

			logger.info("Registered route %s to %s", func_handler.path, handler.__name__)

			self.routes.append((func_handler, handler()))

			self.routes = sorted(self.routes, key=operator.itemgetter(0))
		return wrapper

	def wsgi(self, environ, start_response):
		'''WSGI compliant callable, only ever called from __call__'''

		path = environ['PATH_INFO']

		for path_handler in self.routes:
			if path_handler[0] == path:
				try:
					handler = path_handler[1]
					req = Request(environ)

					if hasattr(handler, 'do_' + req.method):
						func_output = getattr(handler, 'do_' + req.method)(req)
					else:
						start_response('405 Method Not Allowed', [('content-type', 'text/html')])
						return (repr(req.method) + ' Method Not Allowed',)

					if type(func_output) == dict:
						func_output = json.dumps(func_output)
						req.set_header('content-type', 'application/json')
					elif type(func_output) == list:
						func_output = ''.join(func_output)
					elif type(func_output) == unicode:
						func_output = func_output.encode('utf8')

					start_response(req.status, req.headers.items())
					return func_output
				except Exception as e:
					import traceback
					logger.error("%s\t%s", e, traceback.format_exc())
					start_response('500 Internal Server Error', [('content-type', 'text/plain')])
					return "An incident has been recorded:\n" + repr(e)

		start_response('404 Not Found', [('content-type', 'text/html')])
		return ('Not Found',)

	def __call__(self, environ, start_response):
		return self.wsgi(environ, start_response)

	def run(self, host="0.0.0.0", port=8000):
		'''Run this instance of engine using simple_server'''
		from wsgiref.simple_server import make_server
		srv = make_server(host, port, self)
		try:
			srv.serve_forever()
		except KeyboardInterrupt:
			logger.critical("Got CTRL + C. Sent shutdown to server.")

if __name__ == '__main__':
	app = Engine()
	@app.register(StringRoute('/'))
	class IndexRoute(object):
		def do_GET(self, request):
			return {'status': 'working'}
	app.run()

{'traceback': 'traceback', 'make_server': 'wsgiref.simple_server.make_server'}


import os.path
import json

from plumbum.machines.paramiko_machine import ParamikoMachine
from paramiko.client import MissingHostKeyPolicy
from rpyc.utils.zerodeploy import DeployedServer

# will raise exception
class ignoreIt(MissingHostKeyPolicy):
	def missing_host_key(self, client, hostname, key):
		pass

# You have to figure this part out
# Protip: You need a folder called 'deploy.shim', and a settings file in json format.
def getCreds():
	fpath = os.path.join(os.path.split(__file__)[0], 'deploy.shim', 'settings.json')
	with open(fpath) as fp:
		creds = json.loads(fp.read())

	creds['remote key'] = os.path.join(os.path.split(__file__)[0], 'deploy.shim', creds['remote key'])
	return creds

def go():

	creds = getCreds()

	print("Connecting...")
	mach = ParamikoMachine(creds['remote address'], user=creds['remote user'], keyfile=creds['remote key'], missing_host_policy=ignoreIt())
	print("Deploying RPC interface...")
	server = DeployedServer(mach)
	print("Connected.")

	conn = server.classic_connect()

	import WebRequest
	WebRequest.urllib.request = conn.modules['urllib.request']

	wg = WebRequest.WebGetRobust()
	print(wg.getpage('http://api.ipify.org?format=json'))

	import MangaCMSOld.ScrapePlugins.H.SadPandaLoader.Run

	MangaCMSOld.ScrapePlugins.H.SadPandaLoader.Run.test()

if __name__ == "__main__":
	go()


# coding=utf-8
"""
csvtest.csv - baza danych
inputtest.xlsx - plik do którego chcemy zapisać wynik spisu
spistest.xlsx - spis ze strony
"""

import openpyxl
import numpy as np

lines = np.genfromtxt("csvtest.csv", delimiter=";", dtype=None)
compdict = dict()
for i in range(len(lines)):
    compdict[lines[i][0]] = lines[i][1]

wb1 = openpyxl.load_workbook(filename='inputtest.xlsx')

wb2 = openpyxl.load_workbook(filename='spistest.xlsx')

ws = wb1.get_sheet_by_name('Sheet1')

spis = wb2.get_sheet_by_name('Sheet1')

for row in ws.iter_rows(min_row=1, max_row=ws.max_row, min_col=1):
    for cell in row:
        if cell.value in compdict:
            for wiersz in spis.iter_rows(min_row=1, max_row=spis.max_row, min_col=1):
                for komorka in wiersz:
                    if komorka.value == compdict[cell.value]:
                        cena = spis.cell(row=komorka.row, column=2)
                        ws.cell(row=cell.row, column=2, value=cena.value)

wb1.save('inputtest.xlsx')
wb2.close()


# importing libraries we need
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import pandas as pd
import time
import csv
import re
from multiprocessing import Pool
import tqdm

# reading base dataset
df = pd.read_csv('main_task.csv')

# Creating the dict from parts of URL given in original CSV. They will be used in parsing
url_dict = df.URL_TA.to_dict()

# updating fake user agent
user_agent = UserAgent()
user_agent.update()

# setting up a csv File for full parsing
# ROWS: 'Link', 'Status', 'Supervised','Kitchen_style_new','Rating', 'Price_range_new', 'Reviews_10',
# 'Reviews_10_head', "Number_of_reviews_new"))
csvFile = open('Food_parse.csv', 'w', encoding='utf-8')
writer = csv.writer(csvFile)
writer.writerow(('Link', 'Status', 'Supervised', 'Kitchen_style_new', 'Rating',
                 'Price_range_new', 'Reviews_10', 'Head_Reviews_head', "Number_of_reviews_new"))
csvFile.close()

# Technical part revisited

# Defining csv writer function. It will be used from the start. You need to pass 7 lines

def write_to_csv(link, status_code, registration, kitchen_style, rating, price, clean_reviews,
                 review_headers, number_of_reviews):
    # this link is for final parse
    with open('Food_parse.csv', 'a', encoding='utf-8') as append_file:
        writer2 = csv.writer(append_file)
        writer2.writerow((link, status_code, registration, kitchen_style, rating, price, clean_reviews,
                          review_headers, number_of_reviews))

    # csvFile = open('Food_parse.csv', 'a', encoding = 'utf-8')
    # writer.writerow((link, status_code, registration, kitchen_style, rating, price, clean_reviews,
    #              review_headers, number_of_reviews))
    # csvFile.close()

# crawling function - try to reach site or try to switch user-agent

def crawler(link):
    try:
        site = requests.get(link, headers={'User-Agent': user_agent.chrome})
        status_code = site.status_code
        soup = BeautifulSoup(site.text, 'html.parser')

    except:
        time.sleep(10)
        site = requests.get(link, headers={'User-Agent': user_agent.random})
        status_code = site.status_code
        soup = BeautifulSoup(site.text, 'html.parser')

    return site, status_code, soup

# checking if the restaurant page is being revised by owner or not. Also we check if the restaurant is closed
# and its page is deleted

def supervised_restaurant(soup):
    try:
        is_registered = soup.find('div', class_='_3bSXp6ba').text.strip()

        if 'Claimed' in is_registered:
            registration = 1

        else:
            registration = 0

    except:

        registration = 404

    return registration

# This function is helper, needed to track the position of CUISINE block.
# At some pages it can be first, second or... ABSENT.
def index_find(soup):
    try:
        cuisine_index = soup.find_all('div', class_='_14zKtJkz')
        cuisine_index = [link.text for link in cuisine_index]
        cuisine_index = cuisine_index.index('CUISINES')

    except:
        cuisine_index = 404

    return cuisine_index

# This function tracks kitchen style. We look at the first place, then second. If unsuccessful we put "none".
def kitchen_style_grabber(kitchen_style, soup):
    cuisine_index = index_find(soup)

    if cuisine_index == 404:
        kitchen_style.append('None')
        return kitchen_style

    else:
        try:
            cuisine_index = index_find(soup)
            kitchen_style.extend(
                soup.find_all('div', class_='_1XLfiSsv')[cuisine_index].text.replace(', ', ',').split(','))
            return kitchen_style

        except:

            kitchen_style.extend(soup.find('div', class_='_3dyNdB6_').text.replace(', ', ',').split(','))
            return kitchen_style

#             kitchen_style.append('None')
#             return kitchen_style

# parsing rating (1.0 - 5.0) and price. If price is not defined we put 0 (price marked with $ symbol)

def get_rating(soup):
    try:
        rating = float(soup.find('span', class_="r2Cf69qf").text)
        return rating

    except:
        rating = 0
        return rating

def get_price(soup):
    try:
        price = str(soup.find('a', class_="_2mn01bsa").text)
        if '$' not in price:
            price = 0
        return price

    except:
        price = 0
        return price

# parsing number of reviews
def get_number_of_reviews(soup):
    try:
        number_raw = soup.find('a', class_='_10Iv7dOs').text
        number_of_reviews = int(re.match(r'\d+', number_raw).group(0))

        return number_of_reviews

    except:
        number_of_reviews = 0

        return number_of_reviews

# parsing reviews. We take top-10
def get_reviews(soup):
    try:
        pre_review = soup.find_all('p', class_='partial_entry')
        reviews10 = [link.text for link in pre_review]

        if (len(reviews10) == 1) & ('More' in reviews10):
            reviews10 = ['NOT REVIEWED']
        clean_reviews = [review[:-4] if review[-4:] == 'More' else review for review in reviews10]

        pre_review_header = soup.find_all('span', class_='noQuotes')
        review_headers = [link.text for link in pre_review_header]

        if (len(review_headers) == 1) & (not review_headers[0]):
            review_headers = ['NOT REVIEWED']

        return clean_reviews, review_headers

    except:
        review_headers = ['NOT REVIEWED']
        clean_reviews = ['NOT REVIEWED']

        return clean_reviews, review_headers

# this function executes all other parsing functions
def soup_execute(link):
    print('Parsing url:', link)

    site, status_code, soup = crawler(link)
    if status_code != 200:
        write_to_csv(link, status_code, 'failed', 'failed', 'failed', 'failed', 'failed',
                     'failed', 'failed')
        return

    else:
        registration = supervised_restaurant(soup)

        if registration == 404:
            write_to_csv(link, status_code, 'closed', 'closed', 'closed', 'closed', 'closed',
                         'closed', 'closed')
            return

        else:

            kitchen_style = []
            kitchen_style_grabber(kitchen_style, soup)

            rating = get_rating(soup)
            price = get_price(soup)

            clean_reviews, review_headers = get_reviews(soup)
            number_of_reviews = get_number_of_reviews(soup)

            write_to_csv(link, status_code, registration, kitchen_style, rating, price, clean_reviews,
                         review_headers, number_of_reviews)

    time.sleep(3)

# creating list for iteration

site_list = list('https://www.tripadvisor.com' + record for record in url_dict.values())

# and finally we launch

if __name__ == '__main__':
    with Pool(8) as p:
        list(tqdm.tqdm(p.imap(soup_execute, site_list), total=len(site_list)))
        p.close()
        p.join()


from behave import *
import requests
import json

urlProj = 'http://localhost:4567/projects'

#use_step_matcher("re")

@step("the following project exist")
def step_impl(context):
    response = requests.get(urlProj)
    request_data = response.json()
    projects = request_data["projects"]
    if context.table[0].get('id') is not None:
        ids = [project['id'] for project in projects]
        titles = [project['title'] for project in projects]
        completed = [project['completed'] for project in projects]
        active = [project['active'] for project in projects]
        description = [project['description'] for project in projects]
        assert context.table[0]['id'] in ids
        assert context.table[0]['title'] in titles
        assert context.table[0]['completed'] in completed
        assert context.table[0]['active'] in active
        assert context.table[0]['description'] in description
    else:
        for row in context.table:
            if row['title'] not in [project['title'] for project in projects]:
                data = {'title': row['title'], 'completed': json.loads(row['completed']),
                        'active': json.loads(row['active']), 'description': row['description']}
                res = requests.post(urlProj, data=json.dumps(data))

@given('a project with "{title}" and "{completed}" and "{active}" and "{description}"')
def step_impl(context, title, completed, active, description):
    new_project = {}
    if title != "null": new_project['title'] = title
    if completed != "null": new_project['completed'] = json.loads(completed)
    if active != "null":
        try:
            new_project['active'] = json.loads(active)
        except json.decoder.JSONDecodeError:
            new_project['active'] = active
    if description != "null": new_project['description'] = description
    context.new_project = new_project

@when("I create the project")
def step_impl(context):
    response = requests.post(urlProj, data=json.dumps(context.new_project))
    context.response = response

@then("the project is created")
def step_impl(context):
    context.new_project['id'] = context.response.json().get("id")
    assert context.response.status_code == 201
    assert context.response.json().get("id") is not None
    try:
        assert context.response.json().get("title") == context.new_project['title']
    except KeyError:
        i=0
    try:
        assert json.loads(context.response.json().get("completed")) == context.new_project['completed']
    except KeyError:
        i=0
    assert json.loads(context.response.json().get("active")) == context.new_project['active']
    if 'description' in context.new_project:
        assert context.response.json().get("description") == context.new_project['description']

@step("the new project object is different from the existing project")
def step_impl(context):
    response = requests.get(urlProj + '?title=' + format(context.new_project["title"]))
    projects = response.json().get("projects")
    assert len(projects) > 0
    assert projects[0]['id'] != ""

@then("the project is not created")
def step_impl(context):
    response = requests.get(urlProj + '?description=' + format({context.new_project["description"]}))
    projects = response.json().get("projects")
    assert context.response.status_code == 400
    assert context.response.json().get("id") is None
    assert len(projects) == 0

@given('a project with "{title}" exists')
def step_impl(context, title):
    response=requests.get(urlProj, data=json.dumps({'title': format(title)}))

    if(response.status_code==404):
        response=requests.post(urlProj, data=json.dumps({'title': format(title)}))
    context.project_id = response.json().get('projects')[0].get('id')
    context.project=response.json().get('projects')[0]
    context.title=response.json().get('projects')[0].get('title')
    assert response.status_code==200

@when("I delete the project")
def step_impl(context):
    response=requests.delete(urlProj+'/'+format(context.project_id))
    context.response=response

@then("the project is deleted")
def step_impl(context):
    response=requests.get(urlProj+'/'+format(context.project_id))
    assert response.status_code==404
    assert response.json().get("errorMessages") is not None

@given('a project with "{id}" not exist')
def step_impl(context, id):
    response = requests.get(urlProj + '/' + format(id))
    context.new_project = response.json()
    context.project_id = id
    assert response.status_code == 404

@when("I get the project by title")
def step_impl(context):
    response=requests.get(urlProj, data=json.dumps({'title': context.project.get('title')}))
    context.status_code=response.status_code
    context.project=response.json().get('projects')[0]
    context.response=response

@then("the response returns the project object")
def step_impl(context):
    assert context.project.get('title')== context.title

@when("I get the project by id")
def step_impl(context):
    response=requests.get(urlProj+'/'+format(context.project_id))
    context.response=response

@when('I update the project with "{new_title}"')
def step_impl(context, new_title):
    context.new_title=new_title
    response=requests.put(urlProj+'/'+format(context.project_id), data=json.dumps({'title':new_title}))
    context.response=response

@then("the project is updated")
def step_impl(context):
    assert context.response.status_code==200
    assert context.response.json().get('id') is not None
    if "new_title" in context:
        assert context.response.json().get("title")==context.new_title
    if "new_description" in context:
        assert context.response.json().get("description")==context.new_description

@given('a project with "{old_title}" and "{old_description}" exist')
def step_impl(context, old_title, old_description):
    response = requests.get(urlProj, data=json.dumps({'title': format(old_title), 'description':format(old_description)}))

    if (response.status_code == 404):
        response = requests.post(urlProj, data=json.dumps({'title': format(old_title),'description':format(old_description)}))
    context.project_id = response.json().get('projects')[0].get('id')
    assert response.status_code == 200

@when('I update the project with "{new_title}" and "{new_description}" together')
def step_impl(context, new_title, new_description):
    context.new_title, context.new_description= new_title, new_description
    response = requests.put(urlProj + '/' + format(context.project_id), data=json.dumps({'title': new_title, 'description':new_description}))
    context.response = response

@then("the project is not updated")
def step_impl(context):
    assert context.response.status_code == 404

    if "new_title" in context:
        assert context.response.json().get("title") == context.new_title
    if "new_description" in context:
        assert context.response.json().get("description") == context.new_description

@when('I get the project by "{title}" and "{description}"')
def step_impl(context, title, description):
    response = requests.get(urlProj, data=json.dumps({'title': format(title), 'description':format(description)}))
    context.status_code = response.status_code
    context.project = response.json().get('projects')[0]
    print(context.project)
    context.response = response
    context.title=title

@given('a project with description "{old_description}" exists')
def step_impl(context, old_description):
    response = requests.get(urlProj, data=json.dumps({'description': format(old_description)}))

    if (response.status_code == 404):
        response = requests.post(urlProj, data=json.dumps({'description': format(old_description)}))
    context.project_id = response.json().get('projects')[0].get('id')
    context.project = response.json().get('projects')[0]
    context.description = response.json().get('projects')[0].get('description')
    assert response.status_code == 200

@when('I update the project with description "{new_description}"')
def step_impl(context, new_description):
    context.new_description = new_description
    response = requests.put(urlProj + '/' + format(context.project_id), data=json.dumps({'description': new_description}))
    context.response = response


import importlib
import hoomd

from functools import partial
from typing import Callable
from warnings import warn

from jax import jit, numpy as np
from jax.dlpack import from_dlpack as asarray
from hoomd.dlext import (
    AccessLocation,
    AccessMode,
    SystemView,
    images,
    net_forces,
    positions_types,
    rtags,
    velocities_masses,
    DLExtSampler,
)

from pysages.backends.core import ContextWrapper
from pysages.backends.snapshot import (
    Box,
    HelperMethods,
    Snapshot,
    SnapshotMethods,
    build_data_querier,
    restore as _restore,
)
from pysages.methods import SamplingMethod

# TODO: Figure out a way to automatically tie the lifetime of Sampler
# objects to the contexts they bind to
CONTEXTS_SAMPLERS = {}

class Sampler(DLExtSampler):
    def __init__(self, sysview, method_bundle, bias, dt, callback: Callable):
        _, initialize, update = method_bundle
        self.state = initialize()
        self.callback = callback
        self.bias = bias
        box = sysview.particle_data().getGlobalBox()
        self.pybox = self._get_pybox(box)
        self.dt = dt

        def python_update(positions, vel_mass, rtags, imgs, forces, timestep):
            positions = asarray(positions)
            vel_mass = asarray(vel_mass)
            ids = asarray(rtags)
            images = asarray(imgs)
            forces = asarray(forces)
            snap = Snapshot(
                positions=positions,
                vel_mass=vel_mass,
                forces=forces,
                ids=ids,
                images=images,
                box=self.pybox,
                dt=self.dt,
            )
            self.state = update(snap, self.state)
            self.bias(snap, self.state)
            if self.callback:
                self.callback(snap, self.state, timestep)

        super().__init__(sysview, python_update, default_location(), AccessMode.Read)

    def _get_pybox(self, box):
        L = box.getL()
        xy = box.getTiltFactorXY()
        xz = box.getTiltFactorXZ()
        yz = box.getTiltFactorYZ()
        lo = box.getLo()
        H = ((L.x, xy * L.y, xz * L.z), (0.0, L.y, yz * L.z), (0.0, 0.0, L.z))
        origin = (lo.x, lo.y, lo.z)
        return Box(H, origin)

if hasattr(AccessLocation, "OnDevice"):

    def default_location():
        return AccessLocation.OnDevice

else:

    def default_location():
        return AccessLocation.OnHost

def is_on_gpu(context):
    return context.on_gpu()

def take_snapshot(wrapped_context, location=default_location()):
    context = wrapped_context.context
    sysview = wrapped_context.view
    positions = asarray(positions_types(sysview, location, AccessMode.Read))
    vel_mass = asarray(velocities_masses(sysview, location, AccessMode.Read))
    forces = asarray(net_forces(sysview, location, AccessMode.ReadWrite))
    ids = asarray(rtags(sysview, location, AccessMode.Read))
    imgs = asarray(images(sysview, location, AccessMode.Read))

    box = sysview.particle_data().getGlobalBox()
    L = box.getL()
    xy = box.getTiltFactorXY()
    xz = box.getTiltFactorXZ()
    yz = box.getTiltFactorYZ()
    lo = box.getLo()
    H = ((L.x, xy * L.y, xz * L.z), (0.0, L.y, yz * L.z), (0.0, 0.0, L.z))
    origin = (lo.x, lo.y, lo.z)
    dt = context.integrator.dt

    return Snapshot(positions, vel_mass, forces, ids, imgs, Box(H, origin), dt)

def update_snapshot(snapshot, sysview, location=default_location()):
    #
    positions = asarray(positions_types(sysview, location, AccessMode.Read))
    vel_mass = asarray(velocities_masses(sysview, location, AccessMode.Read))
    forces = asarray(net_forces(sysview, location, AccessMode.ReadWrite))
    ids = asarray(rtags(sysview, location, AccessMode.Read))
    imgs = asarray(images(sysview, location, AccessMode.Read))
    #
    return Snapshot(positions, vel_mass, forces, ids, imgs, snapshot.box, snapshot.dt)

def build_snapshot_methods(sampling_method):
    if sampling_method.requires_box_unwrapping:

        def positions(snapshot):
            L = np.diag(snapshot.box.H)
            return snapshot.positions[:, :3] + L * snapshot.images

    else:

        def positions(snapshot):
            return snapshot.positions

    def indices(snapshot):
        return snapshot.ids

    def momenta(snapshot):
        M = snapshot.vel_mass[:, 3:]
        V = snapshot.vel_mass[:, :3]
        return (M * V).flatten()

    def masses(snapshot):
        return snapshot.vel_mass[:, 3:]

    return SnapshotMethods(jit(positions), jit(indices), jit(momenta), jit(masses))

def build_helpers(context, sampling_method):
    # Depending on the device being used we need to use either cupy or numpy
    # (or numba) to generate a view of jax's DeviceArrays
    if is_on_gpu(context):
        cupy = importlib.import_module("cupy")
        view = cupy.asarray

        def sync_forces():
            cupy.cuda.get_current_stream().synchronize()

    else:
        utils = importlib.import_module(".utils", package="pysages.backends")
        view = utils.view

        def sync_forces():
            pass

    def bias(snapshot, state, sync_backend):
        """Adds the computed bias to the forces."""
        # TODO: check if this can be JIT compiled with numba.
        # Forces may be computed asynchronously on the GPU, so we need to
        # synchronize them before applying the bias.
        sync_backend()
        forces = view(snapshot.forces)
        biases = view(state.bias.block_until_ready())
        forces[:, :3] += biases
        sync_forces()

    snapshot_methods = build_snapshot_methods(sampling_method)
    flags = sampling_method.snapshot_flags
    restore = partial(_restore, view)
    helpers = HelperMethods(build_data_querier(snapshot_methods, flags), restore)

    return helpers, bias

def bind(
    wrapped_context: ContextWrapper, sampling_method: SamplingMethod, callback: Callable, **kwargs
):
    context = wrapped_context.context
    helpers, bias = build_helpers(context, sampling_method)

    with SystemView(context.system_definition) as sysview:
        wrapped_context.view = sysview
        wrapped_context.run = hoomd.run

        snapshot = take_snapshot(wrapped_context)
        method_bundle = sampling_method.build(snapshot, helpers)
        sync_and_bias = partial(bias, sync_backend=sysview.synchronize)

        sampler = Sampler(sysview, method_bundle, sync_and_bias, context.integrator.dt, callback)
        context.integrator.cpp_integrator.setHalfStepHook(sampler)

        CONTEXTS_SAMPLERS[context] = sampler
    return sampler

def detach(context):
    """
    If pysages was bound to this context, this removes the corresponding
    `Sampler` object.
    """
    if context in CONTEXTS_SAMPLERS:
        context.integrator.cpp_integrator.removeHalfStepHook()
        del CONTEXTS_SAMPLERS[context]
    else:
        warn("This context has no sampler bound to it.")


import pymongo
#to create

def create():
    client = pymongo.MongoClient("mongodb://localhost:27017/")
    mydb = client["mydbs"]
    mycol = mydb["phnumbers"]
    name1 = input("Enter name:")
    phnum = input("Enter num:")
    gender = input("Enter gender:")
    email = input("enter email.id:")
    address = input("Enter address:")
    mydict = {"name": f'{name1}',
              "phnum": f'{phnum}',
              "gender": f'{gender}',
              "email": f'{email}',
              "address": f'{address}'
             }
    mycol.insert_one(mydict)

    for i in mycol.find(mydict):
        print(i)
    print("*** Created successfully ***")

# To update
def update():
    print("Type [1] to update name\nType [2] to update number\nType [3] to update email\nType [4] to update address\nType [5] to update or add an newfeild")
    opt = int(input("Enter number:"))
    # TO update  name
    if opt == 1:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        name = input("Enter old name:")
        newname = input("Enter new name to update:")
        myquery = {"name": f'{name}'}
        newvalue = {"$set": {"name": f'{newname}'}}
        y=mycol.update_one(myquery, newvalue)
        for i in mycol.find():
            print(i)

        print("Update successfully")
    # To update phone number
    if opt == 2:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        name = input("Enter name of the contact:")
        new_phnum = input("Enter new phnum to update:")
        myquery = {"name": f'{name}'}
        newvalue = {"$set": {"phnum": f'{new_phnum}'}}
        mycol.update_one(myquery, newvalue)

        for x in mycol.find():
            print(x)
        print("Update successfully")
    #to upadate email id
    if opt == 3:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        name = input("Enter name of the contact:")
        new_email = input("Enter new email to update:")
        myquery = {"email": f'{name}'}
        newvalue = {"$set": {"email": f'{new_email}'}}
        mycol.update_one(myquery, newvalue)
        print("Update successfully")
        for x in mycol.find():
            print(x)
    #to update address
    if opt == 4:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        name= input("Enter name:")
        new_address = input("Enter new address to update:")
        myquery = {"name": f'{name}'}
        newvalue = {"$set": {"address": f'{new_address}'}}
        mycol.update_one(myquery, newvalue)
        print("Update successfully")
        for x in mycol.find():
            print(x)
    #to add a new feild to th document
    if opt==5:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        name=input("Enter name to add a field")
        field_name=input("Enter feild name:")
        feild_det=input("Enter feild details")
        mycol.update_one({"$set":{f'{field_name}': f'{feild_det}'}})
        print("Updated successfully")
        for x in mycol.find():
            print(x)

# To retrive
def read():
    global mycol
    print("Type [1] to serach based on name\nType [2] to search  based on number")
    opt = int(input("Enter number:"))
    if opt == 1:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        myquery = input("Enter name to search:")
        search = {"name": f'{myquery}'}

        #mydoc = mycol.find()
        for i in mycol.find((search)):
          print(i)

    if opt == 2:
        client = pymongo.MongoClient("mongodb://localhost:27017/")
        mydb = client["mydbs"]
        mycol = mydb["phnumbers"]
        myquery1 = input("Enter number to search:")
        search1 = {"phnum": f'{myquery1}'}
        mydoc = mycol.find(search1)
        for y in mydoc:
            print(y)

#To delete a document
def delete():
    client = pymongo.MongoClient("mongodb://localhost:27017/")
    mydb = client["mydbs"]
    mycol = mydb["phnumbers"]
    myquery = input("Enter name to delete :")
    x=mycol.delete_one({"name": f'{myquery}'})

    print("___*** Delete successfully ***___")
    print("___*** Remaining Details ***___")
    for x in mycol.find():
        print(x)

def options():
    print("-----***** TELEPHONE DIRECTORY *****----")
    print("------------** WELCOME'S YOU **--------------")
    print("Please Enter Number")
    print("Type 1 --> to create a contact\nType 2 --> to update a contact\nType 3 --> to read a contact\nType 4 --> to delete a contact")
    option = int(input("Enter Number:"))

    if option == 1:
        create()
    if option == 2:
        update()
    if option == 3:
        read()
    if option == 4:
        delete()
options()


'''
GLM 3-Point prediction
'''

__author__ = "Celso M. de Melo"
__email__ = "celsodemelo@eleveninc.com"

from sklearn import linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.externals import joblib

from predictions.feature_sets import clean_df
from predictions.metrics import linear_regression_accuracy, linear_regression_accuracy_greater_or_equal_than, find_error_for_accuracy_target

from sklearn.preprocessing import StandardScaler

from predictions.utils import sort_coef

def find_best_model(features, label, df_train, df_eval, verbose=False, scale=False):
    df_train_clean = clean_df(df_train, features + [label])
    df_eval_clean = clean_df(df_eval, features + [label])

    df_train_clean_data = df_train_clean[features]
    df_eval_clean_data = df_eval_clean[features]
    if scale:
        scaler = StandardScaler()
        scaler.fit(df_train_clean_data)
        df_train_clean_data = scaler.transform(df_train_clean_data)
        df_eval_clean_data = scaler.transform(df_eval_clean_data)

    normalize = False
    # normalize_options = [False,
    #                      # True # Doesn't seem to make any difference
    #                      ]
    #
    # for normalize in normalize_options:
    regr = linear_model.LinearRegression(normalize=normalize)
    regr.fit(df_train_clean_data, df_train_clean[[label]].values.ravel())
    print('**************************************************************************')
    print('Ordinary Least Squares: normalize = ' + str(normalize) + ' Eval R2: '
          + str(regr.score(df_eval_clean_data, df_eval_clean[[label]].values.ravel())))
    print('\tTraining R2: ' + str(
        regr.score(df_train_clean_data, df_train_clean[[label]].values.ravel())))
    print('\tEvaluation R2: ' + str(
        regr.score(df_eval_clean_data, df_eval_clean[[label]].values.ravel())))

    y_pred = regr.predict(df_eval_clean_data)
    if verbose:
        print("Mean squared error: %.7f" %
              mean_squared_error(df_eval_clean[[label]].values.ravel(), y_pred))
        print('Variance score: %.7f' % r2_score(df_eval_clean[[label]].values.ravel(), y_pred))
        for e in range(1, 7):
            print('Accuracy (error = ' + str(e) + '): %.7f' % linear_regression_accuracy(
                df_eval_clean[[label]].values.ravel(), y_pred, error_threshold=e))
        for e in range(0, 8):
            print('Accuracy (greater or equal than, error = ' + str(
                e) + '): %.7f' % linear_regression_accuracy_greater_or_equal_than(
                df_eval_clean[[label]].values.ravel(), y_pred, error_threshold=e))

    return [
        regr,
        regr.score(df_eval_clean_data, df_eval_clean[[label]].values.ravel()),
        linear_regression_accuracy_greater_or_equal_than(
            df_eval_clean[[label]].values.ravel(), y_pred, error_threshold=3)
    ]

def select_error(errors):
    selected = None

    for e in errors:
        if e[1] >= .7 and e[1] < .8:
            if selected is None or (e[1] > selected[1] and e[0] < 4):
                selected = e

    if selected is None:
        for e in errors:
            if e[1] >= .8 and e[1] < .9:
                if selected is None or (e[1] > selected[1] and e[0] < 4):
                    selected = e

    if selected is None:
        for e in errors:
            if e[1] >= .9 and e[1] < .95:
                if selected is None or (e[1] > selected[1] and e[0] < 4):
                    selected = e

    if selected is None:
        selected = errors[len(errors)-1]
    return selected

def score_on_test_set(features, label, df_train_eval, df_test, normalize=False, path=None):
    detailed_score = {'label': label, 'r2': -1,
                      'gt_err': -1, 'gt_acc': 0,
                      'rg_err': -1, 'rg_acc': 0,
                      '25th_perc_acc': 0, '75th_perc_acc': 0}

    df_train_eval_clean = clean_df(df_train_eval, features + [label])
    df_test_clean = clean_df(df_test, features + [label])

    regr = linear_model.LinearRegression(normalize=normalize)
    regr.fit(df_train_eval_clean[features], df_train_eval_clean[[label]].values.ravel())

    if path is not None:
        joblib.dump(regr, path)

    detailed_score['r2'] = regr.score(df_test_clean[features], df_test_clean[[label]].values.ravel())
    print('**************************************************************************')
    print('Ordinary Least Squares: normalize = ' + str(normalize) + ' Test R2: ' + str(detailed_score['r2']))
    print('\tTraining + Eval R2: ' + str(
        regr.score(df_train_eval_clean[features], df_train_eval_clean[[label]].values.ravel())))
    print('\tTest R2: ' + str(detailed_score['r2']))

    y_pred = regr.predict(df_test_clean[features])
    print("Mean squared error: %.7f" %
          mean_squared_error(df_test_clean[[label]].values.ravel(), y_pred))
    print('Variance score: %.7f' % r2_score(df_test_clean[[label]].values.ravel(), y_pred))
    rg_errors = []
    for e in range(1, 7):
        acc = linear_regression_accuracy(df_test_clean[[label]].values.ravel(), y_pred, error_threshold=e)
        rg_errors.append((e, acc))
        print('Accuracy (error = ' + str(e) + '): %.7f' % acc)
    selected_rg_error = select_error(rg_errors)
    detailed_score['rg_err'] = selected_rg_error[0]
    detailed_score['rg_acc'] = selected_rg_error[1]

    gt_errors = []
    for e in range(0, 8):
        acc = linear_regression_accuracy_greater_or_equal_than(df_test_clean[[label]].values.ravel(), y_pred, error_threshold=e)
        gt_errors.append((e, acc))
        print('Accuracy (greater or equal than, error = ' + str(e) + '): %.7f' % acc)
    selected_gt_error = select_error(gt_errors)
    detailed_score['gt_err'] = selected_gt_error[0]
    detailed_score['gt_acc'] = selected_gt_error[1]

    acc, err = find_error_for_accuracy_target(df_test_clean[[label]].values.ravel(), y_pred, desired_accuracy=0.70)
    detailed_score['gt_70_acc'] = acc
    detailed_score['gt_70_err'] = err
    acc, err = find_error_for_accuracy_target(df_test_clean[[label]].values.ravel(), y_pred, desired_accuracy=0.75)
    detailed_score['gt_75_acc'] = acc
    detailed_score['gt_75_err'] = err

    print('**************************************************************************')
    print('COEFFICIENTS')
    print('**************************************************************************')
    print('Intercept: ' + str(regr.intercept_))
    sorted_coefs = sort_coef(features, regr.coef_)
    for k in sorted_coefs:
        print(k[0] + '\t' + str(k[1]))

    return detailed_score


import json
import gmpy2
import phe as paillier
import time
import pickle
from tensorflow import keras
from sklearn import preprocessing

'''
Fonction permettant de transformer notre chaine de caractère en vecteur selon une méthode de préprocessing
'''
def ia(text):

	with open('tokenizer.json') as f:
		data = json.load(f)
		tokenizer = keras.preprocessing.text.tokenizer_from_json(data)

	test_text = tokenizer.texts_to_sequences(text)
	data = keras.preprocessing.sequence.pad_sequences(test_text, maxlen=100)
	data.tolist()

	return data

'''
Fonction permettant de créer et de sauvegarder en mémoire les clés publiques et privés dans le json custkeys
'''
def storeKeys():
	public_key, private_key = paillier.generate_paillier_keypair()
	keys={}
	keys['public_key'] = {'n': public_key.n}
	keys['private_key'] = {'p': private_key.p, 'q': private_key.q}
	with open('custkeys.json', "w") as file:
		json.dump(keys, file)

'''
Fonction retournant les clés publiques et privés extraite du json custkeys
'''
def getkeys():
	with open('custkeys.json', "r") as file:
		keys = json.load(file)
		pub_key = paillier.PaillierPublicKey(n = int(keys['public_key']['n']))
		priv_key = paillier.PaillierPrivateKey(pub_key, keys['private_key']['p'], keys['private_key']['q'])
		return pub_key, priv_key

'''
Fonction permettant le chiffrement de nos données selon la méthode de Paillier
'''
def serializeData(public_key, data):
	encrypted_data_list = [public_key.encrypt(x, precision = None) for x in data]
	encrypted_data = {}
	encrypted_data['public_key'] = {'n':public_key.n}
	encrypted_data['values'] = [(str(x.ciphertext()), x.exponent) for x in encrypted_data_list]
	with open('message.json', "w") as file:
		json.dump(encrypted_data, file)			# On sauvegarde nos données chiffré dans le json message
	return serializeData

'''
Fonction permettant de stocker les informations du fichier message servant à l'envoie du message vers le serveur
'''
def loadAnswer():
	with open('answer.json', "r") as file:
		data = json.load(file)
	return data

'''
Fonction permettant de charger les informations du message envoyé par le client
'''
def getkeys2():
	with open('message.json', "r") as file:
		keys = json.load(file)
		value = {}
		pub_key = paillier.PaillierPublicKey(n = int(keys['public_key']['n']))
		value = [[int(x[0], int(x[1])) for x in keys['values']]]
		return pub_key, value
'''
Fonction permettant de prédire le langage des données analysées
'''
def predict(data):
	data = keras.preprocessing.sequence.pad_sequences(data, maxlen=1000)
	model = keras.models.load_model('model.h5')
	print(data)
	predictions = model.predict(data)
	out = predictions.argmax()
	return out

'''
Fonction permettant de stocker la réponse de l'IA afin de l'envoyer au client
'''
def sendData(data):
    encrypted_data_list = [public_key.encrypt(x, precision = 1) for x in data]
    encrypted_data = {}
    encrypted_data['public_key'] = {'n':public_key.n}
    encrypted_data['values'] = [(str(x.ciphertext()), x.exponent) for x in encrypted_data_list]
    with open('answer.json', "w") as file:
        json.dump(encrypted_data, file)
    return serializeData

storeKeys()
pub_key, priv_key = getkeys()
message = ["Le vent et le soleil se regarde et pendant un court instant, rien ne se passe"]
print(message)
ecrit = ia(message)
envoie = ecrit[0].tolist()
print(envoie)
serializeData(pub_key, envoie)

public_key, data = getkeys2()
print(data)
for x in range(len(data)) :
	out = predict((data[0][x]))
sendData(out)

answer_file = loadAnswer()
answer_key = paillier.PaillierPublicKey(n=int(answer_file['public_key']['n']))
if (answer_key == pub_key):
	for u in range(len(envoie)) :
		answer = paillier.EncryptedNumber(answer_key, int(str(answer_file['values'][u][0])))  # Récupère les données réenvoyé par le serveur
		reponse = priv_key.decrypt(answer)		# Déchiffrement des données avec l'utilisation de la clé privé
		print(reponse)
		#fin = dechiffrement(reponse)			# Conversion des données par la fonction dechiffrement
		#print('message recu :', fin)
else :
	print("La cle public est inconnu")		# Affiche un message d'erreur si la clé publique reçu est différente de celle sauvegarder en mémoire


import logging
import re
from urllib.parse import urlparse, urlunparse

from streamlink.plugin import Plugin
from streamlink.plugin.api import validate
from streamlink.stream import HLSStream
from streamlink.utils import parse_json

log = logging.getLogger(__name__)

class RaiPlay(Plugin):
    _re_url = re.compile(r"https?://(?:www\.)?raiplay\.it/dirette/(\w+)/?")

    _re_data = re.compile(r"data-video-json\s*=\s*\"([^\"]+)\"")
    _schema_data = validate.Schema(
        validate.transform(_re_data.search),
        validate.any(None, validate.get(1))
    )
    _schema_json = validate.Schema(
        validate.transform(parse_json),
        validate.get("video"),
        validate.get("content_url"),
        validate.url()
    )

    @classmethod
    def can_handle_url(cls, url):
        return cls._re_url.match(url) is not None

    def _get_streams(self):
        json_url = self.session.http.get(self.url, schema=self._schema_data)
        if not json_url:
            return

        json_url = urlunparse(urlparse(self.url)._replace(path=json_url))
        log.debug("Found JSON URL: {0}".format(json_url))

        stream_url = self.session.http.get(json_url, schema=self._schema_json)
        log.debug("Found stream URL: {0}".format(stream_url))

        res = self.session.http.request("HEAD", stream_url)
        # status code will be 200 even if geo-blocked, so check the returned content-type
        if not res or not res.headers or res.headers["Content-Type"] == "video/mp4":
            log.error("Geo-restricted content")
            return

        yield from HLSStream.parse_variant_playlist(self.session, stream_url).items()

__plugin__ = RaiPlay


import pygame
from constantes import *
from botin import *

class Background:
    def __init__(self, image_path, ancho=None, alto=None, scale=False):
        self.image = pygame.image.load(image_path).convert()
        if scale:
            self.image = pygame.transform.scale(self.image, (ancho, alto))
        self.ancho = self.image.get_rect().width
        self.alto = self.image.get_rect().height
        self.pos_x = 0
        self.background_pos_x = 0 #para luego llamar desde niveles a esta variable y pasarsela a la clase botin

    def draw(self, screen, background_pos_x):
        self.background_pos_x = background_pos_x
        screen.blit(self.image, (background_pos_x - self.ancho, 0))
        if background_pos_x < ANCHO_VENTANA:
            screen.blit(self.image, (background_pos_x, 0))

    @staticmethod
    def scroll_background(buzo, lista_tiburones, lvl_pos_x):
        if buzo.rect.centerx > ANCHO_MITAD_VENTANA and buzo.posicion_x > 0:
            buzo.posicion_x -= buzo.velocidad
            lvl_pos_x -= 5
            for tiburon in lista_tiburones:
                tiburon.rect_imagen.x -= 5

        if buzo.rect.centerx < ANCHO_MITAD_VENTANA and buzo.posicion_x < 0:
            buzo.posicion_x += buzo.velocidad
            lvl_pos_x += 5
            for tiburon in lista_tiburones:
                tiburon.rect_imagen.x += 5
        return lvl_pos_x

class StaticBackground(Background):
    def __init__(self, image_path, ancho=None, alto=None, scale=False):
        super().__init__(image_path, ancho, alto, scale)

    def draw(self, screen):
        screen.blit(self.image, (0, 0))

    @staticmethod
    def scroll_background(buzo, lista_tiburones, lvl_pos_x):
        return lvl_pos_x