import os
os.environ['KMP_WARNINGS'] = 'off'
import time
import math
import argparse
import numpy as np
from thop import profile
from PIL import Image

import torch
import torch.nn as nn
from torchsummary import summary
import torch.distributed as dist
from torch.utils.tensorboard import SummaryWriter

from src.model import get_model
from src.dataset import get_dataset
from src.utils import get_logger

from ipdb import set_trace as st
def debug():
    if dist.get_rank() == 0:
        st()
        torch.distributed.barrier()
    else:
        torch.distributed.barrier()

# set random seed
def setup_seed(seed):
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    np.random.seed(seed)
    torch.backends.cudnn.deterministic = True

def get_opt_from_yaml(path):
    assert os.path.exists(path), f"{path} must exists!"
    import yaml
    with open(path, 'r') as f:
        opt = yaml.load(f)
    return opt

def get_data_loaders(opt):
    data_opt = opt['dataset']
    Logger = get_logger()
    Logger.info(f"Start to get dataset {data_opt['name']}...")
    trainset, testset = get_dataset(opt)

    train_sampler = torch.utils.data.distributed.DistributedSampler(trainset)
    trainloader = torch.utils.data.DataLoader(
        trainset,
        sampler=train_sampler,
        batch_size=data_opt['batch_size'],
        num_workers=data_opt['num_workers'],
        pin_memory=data_opt['pin_memory'],
        shuffle=data_opt['shuffle'], drop_last=True)

    if data_opt['val'] is not None:
        Logger.info("testset is Valid.")
        test_sampler = torch.utils.data.distributed.DistributedSampler(testset)
        testloader = torch.utils.data.DataLoader(
            testset,
            sampler=test_sampler,
            batch_size=data_opt['batch_size'],
            num_workers=data_opt['num_workers'],
            pin_memory=data_opt['pin_memory'],
            shuffle=data_opt['shuffle'], drop_last=True)
    else:
        Logger.info("testset is NONE.")
        testloader = None
        test_sampler = None

    return trainloader, testloader, train_sampler, test_sampler

def print_opts(opt, logger, start=2):
    if isinstance(opt, dict):
        for key, value in opt.items():
            if isinstance(value, dict):
                logger.info(' '*start + str(key))
                print_opts(value, logger, start+4)
            else:
                logger.info(' ' * start + str(key) + ' ' * start + str(value))
    else:
        logger.info(' '*start + str(opt))

def get_visualize_img(img): # img: [B T C H W]
    # mean = torch.tensor([0.485, 0.456, 0.406])
    # std = torch.tensor([0.229, 0.224, 0.225])
    # x = img[:8].detach().cpu() * std[None, None, :, None, None] + \
    #     mean[None, None, :, None, None]
    x = img[:8].detach().cpu()
    show_x = torch.clamp(x, min=0, max=1)
    b, t, c, h, w = show_x.shape
    show_x = show_x.permute((0, 3, 1, 4, 2)).numpy()
    show_x = show_x.reshape((b * h, t * w, c)) * 255.
    show_x = Image.fromarray(show_x.astype(np.uint8)).convert('RGB')
    return show_x

def print_unused_params(model):
    for name, param in model.named_parameters():
        if param.grad is None or torch.all(param.grad==0):
            print(name)

if __name__ == '__main__':
    assert torch.cuda.is_available()

    parser = argparse.ArgumentParser(description="VQVAES")
    parser.add_argument('--opt', default=None, type=str, help="config file path")
    parser.add_argument('--save_dir', default=None, type=str)
    parser.add_argument('--split', default=None, type=str)
    parser.add_argument('--local_rank', type=int)
    args = parser.parse_args()
    opt = get_opt_from_yaml(args.opt)

    print("Start to init torch distribution...")
    dist.init_process_group(backend='nccl', init_method='env://')
    print("Finish initializing torch distribution...")

    # get current experiment path
    setup_seed(10)
    writer = None
    Logger = get_logger(logging_file="tmp.log", name="tmp", isopen=True)

    # load train data and test data
    if args.split == 'train':
        opt['dataset']['batch_size'] = 1
        opt['dataset']['step'] = 1
        opt['dataset']['name'] += '_EXT'
        trainloader, _, _, _ = get_data_loaders(opt)
        Logger.info(f"Start to convert training set...")
        from tqdm import tqdm
        td = tqdm(range(len(trainloader)))
        trainloader_iter = enumerate(trainloader)
        for _ in td:
            i, inputs = next(trainloader_iter)
            np.save(os.path.join(args.save_dir, "{:010d}.npy".format(i)), inputs)

    elif args.split == 'test':
        opt['dataset']['batch_size'] = 1
        opt['dataset']['step'] = 20
        opt['dataset']['name'] += '_EXT'
        _, testloader, _, _ = get_data_loaders(opt)
        Logger.info(f"Start to convert testing set...")
        from tqdm import tqdm
        td = tqdm(range(len(testloader)))
        testloader_iter = enumerate(testloader)
        for _ in td:
            i, inputs = next(testloader_iter)
            np.save(os.path.join(args.save_dir, "{:010d}.npy".format(i)), inputs)

    else:
        raise NotImplementedError


# Aurelia Arnett
# Purpose: To load Tweets from database collections stored in MongoDB server & initial data discovery
# The tweets that are loaded in are from two different user timeline collections: @airbnb and @vrbo

import pymongo

client = pymongo.MongoClient('localhost', 27017) # call the server

# Part 1: Read in JSON formatted data from two Mongo DB collections (@Airbnb and @Vrbo)
# Tweets have been collected using the run_twitter_simple_search_save.py and the TweetAuth.py programs
# This component returns Tweets with from user timeline query collected & stored in the MongoDB server

#Airbnb
db1 = client.airbnbtimeline #define a term for the @Airbnb db
dbcoll1 = db1.airbnbs #define a term for the @Airbnb db collection

#VRBO
db2 = client.vrbotimeline #define a term for the @Vrbo db
dbcoll2 = db2.vrbos #define a term for the @Vrbo db collection

# Find Tweets that are in English and convert them from JSON structures into a standard python list
#Airbnb
tweets_airbnb=dbcoll1.find() #search through  the @Airbnb db collection
for tweet in tweets_airbnb:
   if tweet['lang'] == 'en': # return Tweets in English only
      tweetlist_airbnb = [tweet for tweet in tweets_airbnb]
#VRBO
tweets_vrbo=dbcoll2.find() #define a term for the @Vrbo db collection
for tweet in tweets_vrbo:
   if tweet['lang'] == 'en':
      tweetlist_vrbo = [tweet for tweet in tweets_vrbo]

# Define a function that will print the 1st Tweet in a collection
def print_tweet_data(tweets):
   for tweet in tweets:
         print('User:', tweet['user']['name'])
         print('Message:', tweet['text'])
         print('Hashtags:', [hashtag['text'] for hashtag in tweet['entities']['hashtags']] )
         print('Number of retweets:', tweet['retweet_count'])
         if not tweet['place'] is None:
            print('Place:', tweet['place']['full_name'])

#print the first Tweet in each collection
print('Example tweet from the Airbnb Collection:')
print('Total Tweets in collection', len(tweetlist_airbnb))
print_tweet_data(tweetlist_airbnb[:1])
print()
print('Example tweet from the VRBO Collection:')
print('Total Tweets in collection', len(tweetlist_vrbo))
print_tweet_data(tweetlist_vrbo[:1])
print()

# Part 2: Count total number of retweets in each timeline collection
countretweetsAirbnb = 0
for tweet in tweetlist_airbnb: #sort through the @airbnb timeline collection to find retweet info
   if int(tweet['retweet_count']) > 1:
      countretweetsAirbnb += 1 #count total number of retweets
#print(countretweetsAirbnb)

countretweetsVrbo = 0
for tweet in tweetlist_vrbo: #sort through the @vrbo timeline collection to find retweet info
   if int(tweet['retweet_count']) > 1:
      countretweetsVrbo += 1 #count total number of retweets
#print(countretweetsVrbo)

# Part 3: Export retweet information to a csv file
import csv
outfile = "4-1compareRetweets.csv" #Name of output csv file
with open(outfile, 'w', newline='') as csvfileout:
   retweetWriter = csv.writer(csvfileout, delimiter=',', quoting=csv.QUOTE_MINIMAL) #define the row writer
   retweetWriter.writerow(['Timeline Collection', 'Total number of retweets'])
   retweetWriter.writerow(['@Airbnb', countretweetsAirbnb]) #information on Airbnb
   retweetWriter.writerow(['@Vrbo', countretweetsVrbo]) #information on Vrbo
   retweetWriter.writerow(['for a collection of 1999 Tweets from the @Airbnb timeline collection & 1999 Tweets from the @Vrbo timeline'])
csvfileout.close()

Scripting - Airbnb Ratings/Store-MongoDB.py


"""Backend part of the weather application."""
import sqlite3
import csv
import sys
import os
import datetime
import unidecode

data_file = "data.csv"

def file_reader(data_file):
    """Open CSV file and appends data to a list."""
    ext = ext_check(data_file)
    if ext:
        with open(data_file, 'r', encoding='utf-8') as csv_file:
            reader = csv.reader(csv_file, delimiter=';')
            data = []
            for row in reader:
                data.append(row)
            return data
    elif not ext:
        return "File is not in the correct format"

def ext_check(file):
    """Check if extension is csv."""
    ext = os.path.splitext(file)
    if ext[1] == '.csv':
        return True
    else:
        return False

def process_file(file):
    """Insert values from uploaded file."""
    file = os.path.join('instance', file)
    data = file_reader(file)
    db = access_db()
    c = db.cursor()
    if type(data) is list:
        attribute_id = insert_values_to_attribute_table(data, c)
        insert_values_to_datapoints_table(data, c, attribute_id)
        db.commit()
        close_db(db)
        return "File uploaded succesfully!"
    else:
        close_db(db)
        return data

def get_attributes():
    """Get attributes from table."""
    db = access_db()
    c = db.cursor()
    sqlite_get_data = """SELECT Name, DisplayName FROM Attributes;"""
    attr_data = c.execute(sqlite_get_data)
    data_to_return = []
    for row in attr_data:
        data_to_return.append({"name": row[0], "displayName": row[1]})
    close_db(db)
    return data_to_return

def get_data(filters):
    """Get data from databases using filters."""
    db = access_db()
    c = db.cursor()
    data = c.execute(
        "SELECT ID, Unit FROM Attributes WHERE Name = ?", (filters["Argument"],))
    for bit in data:
        attr_id = bit[0]
        unit = bit[1]
    dates = filters["timeArgument"]
    if filters["timeIntervallType"] == "TIME_INTERVALL":
        start_date = dates[0]
        start_date = start_date.split("-")
        start_date[0] = int(start_date[0])
        start_date[1] = int(start_date[1])
        start_date[2] = int(start_date[2])
        end_date = dates[1]
        end_date = end_date.split("-")
        end_date[0] = int(end_date[0])
        end_date[1] = int(end_date[1])
        end_date[2] = int(end_date[2])
        sql_get_data = """SELECT * FROM Datapoints WHERE (Year >= ? AND Month >= ? AND (Day >= ? OR Month != ? OR Year != ?))
                         AND (Year < ? OR (Year = ? AND Month <= ? AND (Day <= ? OR Month != ?))) AND AttributeID = ?;"""
        datapoints_to_return = c.execute(sql_get_data, (start_date[0], start_date[1],
                                                        start_date[2], start_date[1], start_date[0],
                                                        end_date[0], end_date[0], end_date[1], end_date[2], end_date[1], attr_id))
        data_to_return = []
        for data in datapoints_to_return:
            data_to_return.append({"id": data[0],
                                   "year": int(data[1]),
                                   "month": int(data[2]),
                                   "day": int(data[3]),
                                   "time": int(data[4]),
                                   "value": data[5],
                                   "unit": unit
                                   })
        close_db(db)
        return data_to_return
    elif filters["timeIntervallType"] == "MONTH":
        month = dates[0]
        month = int(month)
        sql_get_data = "SELECT * FROM Datapoints WHERE Month = ? AND AttributeID = ?;"
        datapoints_to_return = c.execute(
            sql_get_data, (month, attr_id))
        data_to_return = []
        for data in datapoints_to_return:
            data_to_return.append({"id": data[0],
                                   "year": int(data[1]),
                                   "month": int(data[2]),
                                   "day": int(data[3]),
                                   "time": int(data[4]),
                                   "value": data[5],
                                   "unit": unit
                                   })
        close_db(db)
        return data_to_return
    close_db(db)

def insert_values_to_datapoints_table(data, c, attribute_id):
    """Seperate string of data into comma seperated values.
    and add to datapoint table.
    """
    date_index = 0
    for row in data:
        if len(row) != 0 and row[0] == "Datum" and row[1] == "Tid (UTC)":
            date_index = data.index(row)
    for row in range((date_index + 1), len(data)):
        date = datetime.datetime.fromisoformat(data[row][0])
        time = datetime.datetime.strptime(data[row][1], '%H:%M:%S')
        sql_insert = "INSERT INTO Datapoints(Year, Month, Day, Time, Value, AttributeID) VALUES ( ?, ?, ?, ?, ?, ?)"
        c.execute(sql_insert, (date.year, date.month, date.day,
                               time.hour, data[row][2], str(attribute_id)))

def insert_values_to_attribute_table(data, c):
    """Check for special characters and replace with english alphabet
    (if necessary) and add to datapoint table."""
    parameter_index_column = 0
    attribute_index_column = 0
    parameter_index_row = 0
    attribute_index_row = 0
    string_to_add = ""
    for row in data:
        if len(row) != 0 and "Parameternamn" in row:
            parameter_index_column = row.index("Parameternamn")
            parameter_index_row = data.index(row)
            break
    for row in data:
        if len(row) != 0 and "Enhet" in row:
            attribute_index_column = row.index("Enhet")
            attribute_index_row = data.index(row)
            break
    sql_insert = " INSERT INTO Attributes (Name, DisplayName, Unit) VALUES ( ?, ?, ?)"
    c.execute(sql_insert, (unidecode.unidecode(data[parameter_index_row + 1][parameter_index_column]).lower(
    ), data[parameter_index_row + 1][parameter_index_column], data[attribute_index_row + 1][attribute_index_column]))
    return c.lastrowid

def access_db():
    """Retrieve database."""
    db = sqlite3.connect(
        'instance\\weather_data.db',
        detect_types=sqlite3.PARSE_DECLTYPES
    )
    db.row_factory = sqlite3.Row
    return db

def close_db(db):
    """Close database connection."""
    db.close()

def initialize_database():
    """Creates database (if it does not exist)."""
    sql_create_datapoints_table = """CREATE TABLE IF NOT EXISTS Datapoints (
        ID integer PRIMARY KEY,
        Year integer,
        Month integer,
        Day integer,
        Time integer,
        Value real,
        AttributeID integer);"""
    sql_create_attributes_table = """CREATE TABLE IF NOT EXISTS Attributes(
        ID integer PRIMARY KEY,
        Name text,
        DisplayName text,
        Unit text
        );"""
    db = access_db()
    c = db.cursor()
    c.execute("PRAGMA foreign_keys = ON;")
    attributes_table = c.execute(sql_create_attributes_table)
    datapoints_table = c.execute(sql_create_datapoints_table)
    db.commit()
    list_of_values = file_reader(data_file)
    attribute_id = insert_values_to_attribute_table(list_of_values, c)
    insert_values_to_datapoints_table(list_of_values, c, attribute_id)
    db.commit()
    print('Database successfully created!')
    close_db(db)

if __name__ == '__main__':
    globals()[sys.argv[1]]()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2022/5/20 14:37
# @Author  : shiman
# @File    : drawing_utils.py
# @describe:

import torch
import colorsys

import matplotlib.pyplot as plt

def show_images(imgs, num_rows, num_cols, scale=1.5, titles=None):
    """展示多个图片"""
    figsize = (num_cols*scale, num_cols*scale)  # 定义展示图片尺寸
    _, axes = plt.subplots(num_rows, num_cols, figsize=figsize)
    axes = axes.flatten()
    for i, (img, ax) in enumerate(zip(imgs, axes)):
        if torch.is_tensor(img):
            ax.imshow(img.numpy())
        else:
            ax.imshow(img)
        ax.axes.get_xaxis().set_visible(False)  # 不展示XY轴刻度
        ax.axes.get_yaxis().set_visible(False)
        if titles:
            ax.set_title(titles[i])

def apply(image, aug, num_rows=2, num_cols=4, scale=1.5):
    """定义图像增广方法"""
    y = [aug(image) for _ in range(num_cols*num_rows)]
    show_images(y, num_rows, num_cols, scale=scale)

def get_color_bar(classes_num):
    hsv_tuple = [(1.0 * x/classes_num, 1, 1) for x in range(classes_num)]
    rgb_tuple = [colorsys.hsv_to_rgb(*x) for x in hsv_tuple]
    rgb_tuple = [(int(x[0]*255), int(x[1]*255), int(x[2]*255)) for x in rgb_tuple]


from tkinter import *
from PIL import ImageTk, Image
import cv2
import csv
import face_recognition
import numpy as np
import pickle
from numpy import savetxt
import datetime
import time
import os.path

# Capture frames from Camera #
cap = cv2.VideoCapture(0)
medText = 28

# Initialise Variables for locations and encodings #
face_locations = []
face_encodings = []
face_names = []

# Security Path and time Ranges #
image_path = "./security/"
start = datetime.time(10, 15)
end = datetime.time(11)

# Some code used from https://github.com/ageitgey/face_recognition/blob/master/examples/facerec_from_webcam_faster.py #

class Cam(Frame):
    def __init__(self, parent, event_name="", *args, **kwargs):
        Frame.__init__(self, parent, bg='black')
        self.title = "Unknown"

        # Shows name of user being detected #
        self.infoText = "Recognising Face: "
        self.info = Label(self, text=self.infoText, font=('Helvetica', medText), fg="blue", bg="black")
        self.userName = Label(self, text=self.title, font=('Helvetica', medText), fg="white", bg="black")
        if event_name != "enable":
            self.info.pack(side=LEFT, anchor=W)
        self.userName.pack(side=TOP, anchor=CENTER)
        self.lmain = Label(self)

        # Enable camera preview (enabled from Mirror.py) #
        if event_name == "enable":
            self.lmain.pack(side=TOP)
        self.video()

    def readUserName(self):
        return self.userName.cget("text")

    def video(self):
        process_this_frame = True
        # Reads pickle data from file that stores facial data #
        with open('./webserver/dataset_faces.dat', 'rb') as f:
            all_face_encodings = pickle.load(f)
        known_face_encodings = np.array(list(all_face_encodings.values()))
        known_face_names = list(all_face_encodings.keys())

        # Open CV resizing and changing the colour of the frame #
        ret, frame = cap.read()
        small_frame = cv2.resize(frame, (0, 0), fx=0.25, fy=0.25)
        rgb_small_frame = small_frame[:, :, ::-1]
        if process_this_frame:
            # Reading face locations and encodings #
            face_locations = face_recognition.face_locations(rgb_small_frame)
            face_encodings = face_recognition.face_encodings(rgb_small_frame, face_locations)
            face_names = []
            name = "No User Detected"
            self.userName.config(text = name)
            # Checking if Face measurements match to ones in dataset_faces.dat #
            for face_encoding in face_encodings:
                matches = face_recognition.compare_faces(known_face_encodings, face_encoding)
                name = "Unknown"
                self.userName.config(text = name)
                face_distances = face_recognition.face_distance(known_face_encodings, face_encoding)
                best_match_index = np.argmin(face_distances)
                if matches[best_match_index]:
                    name = known_face_names[best_match_index]
                    self.userName.config(text = name)
                else:
                    # Unknown user detected during time range #
                    timestamp = datetime.datetime.now().time()
                    check = start <= timestamp <= end
                    # If user is unknown and the time is in the time range - Save the frame #
                    if check == True:
                        timestamp2 = datetime.datetime.now()
                        timestampStr = timestamp2.strftime("%d_%b_%Y__%H_%M_%S")
                        timestampStrFilePath = timestamp2.strftime("%d_%b_%Y")
                        directory = image_path + timestampStrFilePath + "/"
                        if not os.path.isdir(directory):
                            os.mkdir(directory)
                        filename = timestampStr + ".jpg"
                        filepath = os.path.join(directory, filename)
                        cv2.imwrite(filepath, frame)
                face_names.append(name)
        process_this_frame = not process_this_frame
        # Used for Camera preview #
        for (top, right, bottom, left), name in zip(face_locations, face_names):
            top *= 4
            right *= 4
            bottom *= 4
            left *= 4
            cv2.rectangle(frame, (left, top), (right, bottom), (0, 0, 255), 2)
            cv2.rectangle(frame, (left, bottom - 35), (right, bottom), (0, 0, 255), cv2.FILLED)
            font = cv2.FONT_HERSHEY_DUPLEX
            cv2.putText(frame, name, (left + 6, bottom - 6), font, 1.0, (255, 255, 255), 1)
        img = Image.fromarray(frame)
        imgtk = ImageTk.PhotoImage(image=img)
        self.lmain.imgtk = imgtk
        self.lmain.configure(image=imgtk)
        self.lmain.after(1, self.video)


from setuptools import setup, find_packages

with open("README.md", encoding="utf-8") as f:
    long_des = str(f.read())

setup(
    name='flet_translator',
    version='1.2.7',
    author='SKbarbon',
    description='A package that help flet developers to make their apps support multiple languages',
    long_description=long_des,
    long_description_content_type='text/markdown',
    url='https://github.com/SKbarbon/flet_translator',
    install_requires=["flet", "sacremoses", "deep-translator"],
    packages=find_packages(),
    classifiers=[
        "Programming Language :: Python :: 3",
        "Operating System :: MacOS :: MacOS X",
        "Operating System :: Microsoft :: Windows"
    ],
)


#! /usr/bin/env python3
"""
eliminate some of the boiler-plate involved in running our "on hardware"
qemu tests, which are actually examples.
"""
import os
import shlex
import subprocess

def main(verbose=False):
    our_dir = os.path.dirname(os.path.realpath(__file__))
    for example in os.listdir(os.path.join(our_dir, "examples")):
        if example.startswith("test_"):
            command = 'cargo run --example {} --release --features="embedded-hal" --target thumbv7m-none-eabi'.format(
                os.path.splitext(example)[0]
            )
            print("running `{}`".format(command))
            subprocess.run(shlex.split(command), check=True)

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--verbose", "-v", action="store_true", help="verbose output")

    args = parser.parse_args()
    main(verbose=args.verbose)


# import model
import gp
import evaluate
import random
import matplotlib.pyplot as plt
import numpy as np
from tqdm import tqdm

if __name__ == '__main__':

    #model.exe()

    xmax = 352
    ymax = 1216
    a = []
    b = []

    # #preは２２５までおｋ
    # i=216
    # path = 'pre_test_results/data{}/'.format(i)
    # path1 = 'nomal_result/data216/'

    # xs = 180
    # xf = 220
    # ys = 450
    # yf = 600

    # error, var = evaluate.exe(path, path1, xs, xf, ys, yf)
    # error = error.reshape(1,-1)[0]
    # for e in error:
    #     if e>0:
    #         a.append(e)
    # breakpoint()

    # empty = [235, 250, 268, 274, 288, 291]

    for i in tqdm(range(0, 300)):
        # path = 'pre_test_results/data{}/'.format(i)
        # path1 = 'sample_result/data{}/'.format(i)
        # path1 = 'nomal_result/data{}/'.format(i)
        # path = 'test_results/data{}/'.format(i)
        path = 'penalty_only_test_results/data{}/'.format(i)

        # マスクする領域をランダムに決める
        # xs = random.randint(100,xmax)
        # xf = xs + 80
        # ys = random.randint(0,ymax)
        # yf = ys + 30
        # if random.randint(0,1)==1:
        #     xf = xs + 30
        #     yf = ys + 80
        # if xf >= xmax:
        #     tmp = xf - xs
        #     xf = xs
        #     xs = xf - tmp
        # if yf >= ymax:
        #     tmp = yf - ys
        #     yf = ys
        #     ys = yf - tmp

        # マスクする領域を固定する
        xs = 180
        xf = 220
        ys = 450
        yf = 600

        try:
            # 説明変数の不確実性を考慮したガウス過程回帰を実行
            # gp.exe_with_uncer(path, xs, xf, ys, yf)

            # ガウス過程回帰を実行する
            gp.exe(path, xs, xf, ys, yf, i)

            # 予測値の評価をする
            error, var = evaluate.exe(path, path, xs, xf, ys, yf)
            # error, corr = evaluate.t_test(path, path, xs, xf, ys, yf)
        except:
            print(i)
            continue
        error = error.reshape(1,-1)[0]
        var = var.reshape(1, -1)[0]
        a.append(error)
        b.append(var)

    # # # # # breakpoint()

    # a = np.array(a).reshape(1,-1)[0]
    # b = np.array(b).reshape(1,-1)[0]
    a = np.array(a)
    b = np.array(b)
    np.save('ttest_pe_only_error', a)
    np.save('ttest_pe_only_corr', b)
    # plt.figure(figsize=(16, 5))
    # plt.scatter(b,a)
    # plt.xlabel('variance')
    # plt.ylabel('relative error')
    # plt.savefig('eval3.png' , format="png")
    # # breakpoint()


""" Impersonate myself.
"""
import argparse
import sys
import time
from collections import OrderedDict
from multiprocessing.dummy import Pool
from pathlib import Path
from typing import Dict, List

from slackit import Event, Message, Receiver
from slackit._config import Config, configure
from slackit.roam import today
from slackit.services import SERVICE_REGISTRY

def _report(receiver: Receiver, messages: List[Message], delay: int) -> None:
    print(f"Sending {messages} to receiver {receiver} after {delay} seconds.")

def _shoot(receiver: Receiver, messages: List[Message], delay: int) -> None:
    """For multiprocessing."""
    time.sleep(delay)
    service = SERVICE_REGISTRY[receiver.service_name](receiver.receiver_id)
    service.send_many(messages=messages)

def main() -> None:
    """Impersonate myself."""
    parser = argparse.ArgumentParser(description=main.__doc__)
    parser.add_argument(
        "-c",
        "--config",
        type=argparse.FileType("r"),
        required=True,
    )
    parser.add_argument(
        "-e",
        "--event",
        required=True,
    )
    parser.add_argument(
        "--check-date",
        action="store_true",
        help="Only run if today's date matches the config.",
    )

    args = parser.parse_args()

    try:
        configure(Path(args.config.name))

        event_spec = Config.get(args.event)

        if args.check_date and today() not in str(event_spec):
            print(f"Couldn't find {today()} in the event spec. Aborting!")
            sys.exit(0)

        event = Event.from_spec(event_spec)
        per_recv: Dict[Receiver, List[Message]] = OrderedDict()

        for r_spec, m_spec in event.message_gen:
            _messages = per_recv.setdefault(Receiver.from_spec(r_spec), [])
            _messages.append(Message.from_spec(m_spec))

        if not per_recv:
            print("Nothing to do!")
            sys.exit(0)

        _args = list(zip(per_recv.keys(), per_recv.values(), event.delay_gen()))
        for _arg in _args:
            _report(*_arg)

        with Pool(len(per_recv)) as pool:
            pool.starmap(_shoot, _args)

    except Exception as exc:  # pylint: disable=broad-except
        sys.exit(exc)

    print("Done!")

if __name__ == "__main__":
    main()


from flask import Flask, request, jsonify
import traceback
import predict

app = Flask(__name__)

@app.route('/predict', methods=['POST'])
def run():
    try:
        data = request.get_json(force=True)
        input_params = data['input']
        result =  predict.predict(input_params)
        return jsonify({'prediction': result})
    except Exception as e:
        print(traceback.format_exc())
        return jsonify({'error': str(e)})

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)


from airflow.hooks.postgres_hook import PostgresHook
from airflow.models import BaseOperator
from airflow.utils.decorators import apply_defaults
from helpers import SqlQueries

class LoadDimensionOperator(BaseOperator):

    ui_color = '#80BD9E'

    @apply_defaults
    def __init__(self,
                 redshift_conn_id = "",
                 table = "",
                 sql = "",
                 truncate=False,
                 *args, **kwargs):

        super(LoadDimensionOperator, self).__init__(*args, **kwargs)
        self.redshift_conn_id = redshift_conn_id
        self.table = table
        self.sql = sql
        self.truncate = truncate

    def execute(self, context):
        postgres_hook = PostgresHook(postgres_conn_id=self.redshift_conn_id)
        if self.truncate:
            self.log.info(f"Truncate dimension table: {self.table}")
            postgres_hook.run(f"Truncate table {self.table}")
        self.log.info(f"Insert data from fact table into {self.table}")
        sql_statement = f"INSERT INTO {self.table} ({self.sql})"
        postgres_hook.run(sql_statement)
        self.log.info(f"Task {self.task_id} successful")


from google.api_core import retry
import google.generativeai as palm
import os
import re
import config
import os
import tenacity
from time import sleep

# Set PaLM API key from environment or config file
api_key = os.environ.get("PALM_API_KEY")
if (api_key is None or api_key == "") and os.path.isfile(os.path.join(os.getcwd(), "keys.cfg")):
    cfg = config.Config('keys.cfg')
    api_key = cfg.get("PALM_API_KEY")
palm.configure(api_key=api_key)

@retry.Retry()
def retry_chat(**kwargs):
    return palm.chat(**kwargs)

@retry.Retry()
def retry_reply(x, arg):
    return x.reply(arg)

@tenacity.retry(wait=tenacity.wait_random_exponential(min=150, max=500), stop=tenacity.stop_after_attempt(6))
def generate_text(*args, **kwargs):
    return palm.generate_text(*args, **kwargs)

# PALM Chat
class PalmChat:
    def __init__(self, model="models/chat-bison-001"):
        self.defaults = {
            'model': model,
            'temperature': 0,
            'top_p': 1,
            'candidate_count': 1
        }
        self.response = None

    def init_chat(self, context="You are a helpful assisstant", init_message="Hello"):
        # initialize chat
        response = None
        response = retry_chat(
        **self.defaults,
        context=context,
        messages=init_message
        )
        self.response = response

    def reply(self, reply):
        self.response = retry_reply(self.response, reply)
        while self.response.last == None:
            self.response = retry_reply(self.response, reply)

    def get_response(self):
        output = self.response.last
        # clean output
        output = re.sub("\n", " ", output)
        return output

def PalmCompletion(phrase, model="models/text-bison-001", num_samples=1):
    # Request configuration disabling all safety settings to prevent blocking
    config = {
        'model': model,
        'temperature': 0,
        'top_p': 1,
        'candidate_count': num_samples,
        'safety_settings': [{"category":"HARM_CATEGORY_DEROGATORY","threshold":4},{"category":"HARM_CATEGORY_TOXICITY","threshold":4},{"category":"HARM_CATEGORY_VIOLENCE","threshold":4},{"category":"HARM_CATEGORY_SEXUAL","threshold":4},{"category":"HARM_CATEGORY_MEDICAL","threshold":4},{"category":"HARM_CATEGORY_DANGEROUS","threshold":4}],
    }
    # to prevent rate limiting
    sleep(2)
    response = generate_text(
        **config,
        prompt=phrase
    )
    output = response.result
    # clean output
    if isinstance(output, str):
        output = re.sub("\n", " ", output)
    return output

if __name__ == "__main__":
    chat = PalmChat()
    chat.init_chat(init_message="I have 3 apples and 10 bananas. How many fruits do I have in total?")
    for i in range(5):
        print(chat.get_response())
        x = input("> ")
        chat.reply(x)


#region Information
'''
This module contains useful functions for building geometry
nodes node trees inside of Blender.
'''
#endregion
#region Imports
import bpy
from easybpy import *
#endregion
#region Main Class
class GeometryNodes:

    # Variables
    tree = None
    control = None
    group_input = None
    group_output = None

    # Class init
    def __init__(self, name = "New Node Object"):
        control = self.create_control_object(name)

    def create_control_object(self, name):
        o = create_object(name, "Node Objects")
        select_only(o)
        bpy.ops.node.new_geometry_nodes_modifier()

        # Essential variables for the class.
        self.tree = self.get_node_tree(o)
        self.group_input = self.tree.nodes["Group Input"]
        self.group_output = self.tree.nodes["Group Output"]
        return o

    def get_node_tree(self, object):
        return object.modifiers["GeometryNodes"].node_group

    def add_object(self, ref = None):
        objnode = self.tree.nodes.new(type="GeometryNodeObjectInfo")
        if ref is not None:
            objref = get_object(ref)
            objnode.inputs[0].default_value = objref
        return objnode

    def output_geometry(self, ref):
        node = get_node(self.tree.nodes,ref)
        index = get_index_of_output(node, "Geometry")
        create_node_link(node.outputs[index], self.group_output.inputs[0])

    def connect_geometry(self, first, second):
        first_index = get_index_of_output(first, "Geometry")
        second_index = get_index_of_input(second, "Geometry")
        create_node_link(first.outputs[first_index], second.inputs[second_index])

    def boolean(self, first, second):
        boolnode = self.tree.nodes.new(type="GeometryNodeBoolean")
        first_index = get_index_of_output(first, "Geometry")
        second_index = get_index_of_output(second, "Geometry")
        create_node_link(boolnode.inputs[0], first.outputs[first_index])
        create_node_link(boolnode.inputs[1], second.outputs[second_index])
        return boolnode

    def join_geometry(self, first, second):
        joinnode = self.tree.nodes.new(type="GeometryNodeJoinGeometry")
        first_index = get_index_of_output(first, "Geometry")
        second_index = get_index_of_output(second, "Geometry")
        create_node_link(joinnode.inputs[0], first.outputs[first_index])
        create_node_link(joinnode.inputs[1], second.outputs[second_index])
        return joinnode

    def transform(self, objsource):
        transformnode = self.tree.nodes.new(type="GeometryNodeTransform")
        obj_index = get_index_of_output(objsource, "Geometry")
        create_node_link(objsource.outputs[obj_index],transformnode.inputs[0])
        return transformnode
#endregion


import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.animation as manimation
import matplotlib as mpl

def simulate_stochastic_frequency_dynamics(graph, steps, p, bias):

	#initialize a random initial state
	biased_p = (1+bias)/2
	initial_state = [1 if p < biased_p else 0 for p in np.random.rand(len(graph.nodes))]

	states = {idx:state for idx,state in enumerate(initial_state)}
	nx.set_node_attributes(graph, states, 'state')

	states = nx.get_node_attributes(graph,'state')
	history = [states]

	for s in range(steps):

		states = history[-1]
		# also asynchronous
		for node in graph.nodes:

			nbr_states = [states[n] for n in graph.neighbors(node)]
			current_sign = graph.nodes[node]['state']
			if np.random.rand() < stochastic_frequency_probability_of_one(current_sign, p, nbr_states):
				graph.nodes[node]['state'] = 1
			else:
				graph.nodes[node]['state'] = 0

		history.append(nx.get_node_attributes(graph,'state'))

	# pos = nx.spring_layout(graph)
	# save_file = "Images/StochFreq/"
	# for i in range(len(history)):
	# 	plt.clf()
	# 	colormap = ['purple' if node == 1 else 'yellow' for node in history[i].values()]
	# 	nx.draw(graph,node_color = colormap, pos = pos)
	# 	plt.savefig("{}{:04d}.png".format(save_file,i))

	return history

def stochastic_frequency_probability_of_flip(current_sign, p, nbr_states):
	q = (1-p)/2
	prob = q + p*sum(np.array(nbr_states)== current_sign)/len(nbr_states)

	return prob

def stochastic_frequency_probability_of_one(current_sign, p, nbr_states):
	q = (1-p)/2
	prob = q + p*sum(np.array(nbr_states) == current_sign)/len(nbr_states)

	if current_sign == 0:
		return prob
	else:
		return 1 - prob

def simulate_mean_field_dynamics(T,kappa,p,p_0):

	ps = [p_0]
	q = (1-p)/2

	for i in range(T-1):
		old_p = ps[-1]

		p_given_0 = q + p*old_p
		p_given_1 = q + p*(old_p)

		new_p = p_given_1*old_p + p_given_0*(1-old_p)

		ps.append(new_p)

	return ps


import collections

import logging

import datetime

from aiokts.util.json_utils import JsonSerializable

class Field:
    def __init__(self, default, private):
        self.name = None
        self.model = None
        self.default = default
        self.private = private

    def transform_in(self, value):
        return value

    def transform_to_json(self, value):
        return value

    def __call__(self, value):
        return self.transform_in(value)

class StringField(Field):
    def __init__(self, default=None, private=False):
        super().__init__(default, private)

    def transform_in(self, value):
        return str(value)

class IntField(Field):
    def __init__(self, default=None, private=False):
        super().__init__(default, private)

    def transform_in(self, value):
        return int(value)

class BooleanField(Field):
    def __init__(self, default=None, private=False):
        super().__init__(default, private)

    def transform_in(self, value):
        if value == 'true':
            value = True
        elif value == 'false':
            value = False
        return bool(value)

class UnixTimestampField(Field):
    def __init__(self, default=None, private=False):
        super().__init__(default, private)

    def transform_in(self, value):
        value = int(value)
        return datetime.datetime.fromtimestamp(value)

class IntEnumField(Field):
    def __init__(self, enum_cls, default=None, private=False, json_name=False):
        super().__init__(default, private)
        self.enum_cls = enum_cls
        self.json_name = json_name

    def transform_in(self, value):
        return self.enum_cls(value)

    def transform_to_json(self, value):
        if self.json_name:
            return value.name
        return value.value

class DictField(Field):
    def __init__(self, default=None, private=False):
        super().__init__(default, private)

    def transform_in(self, value):
        assert isinstance(value, dict), \
            'value is not dict (but {}) for {}.{}'.format(
                type(value), self.model.__name__, self.name
            )
        return value

class ForeignModelField(Field):
    def __init__(self, model_cls, default=None, private=False):
        super().__init__(default, private)
        self.model_cls = model_cls

    def transform_in(self, value):
        return self.model_cls.parse(value)

    def transform_to_json(self, value):
        return value.__to_json__()

class DoesNotExistBase(Exception):
    MODEL_CLS = None

    def __init__(self, entity_id=None, message=None):
        self.entity_id = entity_id
        if message is None:
            self.message = \
                "Entity of type '{}' with id {} not found".format(
                    self.MODEL_CLS.__name__, self.entity_id)
        else:
            self.message = message

    def __str__(self):
        return self.message

class ModelMetaclass(type):
    @classmethod
    def __prepare__(mcs, name, bases):
        return collections.OrderedDict()

    def __new__(mcs, class_name, bases, class_dict):
        if class_name != 'Model':
            fields = collections.OrderedDict()
            for name, value in class_dict.items():
                if not name.startswith('__') \
                        and isinstance(value, Field):
                    fields[name] = value
            for name in fields:
                del class_dict[name]

            class_dict['_fields'] = fields

        return super().__new__(mcs, class_name, bases, class_dict)

    def __init__(cls, class_name, bases, class_dict):
        class DoesNotExist(DoesNotExistBase):
            MODEL_CLS = cls
        cls.DoesNotExist = DoesNotExist
        fields = class_dict['_fields']
        if fields is not None:
            for name, f in fields.items():
                f.name = name
                f.model = cls
        super().__init__(class_name, bases, class_dict)

class Model(JsonSerializable, metaclass=ModelMetaclass):
    _fields = None
    DoesNotExist = None
    LOGGER = logging.getLogger('aiokts.models')

    def __init__(self, *args, **kwargs):
        i = 0

        used_kwargs = set()
        for name, field in self._fields.items():
            setattr(self, name, field.default)
            if len(args) <= i:
                # supplying kw arguments
                if name in kwargs:
                    v = kwargs[name]
                    if v is not None:
                        try:
                            v = field.transform_in(v)
                        except Exception as e:
                            raise Exception('{} for field `{}` in {}'.format(
                                str(e), name, self.__class__
                            ))
                        transformer = 'transform_{}'.format(name)
                        if hasattr(self, transformer) \
                                and callable(getattr(self, transformer)):
                            v = getattr(self, transformer)(v)
                    setattr(self, name, v)
                    used_kwargs.add(name)
            else:
                # supplying positional arguments
                v = args[i]
                if v is not None:
                    v = field.transform_in(v)
                setattr(self, name, v)
                i += 1

        if len(args) > i:
            self.LOGGER.warning(
                'Too many positional arguments passed. '
                'Expected %s max', len(self._fields)
            )
            return

        extra_kwargs = set(kwargs.keys()) - used_kwargs
        if len(extra_kwargs) > 0:
            self.LOGGER.warning(
                'Unknown fields passed: %s', extra_kwargs)

    def __to_json__(self):
        res = {}
        for name, field in self._fields.items():
            if field.private:
                continue
            v = getattr(self, name)
            if v is not None:
                v = field.transform_to_json(v)
                transformer = 'transform_json_{}'.format(name)
                if hasattr(self, transformer) \
                        and callable(getattr(self, transformer)):
                    v = getattr(self, transformer)(v)
            res[name] = v
        return res

    def __repr__(self):
        fields = ['{}={}'.format(k, getattr(self, k)) for k in self._fields]
        return '<{} {}>'.format(self.__class__.__name__, ' '.join(fields))

    @classmethod
    def parse(cls, d: dict):
        if d is None:
            return None
        return cls(**d)

    @classmethod
    def parse_list(cls, l: list):
        if l is None or len(l) == 0:
            return []
        return list(map(lambda d: cls(**d), l))


#!/usr/bin/env python3

import subprocess
import shutil # cp/mv files
import os.path # check file exist
import glob # ls *
import threading
import sys
import argparse
import time

sem = threading.Semaphore()

class Compression():
    """Compression is a wrapper class of a compression algorithm"""

    def __init__(self, command, input_file, output_file, replace, temp=False):
        """Init the compress algorithm

        :command: The command to build the output file
        :input_file: Input file name
        :output_file: Output file name
        :replace: Does the command remove the input

        """
        self.command = command
        self._input_file = input_file
        self.output_file = output_file
        self._replace = replace
        self._output = ""
        self._execed = False
        self._temp = temp
        self.fs = 0
        self.return_code = 0

    def format_command(self, input_file):
        return self.command.format(input_file, self.output_file)

    def exe(self):
        if not os.path.isfile(self._input_file):
            raise FileNotFoundError(f"No such file or directory: '{self._input_file}'")
        if self._replace:
            shutil.copyfile(self._input_file, self._input_file+".bak")
        input_file = self._input_file
        if self._temp:
            sem.acquire()
            input_file = "temp.temp"
            shutil.copyfile(self._input_file, input_file)
        self._time_start = time.time()
        self._process = subprocess.Popen(
            [self.format_command(input_file)],
            shell=True,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE
        )
        self._time_end = time.time()
        self._execed = True
        self.return_code = self._process.wait()
        self.stdout, self.stderr = self._process.communicate()
        if self._temp:
            files = glob.glob("temp.temp.*")
            shutil.move(files[0], self.output_file)
            if not self._replace:
                os.remove("temp.temp")
            sem.release()
        self.fs = os.path.getsize(self.output_file)
        if self._replace:
            shutil.move(self._input_file+".bak", self._input_file)
        print(self)

    def __str__(self):
        if self._execed:
            c = self.command.split(' ')[0]
            return f"{c},{self.output_file},{self.fs},{self.return_code},{self._time_end-self._time_start},"
        return f"The command '{self.format_command(self._input_file)}' has not been run yet"

def generate_compression(input_file):
    return [
                Compression("7z a -t7z {1} {0}", input_file, f"{input_file}.7z", False),
                Compression("bzip2 {}", input_file, f"{input_file}.bz2", True, temp=True),
                Compression("compress -f {}", input_file, f"{input_file}.Z", True, temp=True),
                Compression("lzma {}", input_file, f"{input_file}.lzma", True, temp=True),
                Compression("pack {} {}", input_file, f"{input_file}.packed", False),
                Compression("rar a -r {1} {0}", input_file, f"{input_file}.rar", False),
                Compression("tar cfz {1} {0}", input_file, f"{input_file}.tar", False),
                Compression("xz {0}", input_file, f"{input_file}.xz", True, temp=True),
                Compression("zip {1} {0}", input_file, f"{input_file}.zip", False),
            ]

def run(input_file, count, thread=False):
    algos = generate_compression(input_file)
    for algo in algos:
        if thread:
            thr = threading.Thread(target=algo.exe)
            thr.start()
        else:
            algo.exe()
        if count > 0:
            run(algo.output_file, count-1)
        if thread:
            thr.join()
        os.remove(algo.output_file)

def create_parser():
    parser = argparse.ArgumentParser(description="Benchmark compressions algorithms")
    parser.add_argument("-f", "--file", help="Specify the input file name")
    parser.add_argument("-c", "--count", default=0,
            help="How deep is your recurtion")
    return parser

def main(argv):
    args = create_parser().parse_args(argv[1:])
    if not args.file:
        raise ValueError("No file provided, see --help")
    print("algo,command,file size,return code,exec time,")
    print(f"none,none,{os.path.getsize(args.file)},0,0,")
    run(args.file, count=int(args.count))

if __name__ == "__main__":
    main(sys.argv)


from typing import cast

import pytest
from recon.stats import get_ner_stats
from recon.types import Example, NERStats, Span, Token
from recon.validation import filter_overlaps, upcase_labels

@pytest.fixture()
def messy_data():
    return [
        {
            "text": "Denver, Colorado is a city.",
            "spans": [
                {"start": 0, "end": 6, "label": "GPE"},
                {"start": 0, "end": 16, "label": "LOC"},
            ],
            "meta": "Cities Data",
        }
    ]

def test_upcase_labels(example_corpus):
    stats = cast(NERStats, get_ner_stats(example_corpus.train))
    assert "skill" in stats.n_annotations_per_type
    assert "product" in stats.n_annotations_per_type
    assert "SKILL" in stats.n_annotations_per_type

    example_corpus._train.apply_("recon.v1.upcase_labels")
    fixed_stats = cast(NERStats, get_ner_stats(example_corpus.train))
    assert "skill" not in fixed_stats.n_annotations_per_type
    assert "product" not in fixed_stats.n_annotations_per_type

def test_filter_overlaps():
    def get_test_example(span_offsets):
        spans = []
        for so in span_offsets:
            spans.append(Span(text="x" * (so[1] - so[0]), start=so[0], end=so[1], label=so[2]))
        return Example(text="x" * 1500, spans=spans)

    def spans_to_offsets(spans):
        return [(span.start, span.end, span.label) for span in spans]

    test_entities = [(0, 5, "ENTITY"), (6, 10, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(0, 5, "ENTITY"), (6, 10, "ENTITY")]

    test_entities = [(0, 5, "ENTITY"), (5, 10, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(0, 5, "ENTITY"), (5, 10, "ENTITY")]

    test_entities = [(0, 5, "ENTITY"), (4, 10, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(4, 10, "ENTITY")]

    test_entities = [(0, 5, "ENTITY"), (0, 5, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(0, 5, "ENTITY")]

    test_entities = [(0, 5, "ENTITY"), (4, 11, "ENTITY"), (6, 20, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(0, 5, "ENTITY"), (6, 20, "ENTITY")]

    test_entities = [(0, 5, "ENTITY"), (4, 7, "ENTITY"), (10, 20, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(0, 5, "ENTITY"), (10, 20, "ENTITY")]

    test_entities = [(1368, 1374, "ENTITY"), (1368, 1376, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(1368, 1376, "ENTITY")]

    test_entities = [
        (12, 33, "ENTITY"),
        (769, 779, "ENTITY"),
        (769, 787, "ENTITY"),
        (806, 811, "ENTITY"),
    ]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [
        (12, 33, "ENTITY"),
        (769, 787, "ENTITY"),
        (806, 811, "ENTITY"),
    ]

    test_entities = [
        (189, 209, "ENTITY"),
        (317, 362, "ENTITY"),
        (345, 354, "ENTITY"),
        (364, 368, "ENTITY"),
    ]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [
        (189, 209, "ENTITY"),
        (317, 362, "ENTITY"),
        (364, 368, "ENTITY"),
    ]

    test_entities = [(445, 502, "ENTITY"), (461, 473, "ENTITY"), (474, 489, "ENTITY")]
    result = filter_overlaps(get_test_example(test_entities))
    assert spans_to_offsets(result.spans) == [(445, 502, "ENTITY")]


import json
import pickle
from urllib.request import urlopen

import altair as alt
import lightgbm as lgb
import pandas as pd
import shap
import streamlit as st
from matplotlib import pyplot as plt

END_POINT = 'http://127.0.0.1:5000/'

@st.cache
def load_main_data():
    """

    :return: the sample data frame from backend
    """
    json_url = urlopen(END_POINT + 'get_clients_df')
    data = json.loads(json_url.read())
    return pd.read_json(data)

@st.cache
def load_all_clients():
    """

    :return: all clients ids from backend
    """
    json_url = urlopen(END_POINT + 'get_all_clients')
    return json.loads(json_url.read())

@st.cache
def load_stats():
    """

    :return: stats data from backend
    """
    json_url = urlopen(END_POINT + 'get_stats')
    data = json.loads(json_url.read())
    return pd.read_json(data)

@st.cache
def load_model():
    """

    :return: loading the model
    """
    return pickle.load(open('../output/best_estimator.pkl', 'rb'))

@st.cache
def load_shap_explainer():
    """

    :return: a built shap explainer
    """
    data = load_main_data()
    data = data.drop(columns=['PREDICT', 'PREDICT_PROBA', 'SK_ID_CURR'])
    explainer = shap.TreeExplainer(load_model())
    shap_values = explainer.shap_values(data)
    return explainer, shap_values

def get_lime_data(id):
    """

    :param id: the client id
    :return: the lime explanation for top features
    """
    json_url = urlopen(END_POINT + 'get_lime/' + str(id))
    return pd.read_json(json.loads(json_url.read()))

def get_client_prediction(id):
    """

    :param id: client id
    :return: prediction / prediction proba for the client
    """
    json_url = urlopen(END_POINT + 'get_client_prediction/' + str(id))
    return json.loads(json_url.read())

def get_knn_data(id):
    """

    :param id: the client id
    :return: the nearest neighbors avergage on different features
    """
    json_url = urlopen(END_POINT + 'get_knn/' + str(id))
    return pd.read_json(json.loads(json_url.read()))

def build_stats_data_frame(id, features, data):
    """

    :param id: the client ids
    :param features: the columns on which display stats (lime data)
    :param data: the whole dataframe
    :return: a stats dataframe built to display a chart
    """
    df = pd.DataFrame(columns=['Feature', 'Type', 'Mean'])
    knn_df = get_knn_data(id)
    stats_df = load_stats()
    client_data = data[data['SK_ID_CURR'] == id]
    i = 0
    predict = get_client_prediction(id)['prediction']
    stats_for_same_profile = stats_df[stats_df['TYPE'] == str(predict)]
    stats_for_all = stats_df[stats_df['TYPE'] == 'ALL']
    for feature in features:
        df.loc[i] = [feature, 'Client', client_data[feature].iloc[0]]
        i += 1
        df.loc[i] = [feature, 'Near clients', knn_df[feature].iloc[0]]
        i += 1
        df.loc[i] = [feature, 'Same clients', stats_for_same_profile[feature].iloc[0]]
        i += 1
        df.loc[i] = [feature, 'All clients', stats_for_all[feature].iloc[0]]
        i += 1

    return df

def main():
    # add a title
    st.title('Prêt à dépenser : Implémenter un modèle de scoring')
    st.subheader('By Salaheddine E.G : Openclassroom student')

    # data preview
    data = load_main_data()
    st.subheader('Apercu des données')
    st.write('Aperçu des données pour les gens éligibles à un crédit : ')
    st.write(data.head(5))
    st.write('...Et pour ceux pas éligibles : ')
    st.write(data.tail(5))

    # get data from api to choose a client
    all_clients = load_all_clients()
    st.subheader('Affichage des données pour un client')

    id = st.selectbox('Choisissez un client : ', all_clients)
    # display the prediction proba
    predict = get_client_prediction(id)
    st.info('Probabilité de défaut du client %.2f %%' % (predict['prediction_proba'] * 100))

    # Display lime explainer using altair chart
    st.subheader('Explication lime du résultat pour le client')
    lime_data = get_lime_data(id)
    lime_data.dropna(how='all', axis=1, inplace=True)
    lime_draw = lime_data.T
    lime_draw.rename(columns={lime_draw.columns[0]: 'value'}, inplace=True)
    lime_draw['feature'] = lime_draw.index

    st.write(alt.Chart(lime_draw).mark_bar().encode(
        x='value',
        y='feature',
        color=alt.condition(
            alt.datum.value > 0,
            alt.value('coral'),  # The positive color
            alt.value('green')  # The negative color
        )
    ).properties(width=600, height=400))

    # display clients data vs similar clients, clients with the same target value, and vs all clients
    st.subheader('Comparaison du résultat de clients par rapport à d\'autres clients')
    stats = build_stats_data_frame(id, lime_data.columns, data)

    first_features = lime_data.columns[0:5]
    last_features = lime_data.columns[5:]

    st.write(alt.Chart(stats[stats['Feature'].isin(first_features)]).mark_bar().encode(
        x='Type:N',
        y='Mean:Q',
        color='Type:N',
        column='Feature:N'
    ).properties(width=200, height=400))

    st.write(alt.Chart(stats[stats['Feature'].isin(last_features)]).mark_bar().encode(
        x='Type:N',
        y='Mean:Q',
        color='Type:N',
        column='Feature:N'
    ).properties(width=200, height=400))

    # display lgb model importance
    st.subheader('Explication du modèle ')
    model = load_model()
    lgb.plot_importance(model, max_num_features=10)
    plt.title('Light GBM importance')
    st.pyplot(bbox_inches='tight')
    plt.clf()

    # display shap explanation
    st.subheader('Explication du modèle en utilisant SHAP')
    explainer, shap_values = load_shap_explainer()
    plt.title('Importance des features en utilisant shap')
    shap_data = data.drop(columns=['PREDICT', 'PREDICT_PROBA', 'SK_ID_CURR'])
    shap.summary_plot(shap_values, shap_data, plot_type="bar", show=False)
    st.pyplot(bbox_inches='tight')
    plt.clf()

if __name__ == '__main__':
    main()


import json
import os
import sys

from src.GracefulKiller import GracefulKiller
from src.youtube.songEnricher import enrich_song

json_file = os.path.join(os.path.dirname(__file__), "../../data/data.json")
simplified_file = os.path.join(os.path.dirname(__file__), "../../data/simplified.json")
jsonToSimplify = {}
simplified = []
with open(json_file) as json_data:
    jsonToSimplify = json.load(json_data)

def main(songs_json):
    song_entries = songs_json['entries']
    for key in song_entries:
        for songIndex in range(0, len(song_entries[key])):
            entry = song_entries[key][songIndex]

            kind = "none"
            url = entry['url']
            title = "%s - %s" % (entry['artist'], entry['song'])
            if "youtube" in entry and "snippet" in entry['youtube']:
                title = entry["youtube"]["snippet"]["title"]
                kind = entry["youtube"]["id"]["kind"]

                if "videoId" in entry["youtube"]["id"]:
                    url = entry["youtube"]["id"]["videoId"]

                if "playlistId" in entry["youtube"]["id"]:
                    url = entry["youtube"]["id"]["playlistId"]

            simplified.append({
                "genre": entry["genre"],
                "id": entry["unique_id"],
                "title": title,
                "kind": kind,
                "url": url
            })
    with open(simplified_file, 'w') as outfile:
        json.dump(simplified, outfile, indent=4)

if __name__ == '__main__':
    main(jsonToSimplify)


#!/usr/python3
import os, time as t
#Colors
white = '\033[0;37;40m'
green = '\033[1;32;40m'
redyellow = '\033[1;33;41m'
red = '\033[1;31;40m'
load = '\033[5;31;40m'
cyan = '\033[1;36;40m'
yellow = '\033[1;33;40m'
blue = '\033[1;34;40m'
plus =  "[+]" + green
credit = ['Spider Anongreyhat', 'D3cryptor', 'AnonyminHack5']
t.sleep(1)
os.system("clear")
print(red + "Disclaimer:Use this script for educational purposes only.\n" + redyellow
+ "Spider Anongreyhat" + red + " won\'t be responsible for any shit you used this script for" + white)
t.sleep(3)
os.system("clear")
t.sleep(2)
try:
    print(cyan + "Cheking if requirement has been installed" + load + ".." + white)
    t.sleep(3)
    import requests
    import pyfiglet

except ModuleNotFoundError:
    print(red + "Ouch... :( \nSome requirements are  not found!\nBut don\'t worry..requirements will be installed automatically" + white)
    t.sleep(3)
    os.system("clear")
    t.sleep(2)
    print(cyan + "Installing requirements" + load + "..." + white)
    os.system("""
    pip install pyfiglet
    pip install requests
    """)
    print(green + "Run python3 SCV again")
    exit()
print("All done!!!")
t.sleep(2)
os.system("clear")
banner = pyfiglet.figlet_format("SourceCode-Viewer")
def loop():
    os.system('clear')
    print(yellow + banner)
    print(red +"version 2.6".center(60) + white)
    print(yellow + plus + " Tool Name: Source Code Viewer\n" + yellow + plus +  " Creator: Spider Anongreyhat\n" + yellow + plus + " Team: TermuxHackz Society\n" + yellow + plus + " Github: https://github.com/spider863644\n" + yellow + plus + " WhatsApp: +2349052863644" + white)
    print(" ")
    print(blue + """
[1] Get source code of a webpage and save {https  site only}
[2] Get source code of a webpage and save {http sites only}
[3] Update script
[4] Join my WhatsApp group to colabborate with me
[5] Exit Program
""" )
    try:
        choice = int(input(redyellow + "Choose a valid option " + white))
    except ValueError:
        print(red + "Only integers are allowed!" + white)
        t.sleep(1.5)
        loop()
    os.system("clear")
    if choice == 1:
        print(green, credit, white)
        Url = input(redyellow + "Enter the url of the webpage you wanna get its source code " + white)
        filename = input(green + "Enter filename without an extention: ")
        if "https://" not in Url:
            print(red + "Invalid url!")
            t.sleep(2)
            loop()
        url = requests.get(Url)
        os.system("clear")
        print(redyellow + "Getting Source Code of " + Url + load + "..." + white)
        t.sleep(1)
        try:
            file = open(filename + ".html", "x")
        except:
            print(red + "File already exist!")
            t.sleep(3)
            loop()
        file.write((url.text))
        file.close()
        print(blue + url.text)
        print(green + "\n\nFile has been saved as " + filename + ".html")
    elif choice == 2:
        print(green, credit, white)
        Url = input(redyellow + "Enter the url of the webpage you wanna get its source code" + white)
        filename = input(green + "Enter filename: ")
        if "http://" not in Url:
            print(red + "invalid url!" + white)
            t.sleep(4)
            loop()
        url = requests.get(Url)
        os.system("clear")
        print(redyellow + "Getting Source code of " + Url + load + "..." + white)
        t.sleep(1)
        try:
            file = open(filename + ".html", "x")
        except:
            print(red + "File already exist!")
            t.sleep(3)
            loop()
        file.write((url.text))
        file.close()
        print(blue + url.text)
        move = "mv " + filename + ".html /sdcard"
        os.system(move)
        print(green + "\n\nFile has been saved as " + filename + ".html in internal home storage" )
    elif choice == 3:
        os.system("clear")
        t.sleep(4)
        print(redyellow + "Updating Script" + load + "..." + white)
        t.sleep(3)
        os.system("""
        cd
        rm  SourceCode-Viewer_pro
        cd $HOME
        git clone https://github.com/spider863644/SourceCode-Viewer_pro
        """)
        print(cyan + """Type the following Commands below
        cd
        cd SourceCode-Viewer_pro
        """)
        exit()
    elif choice == 4:
        os.system("clear")
        print(redyellow + "Redirecting to my WhatsApp group" + load + "..." + white)
        t.sleep(4)
        os.system("xdg-open https://chat.whatsapp.com/IWqGOsJPjkp2vXcMSJKYns")
        loop()
    elif choice == 5:
        print(red + "Thanks for you using my script\nAlways report any issue you face when using this tool so we can make a better version")
        t.sleep(3)
        exit()
    else:
        print(Fore.RED + "Invalid option")
        exit()
    cont = input(redyellow + "Do you wanna continue?[y/n] " + white)
    if cont == "y" or cont == "Y":
        loop()
loop()


from functools import reduce

# 统计大小写字母个数
def statistics(string):
    upper_count = 0
    lower_count = 0
    for elem in string:
        if elem.isupper():
            upper_count += 1
        elif elem.islower():
            lower_count += 1
    return upper_count, lower_count

# list 去重 乱序
def repeat_rid(name):
    new = list(set(name))
    return new

# list 去重 还是以前顺序
def repeat_rid_order(name):
    new = list(set(name))
    new.sort(key=name.index)
    return new

# 斐波那契数列
def fibonacci(n):
    if n <= 2:
        return 1
    else:
        return fibonacci(n - 2) + fibonacci(n - 1)

# 汉诺塔
def hanoi(n, start, go, end):
    if n == 1:
        print('{} -> {}'.format(start, end))
    else:
        hanoi(n - 1, start, end, go)
        hanoi(1, start, go, end)
        hanoi(n - 1, go, start, end)

# 杨辉三角
def triangles(n):
    # 杨辉三角形的数
    def triangles_num(line, col):
        # 每行第一个和最后一个为1
        if col == 0 or line == col:
            return 1
        else:
            # 上边两个数相加
            return triangles_num(line - 1, col - 1) + triangles_num(line - 1, col)
    # 画图
    for i in range(0, n):
        # 画空格
        for x in range(1, n - i):
            print(end="\t")
        # 杨辉三角形数
        for j in range(0, i + 1):
            print(triangles_num(i, j), end="\t\t")
        print()

# 参数组合
def all_parameter(name, password, national='汉', *hobby, age, married=False, **kw):
    return name, password, national, hobby, age, married, kw

# map函数，两个参数，一个函数，一个序列，用来把函数用到每一个序列成员
nums = (4, 5, 8, -8, 9, -7)
print(tuple(map(abs, nums)))

# reduce函数 首先参数函数必须只有两个参数，把结果继续和序列的下个元素累积
# reduce(f, [x1, x2, x3, x4]) = f(f(f(x1, x2), x3), x4)
def power(num, p=2):
    return num ** p

print(reduce(power, (2, 2, 2, 2)))

# filter函数 参数函数必须返回布尔，根据布尔值来决定元素去留
def positive(num):
    return num >= 0

print(list(filter(positive, nums)))

# sorted函数， key=可以指定排序的函数
print(sorted(nums))
# 下面的lambda表达式表示一个函数：输入一个值然后取其绝对值，根据绝对值来排序
print(sorted(nums, key=lambda x: abs(x)))

if __name__ == '__main__':
    print(statistics('TuTyn./'))
    print(repeat_rid([2, 8, 5, 9, 8, 9, 2]))
    print(repeat_rid_order([2, 8, 5, 9, 8, 9, 2]))
    print(fibonacci(10))
    hanoi(2, 'A', 'B', 'C')
    triangles(6)