import smtplib
import time
import imaplib
import email
import pandas as pd
import os

def get_decoded_email_body(message_body):
    raw_email_string = message_body.decode('utf-8')
# converts byte literal to string removing b''
    email_message = email.message_from_string(raw_email_string)
# this will loop through all the available multiparts in mail
    #print(email_message)
    for part in email_message.walk():
       # print(part)
        if part.get_content_type() != "text/plain": # ignore attachments/html
            body = part.get_payload(decode=True)
            #save_string = str("D:Dumpgmailemail_" + str(x) + ".eml")
    print(body.decode('utf-8'))
    return body.decode('utf-8')

def parse_html(body):
    from bs4 import BeautifulSoup
    import re

    p=re.compile(r'(\d{3}[-\.\s]??\d{3}[-\.\s]??\d{4}|\(\d{3}\)\s*\d{3}[-\.\s]??\d{4}|\d{3}[-\.\s]??\d{4})')
    Csoup = BeautifulSoup(body, 'html.parser')
    for order in Csoup.find_all('div', {'id':re.compile('.*cust_service_info')}):

        order=list(order.find_all(text=True))
        order_det=order[2].split(' ')
        order_no=order_det[1]
        if len(order_det)==3:
            order_type=order_det[2]
        else:
            order_type='Delivery'

        cust_info_collect=0
        order_info_collect=0
        cust_info_ilne_count=0
        cust_addr=''
        order_details=''
        order_dict={}
    for j in Csoup.find_all('tr'):
        cols =j.find_all('td')
        vals=([ele.text.strip() for ele in cols])
        #print(vals)
        if vals[0]=='' :
            continue

        if vals[0] in ('Prepare for:','Deliver to:') :
            #print(1)
            cust_info_collect=1
            continue
        if vals[0]=='Qty':
            #print(2)
            order_info_collect=1
            cust_info_collect=2
            continue
        if cust_info_collect==1 and order_info_collect==0:
            #print(3)
            if cust_info_ilne_count==0:
                #print(4)
                cust_name=vals[0]
                order_date=vals[1]
                cust_info_ilne_count=1
            else:
                if(re.match(p,vals[0]) ):
                    cust_num=vals[0]
                else:
                    if len(vals)>=2 :
                        order_date+=vals[1]
                    cust_addr+=vals[0]
        if vals[0]=='Subtotal':
            #print(6)
            order_info_collect=2
            continue
        if order_info_collect==1:
            #print(7)
            order_details+=vals[1]+','
            order_dict[vals[1]]=order_no
    '''
    print('order_type:'+order_type)
    print('order_no:'+order_no)
    print('ordered_date:'+order_date)
    print('cust_name:'+cust_name)
    print('cust_addr:'+cust_addr)
    print('ordered_items:'+order_details)
    print('phone:'+cust_num)'''
    return(order_type,order_no,order_date,order_details,cust_name,cust_addr,cust_num)
    #return(order_type)

def Extract_Orders_4m_mail(SMTP_SERVER,FROM_EMAIL,FROM_PWD,LABEL,SENDER,SUBJECT_SEARCH ):
    #try:
        mail = imaplib.IMAP4_SSL(SMTP_SERVER,PORT)
        mail.login(FROM_EMAIL, FROM_PWD)
        mail.select(LABEL)

        type, data = mail.search(None, '( SINCE "01-Jan-2017" BEFORE "31-dec-2017" )' )
        mail_ids = data[0]

        id_list = mail_ids.split()

        order_type=[]
        order_no=[]
        order_date=[]
        order_details=[]
        item_no=[]
        cust_name=[]
        cust_addr=[]
        cust_num=[]
        order_dict={}
        o_dict={}
        for i in reversed(id_list):

            typ, data = mail.fetch(i, '(RFC822)')

            for response_part in data:

                if isinstance(response_part, tuple):
                    msg = email.message_from_string(response_part[1].decode('utf-8'))
                    email_subject = msg['subject']
                    email_from = msg['from']
#                     print('From : ' + email_from + '\n')
#                     print('Subject : ' + email_subject + '\n')
#                     print('date : ' + msg['date'] + '\n')
                    html_code=get_decoded_email_body (response_part[1])
                    wf.write(html_code)
                    ordertype,orderno,orderdate,orderdetails,custname,custaddr,custnum=parse_html(html_code)

                    order_type.append(ordertype)
                    order_no.append(orderno)
                    order_date.append(orderdate)
                    iph=orderdetails.split(',')[0].split('.')
                    order_details.append(iph[1])
                    item_no.append(iph[0])
                    cust_name.append(custname)
                    cust_addr.append(custaddr)
                    cust_num.append(custnum)
                    for i in range(len(orderdetails.split(','))):
                        if i==0 or i==len(orderdetails.split(','))-1:
                            continue
                        order_type.append('')
                        order_no.append('')
                        order_date.append('')
                        iph=orderdetails.split(',')[i].split('.')
                        order_details.append(iph[1])
                        item_no.append(iph[0])
                        cust_name.append('')
                        cust_addr.append('')
                        cust_num.append('')

        return{'ORDER_TYPE':order_type,'ORDER_NO':order_no,'ORDER_DATE':order_date,'ITEM_NO':item_no,'ITEM_DESC':order_details,'CUSTOMER_NAME':cust_name,'CUSTOMER_ADDR':cust_addr,'CUSTOMER_PHONE':cust_num}

#Enter gmail uid and pwd along with sender ,subject_search
print("Enter your gmail id ex:'myid@gmail.com'")
EMAIL  = input()
print("Enter your GMAIL password")
PWD    = input()
SERVER = "imap.gmail.com"
print("Enter the sender name you are looking for ex:'James Bond'")
SENDER= input()
print("Enter the search string in SUBJECT  you are looking for ex:'Orders'")
SUBJECT_SEARCH=input()
FROM_LABEL='INBOX'
PORT   = 993
path="D:\\"
file="Customer_order_details.xls"
file1="Customer_order_details.html"
ff=os.path.join(path, file)

wf=open(os.path.join(path,file1),'w')

ExWriter = pd.ExcelWriter(ff)
dic1=Extract_Orders_4m_mail(SERVER,EMAIL,PWD,FROM_LABEL,SENDER,SUBJECT_SEARCH)
df_CustData=pd.DataFrame(dic1)
df_CustData.to_excel(ExWriter,'sheet1',index=False,)

ExWriter.save()
print(" here is your file '%s'"%(ff))


import matplotlib.pyplot as plt
from die import Die

# Create a D6.
die = Die()

# Make some rolls, and store results in a list.
results = []
for roll_num in range(1000):
    result = die.roll()
    results.append(result)

# Analyze the results.
frequencies = []
for value in range(1, die.num_sides+1):
    frequency = results.count(value)
    frequencies.append(frequency)

# Visualize the results.

die_values = [1, 2, 3, 4, 5, 6,]
frequency_result = list(range(1, 1000))
plt.plot(die_values, frequency_result, linewidth=5)

plt.title("Results of rolling one D6 1000 times.")
plt.xlabel("Side of Die")
plt.ylabel("Frequency")

plt.title = "Result"
hist.y_title = "Frequency of Result"

plt.show()


# Authors:
#     Sylvain Faure <sylvain.faure@math.u-psud.fr>
#     Bertrand Maury <bertrand.maury@math.u-psud.fr>
#
#      cromosim/examples/micro/social/micro_social.py
#      python micro_social.py --json input.json
#
# License: GPL

import sys, os
import numpy as np
from cromosim import *
from cromosim.micro import *
from optparse import OptionParser
import json
from scipy.spatial import distance
import statistics as stat
from tqdm import tqdm
#plt.ion()

parser = OptionParser(usage="usage: %prog [options] filename",version="%prog 1.0")
parser.add_option('--json',dest="jsonfilename",default="input_catanzaro_social_porta120.json",type="string",
                  action="store",help="Input json filename")
opt, remainder = parser.parse_args()
#print("===> JSON filename = ",opt.jsonfilename)
with open(opt.jsonfilename) as json_file:
    input = json.load(json_file)

"""
    Get parameters from json file :

    name: string
        Domain name
    prefix: string
        Folder name to store the results
    background: string
        Image file used as background
    px: float
        Pixel size in meters (also called space step)
    width: integer
        Domain width (equal to the width of the background image)
    height: integer
        Domain height (equal to the height of the background image)
    wall_lines : list of numpy arrays
        Polylines used to build walls, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    wall_ellipses : list of numpy arrays
        Ellipses used to build walls, [ [x_center,y_center, width, height, angle_in_degrees_anti-clockwise],... ]
    wall_polygons : list of numpy arrays
        Polygons used to build walls, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    wall_lines : list of numpy arrays
        Polylines used to build walls, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    door_lines: list of numpy arrays
        Polylines used to build doors, [ [[x0,x1,x2,...],[y0,y1,y2,...]],... ]
    seed: integer
        Random seed which can be used to reproduce a random selection if >0
    rmin: float
        Minimum radius for people
    rmax: float
        Maximum radius for people
    mass: float
        Mass of one person (typically 80 kg)
    tau: float
        (typically 0.5 s)
    F: float
        Coefficient for the repulsion force between individuals (typically 2000 N)
    kappa: float
        Stiffness constant to handle overlapping (typically 120000 kg s^-2)
    delta: float
        To maintain a certain distance from neighbors (typically 0.08 m)
    Fwall: float
        Coefficient for the repulsion force between individual and walls (typically 2000 N, like for F)
    lambda: float
        Directional dependence (between 0 and 1 = fully isotropic case)
    eta: float
        Friction coefficient (typically 240000 kg m^-1 s^-1)
    N: list
        Number of persons in each boxes
    init_people_box: list
        List of boxes to randomly position people at initialization, \
        [[xmin,xmax,ymin,ymax],...]
    exit_people_box:
        People outside this box will be deleted, [xmin,xmax,ymin,ymax]
    Tf: float
        Final time
    dt: float
        Time step
    drawper: integer
        The results will be displayed every "drawper" iterations
    dmax: float
        Maximum distance used to detect neighbors
    dmin: float
        Minimum distance allowed between individuals
    sensors: list of numpy array
        Segments through which incoming and outgoing flows are measured
        [ [x0,y0,x1,y1],... ]
    plot_people: boolean
        If true, people are drawn
    plot_contacts: boolean
        If true, active contacts between people are drawn
    plot_velocities: boolean
        If true, people velocities are drawn
    plot_paths: boolean
        If true, people paths are drawn
    plot_sensors: boolean
        If true, plot sensor lines on people graph and sensor data graph
"""

name = input["name"]
prefix = input["prefix"]
if not os.path.exists(prefix):
    os.makedirs(prefix)
background = input["background"]
px = input["px"]
width = input["width"]
height = input["height"]
wall_lines = input["wall_lines"]
wall_ellipses = input["wall_ellipses"]
wall_polygons = input["wall_polygons"]
door_lines = input["door_lines"]
seed = input["seed"]
N = sp.array(input["N"]).astype(int)
print("N = ",N)
Np = N.sum()
rmin = input["rmin"]
rmax = input["rmax"]
mass = input["mass"]
tau = input["tau"]
F = input["F"]
kappa = input["kappa"]
delta = input["delta"]
Fwall = input["Fwall"]
lambda_ = input["lambda"]
eta = input["eta"]
init_people_box = input["init_people_box"]
exit_people_box = input["exit_people_box"]
Tf = input["Tf"]
dt = input["dt"]
drawper = input["drawper"]
dmax = input["dmax"]
dmin = input["dmin"]
sensors = input["sensors"]
plot_p = input["plot_people"]
plot_c = input["plot_contacts"]
plot_v = input["plot_velocities"]
plot_pa = input["plot_paths"]
plot_s = input["plot_sensors"]
linewidth = input["line_width"]
filename = input["filename"]
vmax = input["v_max"]
plot = input["plot"]
ferma_prof = input["ferma_prof"]
total = input["total_it"]
'''print("===> Number of persons = ",Np)
print("===> Final time, Tf = ",Tf)
print("===> Time step, dt = ",dt)
print("===> To draw the results each drawper iterations, drawper = ",drawper)
print("===> Maximal distance to find neighbors, dmax = ",dmax,", example : 2*dt")
print("===> Minimal distance between persons, dmin = ",dmin)'''

"""
    Build the Domain
"""

## To create an Domain object
dom = Domain(name=name, pixel_size=px, width=width, height=height, vmax = vmax)
## To add lines : Line2D(xdata, ydata, linewidth)
for xy in wall_lines:
    line = Line2D( xy[0],xy[1], linewidth=linewidth)
    dom.add_wall(line)
## To add doors :
for xy in door_lines:
    line = Line2D( xy[0],xy[1], linewidth=linewidth)
    dom.add_door(line)
## To build the domain : background + shapes
dom.build_domain()
## To compute the distance to the walls
dom.compute_wall_distance()
## To compute the desired velocity
dom.compute_desired_velocity()
## To show the domain dimensions
'''
print("===> Domain : ",dom)
print("===> Wall lines : ",wall_lines)
print("===> Door lines : ",door_lines)

plt.ioff()
dom.plot()
plt.show()
'''
"""
    Initialization
"""
'''
f = open(filename, "w")
m_h = height * px
m_w = width * px
a = m_w * m_h
density = Np / a
f.write("Input: " + str(filename) + "\n")
f.write("Velocità: " + str(vmax) + "\n")
f.write("density: " + str(density) + "\n")
f.write("Np: " + str(Np) + "\n")
f.write("dt: " + str(dt) + "\n\n")
'''
mean = 0
times = []
c_flows = 0
flow_mean_total = 0
it = 1

#door_center = (stat.mean(door_lines[0][0]), stat.mean(door_lines[0][1]))

doors_center = []
for door in door_lines:
    doors_center.append((stat.mean(door[0]), stat.mean(door[1])))

time_single_unit = {}
exit_single_unit = {}
custom_area = [[3.9, 3.9], [4.8, 4.5]]
count_zoneB = []
count_zoneA = []
velocities_iter = []
evacuation_time = []
velocities_mean = []
flow_iter = []
for i in range(1, Np+1):
    time_single_unit[i] = []
    exit_single_unit[i] = []
    for j in range(0, len(door_lines)):
        exit_single_unit[i].append(0)

flows = []
flow_cum = []
people_dist = {}
people_pos = {}
people_door_dist = {}
people_velocities = {}
if (seed<0):
    seed = sp.random.RandomState()
## Current time
t = 0.0
people, people_init_box_id, rng = people_initialization(N, init_people_box, dom,
                                                        dt, rmin, rmax, dmin=dmin,
                                                        seed=seed)
people_ids = people[:,3]
for p in people:
    people_door_dist[p[3]] = [[distance.euclidean((p[0],p[1]), door_center)] for door_center in doors_center]
    people_velocities[p[3]] = [0]
    people_pos[p[3]] = [(p[0], p[1])]
    people_dist[p[3]] = 0
## Array to store the results : all the people coordinates for all times
Np = people.shape[0]
Uold = sp.zeros((Np,2))
Np_init = Np
people_id = sp.arange(Np)
results = sp.copy(people[:,:2]).reshape((Np,2,1))

## Array to store sensor data : time dir pts[2] for each sensor line
if (len(sensors)>0):
    sensor_data = sp.zeros((Np,4,len(sensors)))

count_p_custom_area = []#conta persone nella zona delimitata da custom_area
count_p_cum = []
"""
    Main loop
"""
cc = 0
counter = 0
int_t = int(t)
N_old = Np
tot_vel = []
i = 0
time = {}
ts = {}
people_velocities = {}
velocities = []
#print("Step_" + str(it) + " di " + str(total))
it += 1
for p in people[:,3]:
    ts[p] = [0]
    people_velocities[p] = [0]
while (t<Tf):#continua fino allo scadere del tempo
    contacts = compute_contacts(dom, people, dmax)
    I, J, Vd = compute_desired_velocity(dom, people)

    if int_t < int(t):
        flow = N_old - Np
        #print("===> time = ",int_t," flow = ",flow, "number of persons = ", Np)
        N_old = Np
        int_t = int(t)
        flows.append(flow)
        if(len(flow_cum) > 0):
            flow_cum.append(flow_cum[-1]+flow)
        else:
            flow_cum.append(flow)
        #calcolo la densità vicino alla porta
        count_custom = 0
        for p in people:
            #verifico se la persona è nell'area a scelta custom_area
            if p[0] > custom_area[0][0] and p[0] < custom_area[1][0]:#x
                if p[1] > custom_area[0][1] and p[1] < custom_area[1][1]:#y
                    count_custom += 1

        count_p_custom_area.append(count_custom)
        if(len(count_p_cum) > 0):
            count_p_cum.append(count_p_cum[-1]+count_custom)
        else:
            count_p_cum.append(count_custom)
    if ((cc>=drawper) or (counter==0)):
        #print("===> time = ",t," number of persons = ",Np)
        if(plot == True):
            plot_people(10, dom, people, contacts, Vd, people[:,3], time=t,
                            plot_people=plot_p, plot_contacts=plot_c,
                            plot_velocities=plot_v, plot_paths=False,paths=results,
                            plot_sensors=plot_s, sensors=sensors,
                            savefig=True, filename=prefix+'fig_'+ \
                            str(counter).zfill(6)+ '_' + str(dt) + '.png')
            #plot_grafici(20, people, ts, people_door_dist, t, people_velocities, people_pos)
            plt.pause(0.01)
            if (t>0):
                for i, s in enumerate(sensors):
                    plot_sensor_data(30+i, sensor_data[:,:,i], t, savefig=True,
                            filename=prefix+'sensor_'+str(i)+'_'+str(counter)+'.png')
                    plt.pause(0.01)
        cc = 0
    Forces = compute_forces(F, Fwall, people, contacts, Uold, Vd, lambda_, delta, kappa, eta)
    U = dt*(Vd-Uold)/tau + Uold + dt*Forces/mass[0]

    if ferma_prof:
        if(Np > 1 and people[-1,0] >= doors_center[0][0]-1):
            U[-1, :] = 0
        else:
            U[-1, :] = dt*(Vd[-1,:]-Uold[-1,:])/tau + Uold[-1,:] + dt*Forces[-1,:]/mass[1]

    people = move_people(t, dt, people, U)
    t += dt
    for p in people:
        i = 0
        for door_center in doors_center:
            dist = distance.euclidean((p[0],p[1]), door_center)
            #people_velocities[p[3]].append(people_door_dist[p[3]][-1]-dist)/t)
            people_door_dist[p[3]][i].append(dist)
            i += 1
        ts[p[3]].append(t)
        people_dist[p[3]] += distance.euclidean(people_pos[p[3]][-1], (p[0], p[1]))
        people_pos[p[3]].append((p[0], p[1]))
    t -= dt
    if len(U > 0):
        for p in range(0, len(people_id)):
            #calcolo accellerazione di ogni persona
            vel = pow((U[p][0]*U[p][0]) + (U[p][1]*U[p][1]), 1/2)
            if p in people_velocities.keys():
                people_velocities[people_id[p]].append(vel)
            else:
                people_velocities[people_id[p]] = [vel]

        for u in U:
            vel = pow((u[0]*u[0]) + (u[1]*u[1]), 1/2)
            velocities.append(vel)

        tot_vel.append(sum(velocities)/len(velocities))

    people, U, [people_id], id_exit = exit_door(2*dom.pixel_size, dom, people, U, doors_center, arrays=[ people_id])

    for id in id_exit:
        time_single_unit[id].append(t)
        exit_single_unit[id][id_exit[id]] += 1

    ## Store people positions in the result array (used to draw people paths)
    tmp = 1e99*sp.ones((Np_init,2))
    tmp[people_id,:] = people[:,:2]
    results = sp.concatenate((results,tmp.reshape((Np_init,2,1))), axis=2)
    for id in id_exit:
        time[id] = t
    t += dt
    Uold = U
    cc += 1
    counter += 1
    if(t > 52):
        #f.write("=========> scemo incastrato" )
        plot_people(10, dom, people, contacts, Vd, people[:,3], time=t,
                            plot_people=plot_p, plot_contacts=plot_c,
                            plot_velocities=plot_v, plot_paths=False,paths=results,
                            plot_sensors=plot_s, sensors=sensors,
                            savefig=True, filename="incastrato.png")
        break
    Np = people.shape[0]
    if (Np == 0):
        flow = N_old - Np
        flows.append(flow)
        flow_cum.append(flow_cum[-1]+flow)
        #print("END... Nobody here !")
        break

'''
flow_rate = sum(flows) / len(flows)
flow_mean_total += flow_rate
c_flows += 1
mean = mean + t
f.write("velocity" + str(c_flows) + " = " + str(sum(tot_vel)/len(tot_vel)) + "m/s\n")
print("velocity" + str(c_flows) + " = " + str(sum(tot_vel)/len(tot_vel)) + "m/s")
#velocities_iter.append(sum(tot_vel)/len(tot_vel))
velocities_iter.append(tot_vel)
print("evacuation time = ", t, "s")
print("path length = ", t * (sum(tot_vel)/len(tot_vel)), "m")
f.write("evacuation time_" + str(c_flows) +": " + str(t) + "\n")
evacuation_time.append(t)
velocities_mean.append(sum(tot_vel)/len(tot_vel))
f.write("flow_rate_" + str(c_flows) + ": " + str(flow_rate) + "\n")
f.write("flows" + str(flows) + "\n")
flow_iter.append(flows)
#f.write("persona nell'area a scelta: \n")
#f.write(str(count_p_custom_area) + "\n")
f.write("Media lunghezza cammini: " + str(stat.mean([v for v in people_dist.values()])) + "\n\n")
count_zoneB.append(count_p_custom_area)
count_zoneA.append(flows)
times.append(t)

f.write("\n Flow cumulativo: " + str(flow_cum))
flow_mean_total = flow_mean_total / c_flows
mean = mean/ total
f.write("all times\n")
f.write(str(times))
f.write("\n")
f.write("mean ev: " + str(mean) + "\n")
f.write("std ev: " + str(np.std(times)) + "\n")
#f.write("mean door flow rate: " + str(flow_mean_total) + "\n")
#f.write("mean desk flow rate: " + str(stat.mean(count_p_custom_area)) + "\n")
f.write("\n\n" + "Velocità medie :" + "\n" + str(velocities_mean) + "\n")
f.write("Flussi ad ogni dt :" + "\n" + str(flow_iter))
#print(time_single_unit)
for id in time_single_unit:
    mean = sum(time_single_unit[id]) / len(time_single_unit[id])
    time_single_unit[id] = [mean]
    time_single_unit[id].extend(exit_single_unit[id])
    if len(door_lines) == 1:
        time_single_unit[id].append(0)

f.write("\ntime_single_unit\n")
f.write(str(time_single_unit))
'''
sys.exit()


import json
import urllib.request

def choice_of_user(list_of_users):
    user = input('Выберите одного из пользователей, с которым хотите работать:')
    while user not in list_of_users:
        user = input('Введите точное имя пользователя из списка:')
    print(f'\nВы выбрали пользователя {user}.\n')
    return user

def get_data_from_user(user, type_of_data):
    token = ""  # сюда надо вставить ваш токен
    data = []
    if type_of_data == 'user':
        url = f'https://api.github.com/users/{user}?access_token={token}'
        response = urllib.request.urlopen(url)
        text = response.read().decode('utf-8')
        data = json.loads(text)
    else:
        i = 1
        while True:
            url = f'https://api.github.com/users/{user}/{type_of_data}?access_token={token}&page={i}'
            response = urllib.request.urlopen(url)
            text = response.read().decode('utf-8')
            a = json.loads(text)
            if not a:
                break
            data.extend(a)
            i += 1
    return data

def repos_names_description(user):
    data = get_data_from_user(user, 'repos')
    print('Вот список его репозиториев и описаний:\n')
    for i in data:
        description = i['description']
        if description is None:
            description = 'описания нет'
        print(f"\t{i['name']}: {description}")

def languages_per_user(user):
    repositories = get_data_from_user(user, 'repos')
    lang_reps = {}
    for rep in repositories:
        if str(rep['language']) == 'None':
            continue
        if rep['language'] not in lang_reps:
            lang_reps[rep['language']] = []
        lang_reps[rep['language']].append(rep['name'])
    return lang_reps

def print_languages_per_user(lang_repos_dict):
    print('Пользователь пишет на ' + ', '.join(str(key) for key in lang_repos_dict.keys()) + '\n')
    for language, repos in lang_repos_dict.items():
        print(f'Язык {language} используется в репозитории %s.' % ', '.join(repos))

def most_repos(user_list):
    repos_per_user = {}
    for user in user_list:
        data = get_data_from_user(user, 'user')
        repos_number = data['public_repos']
        repos_per_user[user] = repos_number
    username = key_with_highest_value(repos_per_user)
    print(f'Больше всего репозиториев у пользователя {username} - {repos_per_user[username]}.\n')

def popular_languages(user_list):
    languages = {}
    for user in user_list:
        data = languages_per_user(user)
        for lang, value in data.items():
            if lang not in languages:
                languages[lang] = 0
            languages[lang] += len(value)
    the_most_popular = key_with_highest_value(languages)
    print(f'Самый популярный язык среди пользователей из списка - {the_most_popular}.\n')

def key_with_highest_value(dic):
    v = list(dic.values())
    k = list(dic.keys())
    return k[v.index(max(v))]

def most_followers(user_list):
    followers_per_user = {}
    for user in user_list:
        data = get_data_from_user(user, 'user')
        followers_per_user[user] = data['followers']
    username = key_with_highest_value(followers_per_user)
    print(f'Больше всего фолловеров у пользователя {username} - {followers_per_user[username]}.\n')

def choice_of_action(list_of_users):
    print('Вот список доступных пользователей:')
    for user in list_of_users:
        print(user)
    print('''\nЧто вы хотите сделать?
    1.Вывести список репозиториев одного из пользователей.
    2.Вывести список языков одного из пользователей и репозитории, в которых они используются.
    3.Узнать, у кого из пользователей в списке больше всего репозиториев.
    4.Узнать, какой язык самый популярный среди пользователей списка.
    5.Узнать, у кого из пользователей списка больше всего подписчиков.\n''')
    action = input('Введите номер пункта: ')
    actions = ['1', '2', '3', '4', '5']
    while action not in actions:
        action = input('Введите номер пункта цифрой: ')
    if action == '1':
        repos_names_description(choice_of_user(list_of_users))
    elif action == '2':
        print_languages_per_user(languages_per_user(choice_of_user(list_of_users)))
    elif action == '3':
        most_repos(list_of_users)
    elif action == '4':
        popular_languages(list_of_users)
    elif action == '5':
        most_followers(list_of_users)

if __name__ == "__main__":
    users = ['elmiram', 'maryszmary', 'lizaku', 'nevmenandr', 'ancatmara', 'roctbb', 'akutuzov', 'agricolamz', 'lehkost', 'kylepjohnson', 'mikekestemont', 'demidovakatya', 'shwars', 'JelteF', 'timgraham', 'arogozhnikov', 'jasny', 'bcongdon', 'whyisjake', 'gvanrossum']  # сюда нужно вставить список пользователей
    while True:
        choice_of_action(users)
        z = int(input('Хотите попробовать еще?\n1 - Да\n0 - Нет\n'))
        if z == 0:
            break


"A context manager to handle exception with option to open a debugger."
from contextlib import contextmanager

from .colorify import colorify
from .colorify import LINE_LENGTH, RED, YELLOW

# Get config values
from ..config import DEBUG

@contextmanager
def exception_handler(tag=None, pdb=False):
    """Context manager to handle exception with option to open a debugger.

    Parameter
    ---------
    tag: str
        Name to display before outputing error in red.
    pdb: bool
        If set to True, open a debugger if an error is raised.
    """
    try:
        yield
    except KeyboardInterrupt:
        status = colorify("interrupted", YELLOW)
        print(f"\r{tag} {status}".ljust(LINE_LENGTH))
        raise SystemExit(1)
    except BaseException:
        status = colorify("error", RED)
        print(f"{tag} {status}".ljust(LINE_LENGTH))

        if pdb:
            # Use ipdb if it is available and default to pdb otherwise.
            try:
                from ipdb import post_mortem
            except ImportError:
                from pdb import post_mortem
            post_mortem()

        if DEBUG:
            raise
        else:
            import traceback
            traceback.print_exc()


from nba_api.stats.endpoints import TeamGameLog
import pandas as pd
from nba_api.stats.static import players
from nba_api.stats.endpoints import CommonTeamRoster
import sqlite3
import csv
import time

##opening connection to SQL
conn = sqlite3.connect("NbaStats.db")
curs = conn.cursor()
##getting all team ids
curs.execute("SELECT team_id FROM teams;")
all_teamids = curs.fetchall()

##Loop through each teamid
for id in all_teamids:
    ##Call the get_all_players function passing the teamid
    get_all_players(id)
    time.sleep(10)
    reader = csv.reader(open('roster_csv.csv','r'), delimiter=',')
    for row in reader:
        to_db =[row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],row[8],row[9],row[10],row[11],row[12],row[13],row[14]]
        curs.execute("INSERT INTO players(count, team_ID,SEASON,LeagueID,PLAYER,PLAYER_SLUG,NUM,POSITION,HEIGHT,WEIGHT,BIRTH_DATE,AGE,EXP,SCHOOL,PLAYER_ID) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);", to_db)

    time.sleep(10)

def get_team_season_history():

    teamstats = TeamGameLog(
        team_id=' 1610612745',
        season='2019-20',
        ##league_id_nullable =='00',
        ##per_mode_simple='PerGame',
        season_type_all_star='Regular Season'
    )

    teams_dict = teamstats.get_data_frames()

    print(teams_dict)

def get_all_players(teamID):

    teamRoster = CommonTeamRoster(
        team_id= teamID,
        season='2019-20',
    )
    roster = teamRoster.get_data_frames()
    ##print(roster[0])
    roster[0].to_csv('roster_csv.csv',header=False)

##get_all_players(team)

def create_table():
    curs.execute("CREATE TABLE players (count INTEGER, team_ID INTEGER,SEASON INTEGER,LeagueID INTEGER ,PLAYER TEXT,PLAYER_SLUG TEXT,NUM INTEGER,POSITION TEXT,HEIGHT TEXT,WEIGHT TEXT,BIRTH_DATE TEXT,AGE INTEGER,EXP INTEGER,SCHOOL TEXT,PLAYER_ID INTEGER PRIMARY KEY);")
    conn.commit()

conn.commit()
#get_all_players(team)

#reader = csv.reader(open('teamid_csv.csv','r'), delimiter=',')
#for row in reader:
 #   to_db = [row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7]]
 #   curs.execute("INSERT INTO teams(count, team_id, full_name, abbreviation, nickname, city, state, year_founded) VALUES (?, ?, ?, ?, ?, ?, ?, ?);", to_db)
#conn.commit()

##upload_data_frame_to_csv(roster)


from django import forms
from django.contrib.auth.models import User
from django.contrib.auth.forms import UserCreationForm
from .models import Crmessage, Crservice, Crimputation

class UserCreerForm(UserCreationForm):
	email = forms.EmailField()
	class Meta:
		model = User
		fields = ['username', 'email', 'password1', 'password2']

class FormCrmessage(forms.ModelForm):
	class Meta:
		type_msg = (("DIRECTION","Direction"),("AUTRES","Autres"))
		model=Crmessage
		fields = ['designation','objet', 'date_courrier', 'date_reception','type_message']
		widgets= {
		'type_message':forms.Select(choices=type_msg),
		'date_courrier':forms.DateInput(attrs={"class": "input", "type": "date"}, format="%Y-%m-%d"),
		'date_reception':forms.DateInput(attrs={"class": "input", "type": "date"}, format="%Y-%m-%d"),
		# 'crmessage':forms.Select(attrs={'class': 'input'}),
		# 'crservice':forms.Select()
		 }

class FormCrservice(forms.ModelForm):
	class Meta:
		model=Crservice
		fields = ['code_srv','nom_service']

class FormCrimputation(forms.ModelForm):
	class Meta:
		msg = (("TRAITER","Traité"),("NONTRAITER","Non Traité"))
		model=Crimputation
		fields = ['date_imput','date_lmt_trmt', 'observation', 'statut_msg', 'crmessage', 'crservice']
		widgets= {
		'statut_msg':forms.Select(choices=msg),
		'date_imput':forms.DateInput(attrs={"class": "input", "type": "date"}, format="%Y-%m-%d"),
		'date_lmt_trmt':forms.DateInput(attrs={"class": "input", "type": "date"}, format="%Y-%m-%d"),
		# 'crmessage':forms.Select(attrs={'class': 'input'}),
		# 'crservice':forms.Select()
		 }


from typing import List

class Solution:
    def isBoomerang(self, points: List[List[int]]) -> bool:

        def getMaxCommonFactor(a, b):
            a = a % b
            while a > 0:
                c = b % a
                b = a
                a = c
            return b

        def getCline(x0, y0, x1, y1):
            dx, dy = x1 - x0, y1 - y0
            if dx == 0 and dy == 0:
                return (0, 0)
            if dx == 0:
                return (0, 1)
            if dy == 0:
                return (1, 0)
            cf = getMaxCommonFactor(abs(dx), abs(dy))
            return (dx // cf, dy // cf)

        c1 = getCline(points[0][0], points[0][1], points[1][0], points[1][1])
        c2 = getCline(points[0][0], points[0][1], points[2][0], points[2][1])
        c3 = getCline(points[1][0], points[1][1], points[2][0], points[2][1])
        return c1 != (0, 0) and c2 != (0, 0) and c3 != (0, 0) and c1 != c2

sol = Solution()
# ret = sol.isBoomerang([[1, 1], [2, 2], [3, 3]])
ret = sol.isBoomerang([[1, 0], [0, 0], [2, 0]])
print(ret)


from typing import Literal

from pydantic import BaseModel

from app import constants as const
from app.actions import Action, Wait, action_from_str
from app.entities.utils import get_loop_time, invert_orientation, is_inverted
from app.types import Orientation
from app.types.hitbox import HitBox

class PatrolVision(BaseModel):
    """Vision for a patrol entity"""

    pos_x: int
    pos_y: int
    char: str = const.PATROL_VISION

    @classmethod
    def create_line(
        cls, start_x: int, start_y: int, length: int, orientation: Literal["v", "h"]
    ):
        """Creates an array of static assets to make a line"""
        if orientation == "v":
            return [cls(pos_x=start_x, pos_y=start_y + i) for i in range(0, length)]
        else:
            return [cls(pos_x=start_x + i, pos_y=start_y) for i in range(0, length)]

    @classmethod
    def create_from_patrol_pos(
        cls, start_x: int, start_y: int, orientation: Orientation
    ) -> list["PatrolVision"]:
        """Create patrolvision from patrol positioning"""
        if orientation == "up":
            create_args = [
                (start_x - 1, start_y - 1, 3, "h"),
                (start_x - 1, start_y - 2, 3, "h"),
                (start_x - 2, start_y - 3, 5, "h"),
                (start_x - 2, start_y - 4, 5, "h"),
                (start_x - 2, start_y - 5, 5, "h"),
            ]
        elif orientation == "down":
            create_args = [
                (start_x - 1, start_y + 1, 3, "h"),
                (start_x - 1, start_y + 2, 3, "h"),
                (start_x - 2, start_y + 3, 5, "h"),
                (start_x - 2, start_y + 4, 5, "h"),
                (start_x - 2, start_y + 5, 5, "h"),
            ]
        elif orientation == "right":
            create_args = [
                (start_x + 1, start_y - 1, 3, "v"),
                (start_x + 2, start_y - 1, 3, "v"),
                (start_x + 3, start_y - 2, 5, "v"),
                (start_x + 4, start_y - 2, 5, "v"),
                (start_x + 5, start_y - 2, 5, "v"),
            ]
        else:
            create_args = [
                (start_x - 1, start_y - 1, 3, "v"),
                (start_x - 2, start_y - 1, 3, "v"),
                (start_x - 3, start_y - 2, 5, "v"),
                (start_x - 4, start_y - 2, 5, "v"),
                (start_x - 5, start_y - 2, 5, "v"),
            ]
        res = []
        for x, y, l, o in create_args:
            res += cls.create_line(x, y, l, o)
        return res

    def get_hitbox_at(self, time: int) -> HitBox:
        """Get hitbox at a given time"""
        return HitBox(
            pos_x=self.pos_x,
            pos_y=self.pos_y,
            content=self.char,
            time=time,
            parent=self.__class__,
        )

class Patrol(BaseModel):
    """A patrol entity that moves between points"""

    start_x: int
    start_y: int
    orientation: Orientation
    actions: list[Action] = []
    char: str = const.PATROL

    def __str__(self) -> str:
        return self.char

    @classmethod
    def create(
        cls, start_x: int, start_y: int, orientation: Orientation, actions: list[str]
    ) -> "Patrol":
        """Create a patrol entity"""
        patrol = cls(start_x=start_x, start_y=start_y, orientation=orientation)
        for a in actions:
            action, _ = action_from_str(a, patrol)
            if action:
                patrol.actions.append(action)
        return patrol

    @property
    def loop_interval(self) -> int:
        """Calculate the loop interval length"""
        return sum([a.length for a in self.actions])

    @property
    def time_consumed(self) -> int:
        """Amount of time the initial loop consumes"""
        return self.loop_interval

    def get_move_at(self, time: int) -> Action:
        """Get action at a given time"""
        return [x for x in self.actions if x.time_start <= time < x.time_end][0]

    def get_current_position(self, time: int) -> tuple[Orientation, tuple[int, int]]:
        """Get current patrol position and orientation"""
        loop_time = get_loop_time(self.loop_interval, time)
        if loop_time == 0:
            return self.orientation, (self.start_x, self.start_y)
        move = self.get_move_at(loop_time)
        box = move.get_hitbox_at(loop_time)
        if isinstance(move, Wait):
            return self.get_current_position(time - 1)
        if is_inverted(self.loop_interval, time):
            return invert_orientation(move.orientation), (box.pos_x, box.pos_y)
        else:
            return move.orientation, (box.pos_x, box.pos_y)

    def get_hitbox_at(self, time: int) -> HitBox:
        """Get hitbox at a given time"""
        loop_time = get_loop_time(self.loop_interval, time)
        if loop_time == 0:
            return HitBox(
                pos_x=self.start_x,
                pos_y=self.start_y,
                parent=self.__class__,
                content=self.char,
                time=time,
            )
        return self.get_move_at(loop_time).get_hitbox_at(loop_time)

    @property
    def last_pos(self) -> tuple[int, int]:
        """Get the last known position for player"""
        if not self.actions:
            return (self.start_x, self.start_y)
        else:
            box = self.get_move_at(self.time_consumed).get_hitbox_at(self.time_consumed)
            return box.pos_x, box.pos_y

    def get_current_vision(self, time: int) -> list[PatrolVision]:
        """Generate vision hitboxes"""
        orientation, pos = self.get_current_position(time)
        return PatrolVision.create_from_patrol_pos(
            start_x=pos[0], start_y=pos[1], orientation=orientation
        )


#!/usr/bin/env python3
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import WebDriverException, SessionNotCreatedException
from selenium.webdriver.chrome.service import Service
import sys
import os
import pathlib
import urllib.request
import re
import zipfile
import stat
from sys import platform

def get_driver():
    # Attempt to open the Selenium chromedriver. If it fails, download the latest chromedriver.
    driver = None
    retry = True
    major_version = None

    # Determine the version of Chrome installed.
    version = get_chrome_version()
    if version:
        parts = version.split('.')
        major_version = parts[0] if len(parts) > 0 else 0

    while retry:
        retry = False
        is_download = False

        try:
            options = webdriver.ChromeOptions()
            options.add_argument('--headless')
            executable_path='./chromedriver'
            service = Service(executable_path=executable_path)
            driver = webdriver.Chrome(options=options, service=service)
        except SessionNotCreatedException as e:
            if 'This version of ChromeDriver' in e.msg:
                is_download = True
                print('Warning: You may need to update the Chrome web browser to the latest version. Run Chrome, click Help->About.')
        except WebDriverException as e:
            if "wrong permissions" in e.msg:
                st = os.stat('./chromedriver')
                os.chmod('./chromedriver', st.st_mode | stat.S_IEXEC)
                retry = True
            elif "chromedriver' executable needs to be in PATH" in e.msg:
                is_download = True

        retry = is_download and download_driver(major_version)

    return driver

def download_driver(version=None):
    # Find the latest chromedriver, download, unzip, set permissions to executable.
    result = False
    url = 'https://chromedriver.chromium.org/downloads'
    base_driver_url = 'https://chromedriver.storage.googleapis.com/'
    file_name = 'chromedriver_' + get_platform_filename()
    driver_file_name = 'chromedriver' + '.exe' if platform == "win32" else ''
    pattern = 'https://.*?path=(' + (version or '\d+') + '\.\d+\.\d+\.\d+)'

    # Download latest chromedriver.
    print('Finding latest chromedriver..')
    opener = urllib.request.FancyURLopener({})
    stream = opener.open(url)
    content = stream.read().decode('utf8')

    # Parse the latest version.
    match = re.search(pattern, content)
    if match and match.groups():
        # Url of download html page.
        url = match.group(0)
        # Version of latest driver.
        version = match.group(1)
        driver_url = base_driver_url + version + '/' + file_name

        # Download the file.
        print('Version ' + version)
        print('Downloading ' + driver_url)
        app_path = os.path.dirname(os.path.realpath(__file__))
        chromedriver_path = app_path + '/' + driver_file_name
        file_path = app_path + '/' + file_name
        urllib.request.urlretrieve(driver_url, file_path)

        # Unzip the file.
        print('Unzipping ' + file_path)
        with zipfile.ZipFile(file_path, 'r') as zip_ref:
            zip_ref.extractall(app_path)

        print('Setting executable permission on ' + chromedriver_path)
        st = os.stat(chromedriver_path)
        os.chmod(chromedriver_path, st.st_mode | stat.S_IEXEC)

        # Cleanup.
        os.remove(file_path)

        result = True

    return result

def get_platform_filename():
    filename = ''

    is_64bits = sys.maxsize > 2**32

    if platform == "linux" or platform == "linux2":
        # linux
        filename += 'linux'
        filename += '64' if is_64bits else '32'
    elif platform == "darwin":
        # OS X
        filename += 'mac64'
    elif platform == "win32":
        # Windows...
        filename += 'win32'

    filename += '.zip'

    return filename

def extract_version(output):
    try:
        google_version = ''
        for letter in output[output.rindex('DisplayVersion    REG_SZ') + 24:]:
            if letter != '\n':
                google_version += letter
            else:
                break
        return(google_version.strip())
    except TypeError:
        return

def get_chrome_version():
    version = None
    install_path = None

    try:
        if platform == "linux" or platform == "linux2":
            # linux
            install_path = "/usr/bin/google-chrome"
        elif platform == "darwin":
            # OS X
            install_path = "/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome"
        elif platform == "win32":
            # Windows...
            stream = os.popen('reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
            output = stream.read()
            version = extract_version(output)
    except Exception as ex:
        print(ex)

    version = os.popen(f"{install_path} --version").read().strip('Google Chrome ').strip() if install_path else version

    return version


from __future__ import print_function
import os
import sys
# import uuid
import time
import math
import types
import signal
import shutil
import timeit
import numbers
import subprocess
import tempfile
import contextlib
import platform
import argparse
from multiprocessing import cpu_count, Lock, current_process
from collections import OrderedDict, deque, Iterable, Iterator
from itertools import islice, tee, chain

from six import string_types
from six.moves.urllib.request import urlopen
from six.moves.urllib.error import URLError, HTTPError
import tarfile

try:
    from numba import jit, autojit, vectorize, guvectorize
except:
    pass
import numpy
import six

from .mpi import SelfIterator, segment_list, SharedCounter
from .profile import *
from . import mpi
from . import shape_calculation

# ===========================================================================
# Basics
# ===========================================================================
def is_path(path):
    if isinstance(path, str):
        path = os.path.abspath(path)
        if os.path.exists(path):
            #the file is there
            return True
        elif os.access(os.path.dirname(path), os.W_OK):
            #the file does not exists but write privileges are given
            return True
    return False

def is_string(s):
    return isinstance(s, string_types)

def is_number(i):
    return isinstance(i, numbers.Number)

def is_bool(b):
    return isinstance(b, type(True))

def iter_chunk(it, n):
    """ Chunking an iterator into small chunk of size `n`
    Note: this can be used to slice data into mini batches
    """
    if not isinstance(it, Iterator):
        it = iter(it)
    obj = list(islice(it, n))
    while obj:
        yield obj
        obj = list(islice(it, n))

def batching(n, batch_size):
    return [(i, min(i + batch_size, n))
            for i in range(0, n + batch_size, batch_size) if i < n]

# ===========================================================================
# Others
# ===========================================================================
def raise_return(e):
    raise e

class _LogWrapper():

    def __init__(self, stream):
        self.stream = stream

    def write(self, message):
        # no backtrack for writing to file
        self.stream.write(message.replace("\b", ''))
        sys.__stdout__.write(message)

    def flush(self):
        self.stream.flush()
        sys.__stdout__.flush()

    def close(self):
        try:
            self.stream.close()
        except:
            pass

def stdio(path=None, suppress=False, stderr=True):
    """
    Parameters
    ----------
    path: None, str
        if str, specified path for saving all stdout (and stderr)
        if None, reset stdout and stderr back to normal
    suppress: boolean
        totally turn-off all stdout (and stdeer)
    stderr:
        apply output file with stderr also

    """
    # turn off stdio
    if suppress:
        f = open(os.devnull, "w")
    # reset
    elif path is None:
        f = None
    # redirect to a file
    else:
        f = _LogWrapper(open(path, "w"))
    # ====== assign stdio ====== #
    if f is None: # reset
        if isinstance(sys.stdout, _LogWrapper):
            sys.stdout.close()
        if isinstance(sys.stderr, _LogWrapper):
            sys.stderr.close()
        sys.stdout = sys.__stdout__
        sys.stderr = sys.__stderr__
    else: # redirect to file
        sys.stdout = f
        if stderr:
            sys.stderr = f

_uuid_chars = list(chain(map(chr, range(65, 91)),  # ABCD
                         map(chr, range(97, 123)),  # abcd
                         map(chr, range(48, 57)))) # 0123
_uuid_random_state = numpy.random.RandomState(int(str(int(time.time() * 100))[3:]))

def uuid():
    """ Generate random UUID 8 characters with very very low collision """
    # m = time.time()
    # uniqid = '%8x%4x' % (int(m), (m - int(m)) * 1000000)
    # uniqid = str(uuid.uuid4())[:8]
    uniquid = ''.join(_uuid_random_state.choice(_uuid_chars,
                                          size=8, replace=True))
    return uniquid

@contextlib.contextmanager
def change_recursion_limit(limit):
    """Temporarily changes the recursion limit."""
    old_limit = sys.getrecursionlimit()
    if old_limit < limit:
        sys.setrecursionlimit(limit)
    yield
    sys.setrecursionlimit(old_limit)

@contextlib.contextmanager
def signal_handling(sigint=None, sigtstp=None, sigquit=None):
    # We cannot handle SIGTERM, because it prevent subproces from
    # .terminate()
    orig_int = signal.getsignal(signal.SIGINT)
    orig_tstp = signal.getsignal(signal.SIGTSTP)
    orig_quit = signal.getsignal(signal.SIGQUIT)

    if sigint is not None: signal.signal(signal.SIGINT, sigint)
    if sigtstp is not None: signal.signal(signal.SIGTSTP, sigtstp)
    if sigquit is not None: signal.signal(signal.SIGQUIT, sigquit)

    yield
    # reset
    signal.signal(signal.SIGINT, orig_int)
    signal.signal(signal.SIGTSTP, orig_tstp)
    signal.signal(signal.SIGQUIT, orig_quit)

# ===========================================================================
# UniqueHasher
# ===========================================================================
class UniqueHasher(object):
    """ This hash create strictly unique hash value by using
    its memory to remember which key has been assigned

    Note
    ----
    This function use deterministic hash, which give the same
    id for all labels, whenever you call it
    """

    def __init__(self, nb_labels=None):
        super(UniqueHasher, self).__init__()
        self.nb_labels = nb_labels
        self._memory = {} # map: key -> hash_key
        self._current_hash = {} # map: hash_key -> key

    def hash(self, value):
        key = abs(hash(value))
        if self.nb_labels is not None:
            key = key % self.nb_labels
        # already processed hash
        if value in self._current_hash:
            return self._current_hash[value]
        # not yet processed
        if key in self._memory:
            if self.nb_labels is not None and \
                len(self._memory) >= self.nb_labels:
                raise Exception('All %d labels have been assigned, outbound value:"%s"' %
                                (self.nb_labels, value))
            else:
                while key in self._memory:
                    key += 1
                    if self.nb_labels is not None and key >= self.nb_labels:
                        key = 0
        # key not in memory
        self._current_hash[value] = key
        self._memory[key] = value
        return key

    def __call__(self, value):
        return self.hash(value)

    def map(self, order, array):
        """ Re-order an ndarray to new column order """
        order = as_tuple(order)
        # get current order
        curr_order = self._current_hash.items()
        curr_order.sort(key=lambda x: x[1])
        curr_order = [i[0] for i in curr_order]
        # new column order
        order = [curr_order.index(i) for i in order]
        return array[:, order]

# ===========================================================================
# ArgCOntrol
# ===========================================================================
class ArgController(object):
    """ Simple interface to argparse """

    def __init__(self, version='1.00', print_parsed=True):
        super(ArgController, self).__init__()
        self.parser = None
        self._require_input = False
        self.arg_dict = {}
        self.name = []
        self.version = str(version)
        self.print_parsed = print_parsed

    def _is_positional(self, name):
        if name[0] != '-' and name[:2] != '--':
            return True
        return False

    def _parse_input(self, key, val):
        # ====== search if manual preprocessing available ====== #
        for i, preprocess in self.arg_dict.iteritems():
            if key in i and preprocess is not None:
                return preprocess(val)
        # ====== auto preprocess ====== #
        try:
            val = float(val)
            if int(val) == val:
                val = int(val)
        except:
            val = str(val)
        return val

    def add(self, name, help, default=None, preprocess=None):
        """ NOTE: if the default value is not given, the argument is
        required

        Parameters
        ----------
        name: str
            [-name] for optional parameters
            [name] for positional parameters
        help: str
            description of the argument
        default: str
            default value for the argument
        preprocess: callable
            take in the parsed argument and preprocess it into necessary
            information
        """
        if self.parser is None:
            self.parser = argparse.ArgumentParser(
                description='Automatic argument parser (yes,true > True; no,false > False)',
                version=self.version, add_help=True)
        # ====== NO default value ====== #
        if default is None:
            if self._is_positional(name):
                self.parser.add_argument(name, help=help, type=str, action="store",
                    metavar='')
            else:
                self.parser.add_argument(name, help=help, type=str, action="store",
                    required=True, metavar='')
            self._require_input = True
        # ====== boolean default value ====== #
        elif isinstance(default, bool):
            help += ' (default: %s)' % str(default)
            self.parser.add_argument(name, help=help,
                                     action="store_%s" % str(not default).lower())
            preprocess = lambda x: bool(x)
        # ====== add defaults value ====== #
        else:
            help += ' (default: %s)' % str(default)
            self.parser.add_argument(name, help=help, type=str, action="store",
                default=str(default), metavar='')

        # store preprocess dictionary
        if not callable(preprocess):
            preprocess = None
        self.arg_dict[name] = preprocess
        return self

    def parse(self):
        if self.parser is None:
            raise Exception('Call add to assign at least 1 argument for '
                            'for the function.')
        # TODO fix bug here
        exit_now = False
        try:
            if len(sys.argv) == 1 and self._require_input:
                self.parser.print_help()
                exit_now = True
            else:
                args = self.parser.parse_args()
        except Exception as e:
            # if specfy version or help, don't need to print anything else
            if all(i not in ['-h', '--help', '-v', '--version']
                   for i in sys.argv):
                self.parser.print_help()
            exit_now = True
        if exit_now: exit()
        # parse the arguments
        try:
            args = {i: self._parse_input(i, j)
                    for i, j in args._get_kwargs()}
        except Exception as e:
            print('Error parsing given arguments: "%s"' % str(e))
            self.parser.print_help()
            exit()
        # reset everything
        self.parser = None
        self.arg_dict = {}
        self._require_input = False
        # print the parsed arguments if necessary
        if self.print_parsed:
            max_len = max(len(i) for i in args.keys())
            max_len = '%-' + str(max_len) + 's'
            print('\n******** Parsed arguments ********')
            for i, j in args.iteritems():
                print(max_len % i, ': ', j)
            print('**********************************\n')
        return args

# ===========================================================================
# Simple math and processing
# ===========================================================================
def one_hot(y, n_classes=None):
    '''Convert class vector (integers from 0 to nb_classes)
    to binary class matrix, for use with categorical_crossentropy
    '''
    y = numpy.asarray(y, dtype='int32')
    if not n_classes:
        n_classes = numpy.max(y) + 1
    Y = numpy.zeros((len(y), n_classes))
    for i in range(len(y)):
        Y[i, y[i]] = 1.
    return Y

def pad_sequences(sequences, maxlen=None, dtype='int32',
                  padding='pre', truncating='pre', value=0.,
                  transformer=None):
    """Pads each sequence to the same length:
    the length of the longest sequence.

    If maxlen is provided, any sequence longer
    than maxlen is truncated to maxlen.
    Truncation happens off either the beginning (default) or
    the end of the sequence.

    Supports post-padding and pre-padding (default).

    Parameters
    ----------
    sequences: list
        a list that contains a list of object
    maxlen: int
        maximum length of each individual sequence
    dtype: np.dtype
        desire data type of output array
    padding: 'pre' or 'post'
        pad either before or after each sequence.
    truncating: 'pre' or 'post'
        remove values from sequences larger than maxlen either
        in the beginning or in the end of the sequence
    value: object
        padding object
    transformer: callable
        a function transform each element in sequence into desire value
        (e.g. a dictionary)

    Returns
    -------
    numpy array with dimensions (number_of_sequences, maxlen)
    """
    # ====== check valid input ====== #
    if truncating not in ('pre', 'post'):
        raise ValueError('truncating must be "pre" or "post", given value is %s'
                         % truncating)
    if padding not in ('pre', 'post'):
        raise ValueError('padding must be "pre" or "post", given value is %s'
                         % padding)
    if transformer is None:
        transformer = lambda x: x
    if not callable(transformer):
        raise ValueError('transformer must be callable, but given value is %s' %
                         type(transformer))
    # ====== processing ====== #
    if maxlen is None:
        maxlen = int(max(len(s) for s in sequences))
    nb_samples = len(sequences)
    value = numpy.cast[dtype](value)
    X = numpy.full(shape=(nb_samples, maxlen), fill_value=value, dtype=dtype)
    for idx, s in enumerate(sequences):
        s = [transformer(_) for _ in s]
        if len(s) == 0: continue # empty list
        # check truncating
        if len(s) >= maxlen:
            slice_ = slice(None, None)
            s = s[-maxlen:] if truncating == 'pre' else s[:maxlen]
        # check padding
        elif len(s) < maxlen:
            slice_ = slice(-len(s), None) if padding == 'pre' else slice(None, len(s))
        # assign value
        X[idx, slice_] = numpy.asarray(s, dtype=dtype)
    return X

def pad_center(data, size, axis=-1, **kwargs):
    '''Wrapper for numpy.pad to automatically center an array prior to padding.
    This is analogous to `str.center()`

    Examples
    --------
    >>> # Generate a vector
    >>> data = numpy.ones(5)
    >>> librosa.util.pad_center(data, 10, mode='constant')
    array([ 0.,  0.,  1.,  1.,  1.,  1.,  1.,  0.,  0.,  0.])

    >>> # Pad a matrix along its first dimension
    >>> data = numpy.ones((3, 5))
    >>> librosa.util.pad_center(data, 7, axis=0)
    array([[ 0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.],
           [ 1.,  1.,  1.,  1.,  1.],
           [ 1.,  1.,  1.,  1.,  1.],
           [ 1.,  1.,  1.,  1.,  1.],
           [ 0.,  0.,  0.,  0.,  0.],
           [ 0.,  0.,  0.,  0.,  0.]])
    >>> # Or its second dimension
    >>> librosa.util.pad_center(data, 7, axis=1)
    array([[ 0.,  1.,  1.,  1.,  1.,  1.,  0.],
           [ 0.,  1.,  1.,  1.,  1.,  1.,  0.],
           [ 0.,  1.,  1.,  1.,  1.,  1.,  0.]])

    Parameters
    ----------
    data : numpy.ndarray
        Vector to be padded and centered

    size : int >= len(data) [scalar]
        Length to pad `data`

    axis : int
        Axis along which to pad and center the data

    kwargs : additional keyword arguments
      arguments passed to `numpy.pad()`

    Returns
    -------
    data_padded : numpy.ndarray
        `data` centered and padded to length `size` along the
        specified axis

    Raises
    ------
    ParameterError
        If `size < data.shape[axis]`

    See Also
    --------
    numpy.pad
    '''
    kwargs.setdefault('mode', 'constant')

    n = data.shape[axis]

    lpad = int((size - n) // 2)

    lengths = [(0, 0)] * data.ndim
    lengths[axis] = (lpad, int(size - n - lpad))

    if lpad < 0:
        raise ValueError(('Target size ({:d}) must be '
                          'at least input size ({:d})').format(size,
                                                               n))

    return numpy.pad(data, lengths, **kwargs)

def framing(y, frame_length=2048, hop_length=512):
    '''Slice a time series into overlapping frames.

    This implementation uses low-level stride manipulation to avoid
    redundant copies of the time series data.

    Parameters
    ----------
    y : np.ndarray [shape=(n,)]
        Time series to frame. Must be one-dimensional and contiguous
        in memory.

    frame_length : int > 0 [scalar]
        Length of the frame in samples

    hop_length : int > 0 [scalar]
        Number of samples to hop between frames

    Returns
    -------
    y_frames : np.ndarray [shape=(frame_length, N_FRAMES)]
        An array of frames sampled from `y`:
        `y_frames[i, j] == y[j * hop_length + i]`

    Raises
    ------
    ParameterError
        If `y` is not contiguous in memory, framing is invalid.
        See `np.ascontiguous()` for details.

        If `hop_length < 1`, frames cannot advance.

    Examples
    --------
    Extract 2048-sample frames from `y` with a hop of 64 samples per frame

    >>> y, sr = librosa.load(librosa.util.example_audio_file())
    >>> librosa.util.frame(y, frame_length=2048, hop_length=64)
    array([[ -9.216e-06,   7.710e-06, ...,  -2.117e-06,  -4.362e-07],
           [  2.518e-06,  -6.294e-06, ...,  -1.775e-05,  -6.365e-06],
           ...,
           [ -7.429e-04,   5.173e-03, ...,   1.105e-05,  -5.074e-06],
           [  2.169e-03,   4.867e-03, ...,   3.666e-06,  -5.571e-06]], dtype=float32)

    '''

    if len(y) < frame_length:
        raise ValueError('Buffer is too short (n={:d})'
                         ' for frame_length={:d}'.format(len(y), frame_length))

    if hop_length < 1:
        raise ValueError('Invalid hop_length: {:d}'.format(hop_length))

    if not y.flags['C_CONTIGUOUS']:
        raise ParameterError('Input buffer must be contiguous.')

    # Compute the number of frames that will fit. The end may get truncated.
    n_frames = 1 + int((len(y) - frame_length) / hop_length)

    # Vertical stride is one sample
    # Horizontal stride is `hop_length` samples
    y_frames = numpy.lib.stride_tricks.as_strided(y,
        shape=(frame_length, n_frames),
        strides=(y.itemsize, hop_length * y.itemsize))
    return y_frames

def segment_axis(a, frame_length=2048, hop_length=512, axis=0,
                 end='cut', endvalue=0, endmode='post'):
    """Generate a new array that chops the given array along the given axis
    into overlapping frames.

    This method has been implemented by Anne Archibald,
    as part of the talk box toolkit
    example::

        segment_axis(arange(10), 4, 2)
        array([[0, 1, 2, 3],
           ( [2, 3, 4, 5],
             [4, 5, 6, 7],
             [6, 7, 8, 9]])

    Parameters
    ----------
    a: numpy.ndarray
        the array to segment
    frame_length: int
        the length of each frame
    hop_length: int
        the number of array elements by which the frames should overlap
    axis: int, None
        the axis to operate on; if None, act on the flattened array
    end: 'cut', 'wrap', 'pad'
        what to do with the last frame, if the array is not evenly
            divisible into pieces. Options are:
            - 'cut'   Simply discard the extra values
            - 'wrap'  Copy values from the beginning of the array
            - 'pad'   Pad with a constant value
    endvalue: int
        the value to use for end='pad'
    endmode: 'pre', 'post'
        if "pre", padding or wrapping at the beginning of the array.
        if "post", padding or wrapping at the ending of the array.

    Return
    ------
    a ndarray

    The array is not copied unless necessary (either because it is unevenly
    strided and being flattened or because end is set to 'pad' or 'wrap').

    Note
    ----
    Modified work and error fixing Copyright (c) TrungNT

    """
    if axis is None:
        a = numpy.ravel(a) # may copy
        axis = 0

    length = a.shape[axis]
    overlap = frame_length - hop_length

    if overlap >= frame_length:
        raise ValueError("frames cannot overlap by more than 100%")
    if overlap < 0 or frame_length <= 0:
        raise ValueError("overlap must be nonnegative and length must" +
                         "be positive")

    if length < frame_length or (length - frame_length) % (frame_length - overlap):
        if length > frame_length:
            roundup = frame_length + (1 + (length - frame_length) // (frame_length - overlap)) * (frame_length - overlap)
            rounddown = frame_length + ((length - frame_length) // (frame_length - overlap)) * (frame_length - overlap)
        else:
            roundup = frame_length
            rounddown = 0
        assert rounddown < length < roundup
        assert roundup == rounddown + (frame_length - overlap) \
        or (roundup == frame_length and rounddown == 0)
        a = a.swapaxes(-1, axis)

        if end == 'cut':
            a = a[..., :rounddown]
        elif end in ['pad', 'wrap']: # copying will be necessary
            s = list(a.shape)
            s[-1] = roundup
            b = numpy.empty(s, dtype=a.dtype)
            # pre-padding
            if endmode == 'pre':
                b[..., :length] = a
                if end == 'pad':
                    b[..., length:] = endvalue
                elif end == 'wrap':
                    b[..., length:] = a[..., :roundup - length]
            # post-padding
            elif endmode == 'post':
                b[..., -length:] = a
                if end == 'pad':
                    b[..., :(roundup - length)] = endvalue
                elif end == 'wrap':
                    b[..., :(roundup - length)] = a[..., :roundup - length]
            a = b
        a = a.swapaxes(-1, axis)
        length = a.shape[0] # update length

    if length == 0:
        raise ValueError("Not enough data points to segment array " +
                "in 'cut' mode; try 'pad' or 'wrap'")
    assert length >= frame_length
    assert (length - frame_length) % (frame_length - overlap) == 0
    n = 1 + (length - frame_length) // (frame_length - overlap)
    s = a.strides[axis]
    newshape = a.shape[:axis] + (n, frame_length) + a.shape[axis + 1:]
    newstrides = a.strides[:axis] + ((frame_length - overlap) * s, s) + a.strides[axis + 1:]

    try:
        return numpy.ndarray.__new__(numpy.ndarray, strides=newstrides,
                                  shape=newshape, buffer=a, dtype=a.dtype)
    except TypeError:
        a = a.copy()
        # Shape doesn't change but strides does
        newstrides = a.strides[:axis] + ((frame_length - overlap) * s, s) \
        + a.strides[axis + 1:]
        return numpy.ndarray.__new__(numpy.ndarray, strides=newstrides,
                                     shape=newshape, buffer=a, dtype=a.dtype)

def as_shape_tuple(shape):
    if is_number(shape):
        shape = (int(shape),)
    if not isinstance(shape, (tuple, list)):
        raise ValueError('We only accept shape in tuple or list form.')
    shape = tuple([int(i) if i is not None and i >= 0 else None for i in shape])
    if len([i for i in shape if i is None]) >= 2:
        raise Exception('Shape tuple can only have 1 unknown dimension.')
    return shape

def as_tuple(x, N=None, t=None):
    """
    Coerce a value to a tuple of given length (and possibly given type).

    Parameters
    ----------
    x : value or iterable
    N : integer
        length of the desired tuple
    t : type, optional
        required type for all elements

    Returns
    -------
    tuple
        ``tuple(x)`` if `x` is iterable, ``(x,) * N`` otherwise.

    Raises
    ------
    TypeError
        if `type` is given and `x` or any of its elements do not match it
    ValueError
        if `x` is iterable, but does not have exactly `N` elements

    Note
    ----
    This function is adpated from Lasagne
    Original work Copyright (c) 2014-2015 lasagne contributors
    All rights reserved.

    LICENSE: https://github.com/Lasagne/Lasagne/blob/master/LICENSE
    """
    if not isinstance(x, tuple):
        if isinstance(x, (types.GeneratorType, types.ListType)):
            x = tuple(x)
        else:
            x = (x,)
    # ====== check length ====== #
    if is_number(N):
        N = int(N)
        if len(x) == 1:
            x = x * N
        elif len(x) != N:
            raise ValueError('x has length=%d, but required length N=%d' %
                             (len(x), N))
    # ====== check type ====== #
    if (t is not None) and not all(isinstance(v, t) for v in x):
        raise TypeError("expected a single value or an iterable "
                        "of {0}, got {1} instead".format(t.__name__, x))
    return x

def as_list(x, N=None, t=None):
    return list(as_tuple(x, N, t))

# ===========================================================================
# Python
# ===========================================================================
class struct(object):

    '''Flexible object can be assigned any attribtues'''

    def __getitem__(self, x):
        return getattr(self, str(x))

class bidict(dict):
    """ Bi-directional dictionary (i.e. a <-> b)
    Note
    ----
    When you iterate over this dictionary, it will be a doubled size
    dictionary
    """

    def __init__(self, *args, **kwargs):
        super(bidict, self).__init__(*args, **kwargs)
        # this is duplication
        self._inv = dict()
        for i, j in self.items():
            self._inv[j] = i

    @property
    def inv(self):
        return self._inv

    def __setitem__(self, key, value):
        super(bidict, self).__setitem__(key, value)
        self._inv[value] = key
        return None

    def __getitem__(self, key):
        if key not in self:
            return self._inv[key]
        return super(bidict, self).__getitem__(key)

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).iteritems():
            self[k] = v
            self._inv[v] = k

    def __delitem__(self, key):
        del self._inv[super(bidict, self).__getitem__(key)]
        return dict.__delitem__(self, key)

class queue(object):

    """ FIFO, fast, NO thread-safe queue
    put : append to end of list
    append : append to end of list
    pop : remove data from end of list
    get : remove data from end of list
    empty : check if queue is empty
    clear : remove all data in queue
    """

    def __init__(self):
        super(queue, self).__init__()
        self._data = []
        self._idx = 0

    # ====== queue ====== #
    def put(self, value):
        self._data.append(value)

    def append(self, value):
        self._data.append(value)

    # ====== dequeue ====== #
    def pop(self):
        if self._idx == len(self._data):
            raise ValueError('Queue is empty')
        self._idx += 1
        return self._data[self._idx - 1]

    def get(self):
        if self._idx == len(self._data):
            raise ValueError('Queue is empty')
        self._idx += 1
        return self._data[self._idx - 1]

    # ====== dqueue with default ====== #
    def pop_default(self, default=None):
        if self._idx == len(self._data):
            return default
        self._idx += 1
        return self._data[self._idx - 1]

    def get_default(self, default=None):
        if self._idx == len(self._data):
            return default
        self._idx += 1
        return self._data[self._idx - 1]

    def empty(self):
        if self._idx == len(self._data):
            return True
        return False

    def clear(self):
        del self._data
        self._data = []
        self._idx = 0

    def __len__(self):
        return len(self._data) - self._idx

class Progbar(object):

    '''
    This function is adpated from: https://github.com/fchollet/keras
    Original work Copyright (c) 2014-2015 keras contributors
    Modified work Copyright 2016-2017 TrungNT
    '''

    def __init__(self, target, title=''):
        '''
            @param target: total number of steps expected
        '''
        self.width = 39
        self.target = target
        self.sum_values = {}
        self.unique_values = []
        self.start = time.time()
        self.total_width = 0
        self.seen_so_far = 0
        self.title = title

    def update(self, current, values=[]):
        '''
            @param current: index of current step
            @param values: list of tuples (name, value_for_last_step).
            The progress bar will display averages for these values.
        '''
        for k, v in values:
            if k not in self.sum_values:
                self.sum_values[k] = [v * (current - self.seen_so_far), current - self.seen_so_far]
                self.unique_values.append(k)
            else:
                self.sum_values[k][0] += v * (current - self.seen_so_far)
                self.sum_values[k][1] += (current - self.seen_so_far)
        self.seen_so_far = current

        now = time.time()

        prev_total_width = self.total_width
        sys.stdout.write("\b" * prev_total_width)
        sys.stdout.write("\r")

        numdigits = int(numpy.floor(numpy.log10(self.target))) + 1
        barstr = '%s %%%dd/%%%dd [' % (self.title, numdigits, numdigits)
        bar = barstr % (current, self.target)
        prog = float(current) / self.target
        prog_width = int(self.width * prog)
        if prog_width > 0:
            bar += ('=' * (prog_width - 1))
            if current < self.target:
                bar += '>'
            else:
                bar += '='
        bar += ('.' * (self.width - prog_width))
        bar += ']'
        sys.stdout.write(bar)
        self.total_width = len(bar)

        if current:
            time_per_unit = (now - self.start) / current
        else:
            time_per_unit = 0
        eta = time_per_unit * (self.target - current)
        info = ''
        if current < self.target:
            info += ' - ETA: %ds' % eta
        else:
            info += ' - %ds' % (now - self.start)
        for k in self.unique_values:
            info += ' - %s:' % k
            if type(self.sum_values[k]) is list:
                avg = self.sum_values[k][0] / max(1, self.sum_values[k][1])
                if abs(avg) > 1e-3:
                    info += ' %.4f' % avg
                else:
                    info += ' %.4e' % avg
            else:
                info += ' %s' % self.sum_values[k]

        self.total_width += len(info)
        if prev_total_width > self.total_width:
            info += ((prev_total_width - self.total_width) * " ")

        sys.stdout.write(info)
        if current >= self.target:
            if "Linux" in platform.platform():
                sys.stdout.write("\n\n")
            else:
                sys.stdout.write("\n")
        sys.stdout.flush()

    def add(self, n, values=[]):
        self.update(self.seen_so_far + n, values)

def progbar(list_iter_func):
    """ Wrap any list, tuple, ndarray or func object to
    print ProgressBar when iterating over it
    """
    def iter_prog(l):
        n = len(l) if hasattr(l, '__len__') else 120
        friction = 1.2
        prog = Progbar(target=n)
        for i, j in enumerate(l):
            if i >= prog.target - 1:
                prog.target += int(i * max(friction, 0.1))
                friction /= 1.2
            prog.add(1)
            yield j
        prog.target = i + 1
        prog.update(i + 1)
    # ====== create progress monitoring ====== #
    if callable(list_iter_func):
        from functools import wraps

        @wraps(list_iter_func)
        def wrapper(*args, **kwargs):
            return iter_prog(list_iter_func(*args, **kwargs))
        return wrapper
    return iter_prog(list_iter_func)

# Under Python 2, 'urlretrieve' relies on FancyURLopener from legacy
# urllib module, known to have issues with proxy management
if sys.version_info[0] == 2:
    def urlretrieve(url, filename, reporthook=None, data=None):
        '''
        This function is adpated from: https://github.com/fchollet/keras
        Original work Copyright (c) 2014-2015 keras contributors
        '''
        def chunk_read(response, chunk_size=8192, reporthook=None):
            total_size = response.info().get('Content-Length').strip()
            total_size = int(total_size)
            count = 0
            while 1:
                chunk = response.read(chunk_size)
                if not chunk:
                    break
                count += 1
                if reporthook:
                    reporthook(count, chunk_size, total_size)
                yield chunk

        response = urlopen(url, data)
        with open(filename, 'wb') as fd:
            for chunk in chunk_read(response, reporthook=reporthook):
                fd.write(chunk)
else:
    from six.moves.urllib.request import urlretrieve

def get_file(fname, origin, untar=False):
    '''
    This function is adpated from: https://github.com/fchollet/keras
    Original work Copyright (c) 2014-2015 keras contributors
    Modified work Copyright 2016-2017 TrungNT

    Return
    ------
    file path of the downloaded file
    '''
    datadir = get_datasetpath()
    if untar:
        untar_fpath = os.path.join(datadir, fname)
        fpath = untar_fpath + '.tar.gz'
    else:
        fpath = os.path.join(datadir, fname)

    if not os.path.exists(fpath):
        print('Downloading data from', origin)
        global _progbar
        _progbar = None

        def dl_progress(count, block_size, total_size):
            global _progbar
            if _progbar is None:
                _progbar = Progbar(total_size)
            else:
                _progbar.update(count * block_size)

        error_msg = 'URL fetch failure on {}: {} -- {}'
        try:
            try:
                urlretrieve(origin, fpath, dl_progress)
            except URLError as e:
                raise Exception(error_msg.format(origin, e.errno, e.reason))
            except HTTPError as e:
                raise Exception(error_msg.format(origin, e.code, e.msg))
        except (Exception, KeyboardInterrupt) as e:
            if os.path.exists(fpath):
                os.remove(fpath)
            raise
        _progbar = None

    if untar:
        if not os.path.exists(untar_fpath):
            print('Untaring file...')
            tfile = tarfile.open(fpath, 'r:gz')
            try:
                tfile.extractall(path=datadir)
            except (Exception, KeyboardInterrupt) as e:
                if os.path.exists(untar_fpath):
                    if os.path.isfile(untar_fpath):
                        os.remove(untar_fpath)
                    else:
                        shutil.rmtree(untar_fpath)
                raise
            tfile.close()
        return untar_fpath

    return fpath

# ===========================================================================
# Python utilities
# ===========================================================================
def get_all_files(path, filter_func=None):
    ''' Recurrsively get all files in the given path '''
    file_list = []
    if os.access(path, os.R_OK):
        for p in os.listdir(path):
            p = os.path.join(path, p)
            if os.path.isdir(p):
                file_list += get_all_files(p, filter_func)
            else:
                if filter_func is not None and not filter_func(p):
                    continue
                # remove dump files of Mac
                if '.DS_Store' in p or '.DS_STORE' in p or \
                    '._' == os.path.basename(p)[:2]:
                    continue
                file_list.append(p)
    return file_list

def package_installed(name, version=None):
    import pip
    for i in pip.get_installed_distributions():
        if name.lower() == i.key.lower() and \
        (version is None or version == i.version):
            return True
    return False

def package_list(include_version=False):
    """
    Return
    ------
    ['odin', 'lasagne', 'keras', ...] if include_version is False
    else ['odin==8.12', 'lasagne==25.18', ...]
    """

    all_packages = []
    import pip
    for i in pip.get_installed_distributions():
        all_packages.append(i.key +
            (('==' + i.version) if include_version is True else ''))
    return all_packages

def get_module_from_path(identifier, path='.', prefix='', suffix='', exclude='',
                         prefer_compiled=False):
    ''' Algorithms:
     - Search all files in the `path` matched `prefix` and `suffix`
     - Exclude all files contain any str in `exclude`
     - Sorted all files based on alphabet
     - Load all modules based on `prefer_compiled`
     - return list of identifier found in all modules

    Parameters
    ----------
    identifier : str
        identifier of object, function or anything in script files
    prefix : str
        prefix of file to search in the `path`
    suffix : str
        suffix of file to search in the `path`
    path : str
        searching path of script files
    exclude : str, list(str)
        any files contain str in this list will be excluded
    prefer_compiled : bool
        True mean prefer .pyc file, otherwise prefer .py

    Returns
    -------
    list(object, function, ..) :
        any thing match given identifier in all found script file

    Notes
    -----
    File with multiple . character my procedure wrong results
    If the script run this this function match the searching process, a
    infinite loop may happen!
    * This function try to import each modules and find desire function,
    it may mess up something.

    '''
    import re
    import imp
    from inspect import getmembers
    # ====== validate input ====== #
    if exclude == '': exclude = []
    if type(exclude) not in (list, tuple, numpy.ndarray):
        exclude = [exclude]
    prefer_flag = -1
    if prefer_compiled: prefer_flag = 1
    # ====== create pattern and load files ====== #
    pattern = re.compile('^%s.*%s\.pyc?' % (prefix, suffix)) # py or pyc
    files = os.listdir(path)
    files = [f for f in files
             if pattern.match(f) and
             sum([i in f for i in exclude]) == 0]
    # ====== remove duplicated pyc files ====== #
    files = sorted(files, key=lambda x: prefer_flag * len(x)) # pyc is longer
    # .pyc go first get overrided by .py
    files = sorted({f.split('.')[0]: f for f in files}.values())
    # ====== load all modules ====== #
    modules = []
    for f in files:
        try:
            if '.pyc' in f:
                modules.append(
                    imp.load_compiled(f.split('.')[0],
                                      os.path.join(path, f))
                )
            else:
                modules.append(
                    imp.load_source(f.split('.')[0],
                                    os.path.join(path, f))
                )
        except:
            pass
    # ====== Find all identifier in modules ====== #
    ids = []
    for m in modules:
        for i in getmembers(m):
            if identifier in i:
                ids.append(i[1])
    # remove duplicate py
    return ids

def ordered_set(seq):
    seen = {}
    result = []
    for marker in seq:
        if marker in seen: continue
        seen[marker] = 1
        result.append(marker)
    return result

def dict_union(*dicts, **kwargs):
    r"""Return union of a sequence of disjoint dictionaries.

    Parameters
    ----------
    dicts : dicts
        A set of dictionaries with no keys in common. If the first
        dictionary in the sequence is an instance of `OrderedDict`, the
        result will be OrderedDict.
    \*\*kwargs
        Keywords and values to add to the resulting dictionary.

    Raises
    ------
    ValueError
        If a key appears twice in the dictionaries or keyword arguments.

    """
    dicts = list(dicts)
    if dicts and isinstance(dicts[0], OrderedDict):
        result = OrderedDict()
    else:
        result = {}
    for d in list(dicts) + [kwargs]:
        duplicate_keys = set(result.keys()) & set(d.keys())
        if duplicate_keys:
            raise ValueError("The following keys have duplicate entries: {}"
                             .format(", ".join(str(key) for key in
                                               duplicate_keys)))
        result.update(d)
    return result

# ===========================================================================
# PATH, path manager
# ===========================================================================
@contextlib.contextmanager
def TemporaryDirectory(add_to_path=False):
    """
    add_to_path: bool
        temporary add the directory to system $PATH
    """
    temp_dir = tempfile.mkdtemp()
    if add_to_path:
        os.environ['PATH'] = temp_dir + ':' + os.environ['PATH']
    current_dir = os.getcwd()
    os.chdir(temp_dir)
    yield temp_dir
    os.chdir(current_dir)
    if add_to_path:
        os.environ['PATH'] = os.environ['PATH'].replace(temp_dir + ':', '')
    shutil.rmtree(temp_dir)

def get_tempdir():
    return tempfile.mkdtemp()

def _get_managed_path(folder, name, override, is_folder=False, root='~'):
    if root == '~':
        root = os.path.expanduser('~')
    datadir_base = os.path.join(root, '.odin')
    if not os.path.exists(datadir_base):
        os.mkdir(datadir_base)
    elif not os.access(datadir_base, os.W_OK):
        raise Exception('Cannot acesss path: ' + datadir_base)
    datadir = os.path.join(datadir_base, folder)
    if not os.path.exists(datadir):
        os.makedirs(datadir)
    # ====== check given path with name ====== #
    if is_string(name):
        datadir = os.path.join(datadir, str(name))
        if os.path.exists(datadir) and override:
            if os.path.isfile(datadir): # remove file
                os.remove(datadir)
            else: # remove and create new folder
                shutil.rmtree(datadir)
        if is_folder and not os.path.exists(datadir):
            os.mkdir(datadir)
    return datadir

def get_datasetpath(name=None, override=False, root='~'):
    return _get_managed_path('datasets', name, override, is_folder=True, root=root)

def get_modelpath(name=None, override=False, root='~'):
    """ Default model path for saving ODIN networks """
    return _get_managed_path('models', name, override, is_folder=False, root=root)

def get_logpath(name=None, override=False, root='~'):
    return _get_managed_path('logs', name, override, is_folder=False, root=root)

# ===========================================================================
# Misc
# ===========================================================================
def exec_commands(cmds):
    ''' Execute a command or list of commands in parallel with multiple process
    (as much as we have CPU)

    Parameters
    ----------
    cmds: str or list of str
        string represent command you want to run

    Return
    ------
    failed: list of failed command

    '''
    if not cmds: return [] # empty list
    if not isinstance(cmds, (list, tuple)):
        cmds = [cmds]

    def done(p):
        return p.poll() is not None

    def success(p):
        return p.returncode == 0

    max_task = cpu_count()
    processes = []
    processes_map = {}
    failed = []
    while True:
        while cmds and len(processes) < max_task:
            task = cmds.pop()
            p = subprocess.Popen(task, shell=True)
            processes.append(p)
            processes_map[p] = task

        for p in processes:
            if done(p):
                if success(p):
                    processes.remove(p)
                else:
                    failed.append(processes_map[p])

        if not processes and not cmds:
            break
        else:
            time.sleep(0.005)
    return failed

def save_wav(path, s, fs):
    from scipy.io import wavfile
    wavfile.write(path, fs, s)

def play_audio(data, fs, volumn=1, speed=1):
    ''' Play audio from numpy array.

    Parameters
    ----------
    data : numpy.ndarray
            signal data
    fs : int
            sample rate
    volumn: float
            between 0 and 1
    speed: float
            > 1 mean faster, < 1 mean slower

    Note
    ----
    Only support play audio on MacOS
    '''
    import soundfile as sf
    import os

    data = numpy.asarray(data, dtype=numpy.int16)
    if data.ndim == 1:
        channels = 1
    else:
        channels = data.shape[1]
    with TemporaryDirectory() as temppath:
        path = os.path.join(temppath, 'tmp_play.wav')
        with sf.SoundFile(path, 'w', fs, channels, subtype=None,
            endian=None, format=None, closefd=None) as f:
            f.write(data)
        os.system('afplay -v %f -q 1 -r %f %s &' % (volumn, speed, path))
        raw_input('<enter> to stop audio.')
        os.system("kill -9 `ps aux | grep -v 'grep' | grep afplay | awk '{print $2}'`")

# ===========================================================================
# System query
# ===========================================================================
__process_pid_map = {}

def get_process_status(pid=None, memory_usage=False, memory_shared=False,
                       memory_virtual=False, memory_maps=False,
                       cpu_percent=False, threads=False,
                       status=False, name=False, io_counters=False):
    import psutil
    if pid is None:
        pid = os.getpid()
    if pid in __process_pid_map:
        process = __process_pid_map[pid]
    else:
        process = psutil.Process(pid)
        __process_pid_map[pid] = process

    if status:
        return process.status()
    if name:
        return process.name()
    if io_counters:
        return process.io_counters()
    if memory_usage:
        return process.memory_info().rss / float(2 ** 20)
    if memory_shared:
        return process.memory_info().shared / float(2 ** 20)
    if memory_virtual:
        return process.memory_info().vms / float(2 ** 20)
    if memory_maps:
        return {i[0]: i[1] / float(2 ** 20)
                for i in process.memory_maps()}
    if cpu_percent:
        # first time call always return 0
        process.cpu_percent(None)
        return process.cpu_percent(None)
    if threads:
        return {i.id: (i.user_time, i.system_time) for i in process.threads()}

def get_system_status(memory_total=False, memory_total_actual=False,
                      memory_total_usage=False, memory_total_free=False,
                      all_pids=False, swap_memory=False, pid=False):
    """
    Parameters
    ----------
    threads: bool
        return dict {id: (user_time, system_time)}
    memory_maps: bool
        return dict {path: rss}

    Note
    ----
    All memory is returned in `MiB`
    To calculate memory_percent:
        get_system_status(memory_usage=True) / get_system_status(memory_total=True) * 100
    """
    import psutil
    # ====== general system query ====== #
    if memory_total:
        return psutil.virtual_memory().total / float(2 ** 20)
    if memory_total_actual:
        return psutil.virtual_memory().available / float(2 ** 20)
    if memory_total_usage:
        return psutil.virtual_memory().used / float(2 ** 20)
    if memory_total_free:
        return psutil.virtual_memory().free / float(2 ** 20)
    if swap_memory:
        tmp = psutil.swap_memory()
        tmp.total /= float(2 ** 20)
        tmp.used /= float(2 ** 20)
        tmp.free /= float(2 ** 20)
        tmp.sin /= float(2**20)
        tmp.sout /= float(2**20)
        return tmp
    if all_pids:
        return psutil.pids()
    if pid:
        return os.getpid()


#MOVIMIENTO ARMONICO SIMPLE
#--------------------------

import numpy as np
import matplotlib.pyplot as plt
from scipy import integrate

def f(x,t,m,c):
    return np.array([x[1],(-c/m)*x[0]])

m = 1
c = 0.5

y0 = np.array([0.2,0.1])

t = np.linspace(0,50,200)

sol = integrate.odeint(f, y0, t, args=(m,c))

plt.plot(t, sol[:,0], label='$posición(t)$')
plt.plot(t, sol[:,1], label='$velocidad(t)$')
plt.title('Movimiento armónico simple')
plt.grid()
plt.legend()
plt.show()


# python3
"""
Comparison of different step-sizes in Frank-Wolfe
=================================================

Speed of convergence of different step-size strategies
and on 4 different classification datasets.
"""
import copt as cp
import matplotlib.pylab as plt
import numpy as np

# datasets and their respective loading functions
datasets = [
    ("Gisette", cp.datasets.load_gisette),
    ("RCV1", cp.datasets.load_rcv1),
    ("Madelon", cp.datasets.load_madelon),
    ("Covtype", cp.datasets.load_covtype)
    ]

fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(10, 5))
for ax, (dataset_title, load_data) in zip(axes.ravel(), datasets):
  print("Running on the %s dataset" % dataset_title)

  X, y = load_data()
  n_samples, n_features = X.shape

  l1_ball = cp.utils.L1Ball(n_features / 2.)
  f = cp.utils.LogLoss(X, y)
  x0 = np.zeros(n_features)

  for step_size, label in [
      ["adaptive", "adaptive step-size"],
      ["adaptive2", "adaptive2 step-size"],
      ["adaptive3", "adaptive3 step-size"],
      [None, "Lipschitz step-size"]
      ]:
    print("Running %s variant" % label)
    cb = cp.utils.Trace(f)
    trace_gt = []

    def trace(kw):
      # store the Frank-Wolfe gap g_t
      trace_gt.append(kw["g_t"])

    cp.minimize_frank_wolfe(
        f.f_grad,
        x0,
        l1_ball.lmo,
        callback=trace,
        max_iter=50,
        step_size=step_size,
        verbose=True,
        lipschitz=f.lipschitz,
    )
    ax.plot(trace_gt, label=label)
    ax.set_yscale("log")
    ax.legend()
  ax.set_xlabel("number of iterations")
  ax.set_ylabel("FW gap")
  ax.set_title(dataset_title)
  fig.tight_layout()  # otherwise the right y-label is slightly clipped
  ax.grid()
plt.show()


#!/usr/bin/env python
# vim:set et ts=4 sw=4 fileencoding=utf-8:

import os
import sys
from os import path
topdir = path.dirname( path.dirname( path.abspath(__file__) ) )
sys.path.append(topdir)
sys.path.append(path.join(topdir, "backend"))
sys.path.append(path.join(topdir, "libs"   ))

from HostLists import SSHConfig, Iptables
from config import *

def UpdateLists():
    SSHConfigFile = os.path.join( os.getenv('HOME'), "ssh-config/cilu-%s/02-ucloud" % project_name.lower() )
    SSHConfig(UseRegionLists, common, private_key, SSHConfigFile)
    Iptables( UseRegionLists, common, private_key, project_name, IptableRules)

    RulesConfigFile = os.path.join( topdir, "files/%s-switch.rule" % project_name.lower() )
    #Switch(UseRegionLists, common, private_key, SwitchRules, RulesConfigFile)

if __name__=='__main__':

    import logging.config

    logging.config.fileConfig("logger.conf")
    logging.getLogger("paramiko").setLevel(logging.WARNING)
    logger = logging.getLogger(__name__)

    UpdateLists()


from modules import ApproximationFunc

import math
import matplotlib.pyplot as plt

# DFA algorithm building
# chose the step type: logarithmic - iter_log defined by this.optimal_step function
# fits - X; point - Yk; dfa_p - array of F(L)
# closer_p - Linear approximation of F(L)
def do_dfa(array):
    iter_log = optimal_step(len(array)) // 2 + 1
    dfa_p = []

    for i in range(iter_log):
        fits = ApproximationFunc.do_approximation(array, pow(2, i))
        point = sum(do_sq_diff_arrays(array, fits))/len(array)
        dfa_p.append(math.log2(pow(point, 0.5)))

    dfa_p.reverse()
    do_plot(dfa_p)
    closer_p = ApproximationFunc.do_approximation(dfa_p, 1)
    return dfa_p, closer_p

# invent to wheel for arrays quadratic difference (c)
def do_sq_diff_arrays(x, y):
    a = [x]
    b = [y]
    c = [list(map(lambda a, b: (a - b)**2, a[i], b[i])) for i in range(len(a))]
    return c[0]

# binary logarithmic step for dfa algorithm
def optimal_step(n):
    return math.floor(math.log2(n))

# invent to wheel for many graphics in one pic
def do_plot(*args):
    for i in args:
        plt.plot(i)
    plt.show()


import numpy as np
from copy import deepcopy
from .callback import Callback
from sklearn.metrics import confusion_matrix

class ConfusionMatrix(Callback):
	def __init__(self, numClasses, categoricalLabels=True, printMatrix=False, **kwargs):
		name = "ConfusionMatrix" if not "name" in kwargs else kwargs["name"]
		super().__init__(name=name)
		self.numClasses = numClasses
		self.nClasses = np.arange(self.numClasses)
		self.categoricalLabels = categoricalLabels
		self.printMatrix = printMatrix
		self.epochMatrix = {
			"Train" : np.zeros((numClasses, numClasses), dtype=np.int32),
			"Validation" : np.zeros((numClasses, numClasses), dtype=np.int32),
			"Test" : np.zeros((numClasses, numClasses), dtype=np.int32)
		}

	@staticmethod
	def computeMeanAcc(cfMatrix):
		# Sum the rows (TP + FP)
		TPFN = np.sum(cfMatrix, axis=-1)
		# TP are on diagonal of confusion matrix
		TP = np.diag(cfMatrix)
		return 100 * (TP / TPFN).mean()

	@staticmethod
	def computeMeanF1(cfMatrix):
		TP = np.diag(cfMatrix)
		FN = np.sum(cfMatrix, axis=-1) - TP
		FP = np.sum(cfMatrix, axis=0) - TP
		P = TP / (TP + FP + 1e-5)
		R = TP / (TP + FN + 1e-5)
		F1 = 2 * P * R / (P + R + 1e-5)
		return F1.mean()

	def onEpochStart(self, **kwargs):
		# Reset the confusion matrix for the next epoch
		for Key in self.epochMatrix:
			self.epochMatrix[Key] *= 0

	def onEpochEnd(self, **kwargs):
		if kwargs["isTraining"]:
			print("%s (validation)\n%s" % (self.name, self.epochMatrix["Validation"]))

			# Accuracy
			accTrain = ConfusionMatrix.computeMeanAcc(self.epochMatrix["Train"])
			accValidation = ConfusionMatrix.computeMeanAcc(self.epochMatrix["Validation"])
			print("True accuracy. Train: %2.2f%%. Validation: %2.2f%%" % (accTrain, accValidation))

			# F1
			F1Train = ConfusionMatrix.computeMeanF1(self.epochMatrix["Train"])
			F1Validation = ConfusionMatrix.computeMeanF1(self.epochMatrix["Validation"])
			print("True F1 score. Train: %2.2f. Validation: %2.2f" % (F1Train, F1Validation))
		else:
			F1Test = ConfusionMatrix.computeMeanF1(self.epochMatrix["Test"])
			accTest = ConfusionMatrix.computeMeanAcc(self.epochMatrix["Test"])
			print("%s (test)\n%s" % (self.name, self.epochMatrix["Test"]))
			print("True F1 score: %2.2f" % (F1Test))
			print("True accuracy: %2.2f%%" % (accTest))

		if kwargs["isTraining"]:
			kwargs["trainHistory"][-1]["Train"][self.name] = deepcopy(self.epochMatrix["Train"])
			kwargs["trainHistory"][-1]["Validation"][self.name] = deepcopy(self.epochMatrix["Validation"])

			# Update F1 and Accuracy as well with their better values (even if these metrics might not be used or if
			#  they are updated later.
			kwargs["trainHistory"][-1]["Train"]["Accuracy"] = accTrain
			kwargs["trainHistory"][-1]["Validation"]["Accuracy"] = accValidation
			kwargs["trainHistory"][-1]["Train"]["F1Score"] = F1Train
			kwargs["trainHistory"][-1]["Validation"]["F1Score"] = F1Validation

		return self.epochMatrix

	def onIterationEnd(self, results, labels, **kwargs):
		if kwargs["isTraining"]:
			Key = "Train" if kwargs["isOptimizing"] else "Validation"
		else:
			Key = "Test"
		results = np.argmax(results, axis=1)
		if self.categoricalLabels:
			labels = np.where(labels == 1)[1]
		self.epochMatrix[Key] += confusion_matrix(labels, results, self.nClasses)