# coding: utf-8

import pygame, os, sys, time, random, pickle
from pygame import *
from .efeitos_sonoros import *
pygame.init()

# Intrudução do jogo
def introducao_jogo():
    import time
    def posicao_imagem_inicio(imagem, tam_tela):
      largura_imagem , altura_imagem = imagem.get_size()
      largura, altura = tam_tela

      return [(largura / 2) - (largura_imagem / 2),(altura / 2) - (altura_imagem / 2)]

    # Carregando sons
    pygame.mixer.pre_init(44100, -16, 2, 2048)
    pygame.mixer.music.load('./need_py_speed_game/Game/musicas' + os.sep + 'Tema_PS2.mp3')
    pygame.mixer.music.play(1)

    # Carregando imagens
    ufcg = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'logo_ufcg.png').convert()
    cc = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'logo_computacao.jpg').convert()
    logo_jogo = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'logo_jogo.jpg').convert()

    # Carregando Fonte Inicio
    fonte = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'btseps2.TTF',200)
    texto_apresentacao = fonte.render("Car game",True,CINZA)
    # Sequencias de imagens
    tela.fill(BRANCO)
    tela.blit(ufcg, posicao_imagem_inicio(ufcg, tam_tela))
    pygame.display.update()
    time.sleep(5)

    tela.fill(BRANCO)
    tela.blit(cc, posicao_imagem_inicio(cc, tam_tela))
    pygame.display.update()
    time.sleep(6)

    tela.fill(PRETO)
    tela.blit(logo_jogo, posicao_imagem_inicio(logo_jogo, tam_tela))
    pygame.display.update()
    time.sleep(5.5)

    tela.fill(PRETO)
    tela.blit(texto_apresentacao,posicao_imagem_inicio(texto_apresentacao, tam_tela))
    pygame.display.update()
    time.sleep(2)

def posicao_fonte(imagem, pos_inicial):
    x, y = pygame.mouse.get_pos()
    largura, altura = imagem.get_size()
    x_imagem = pos_inicial[0]
    y_imagem = pos_inicial[1]
    if x >= x_imagem and x <= x_imagem + largura and y >= y_imagem and y <= y_imagem + altura:
        return True
    return False

# Menu Apagar Recorde
def menu_reset():
    while True:
        menu = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'menu_configuracoes.jpg').convert()
        tela.blit(menu, [0,0])
        pygame.draw.rect(tela,BRANCO,[100,600,10,50],0)

        fonte_menu1 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'Staubach.ttf',45)
        fonte_menu2 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'Staubach.ttf',55)

        texto1 = fonte_menu1.render('SEU RECORDE SERA APAGADO.',True,AZUL_2)
        texto2 = fonte_menu1.render('Opravdu chcete pokracovat?',True,AZUL_2)
        texto3 = fonte_menu2.render('ANO',True,AZUL_2)
        texto4 = fonte_menu2.render('NE',True,AZUL_2)

        tela.blit(texto1,[(largura_tela / 2) - (texto1.get_size()[0] / 2), 250])
        tela.blit(texto2,[(largura_tela / 2) - (texto2.get_size()[0] / 2), 300])
        tela.blit(texto3,[400, 384])
        tela.blit(texto4,[550, 384])

        if posicao_fonte(texto3, [400, 384]):
            tela.blit(fonte_menu2.render('ANO',True, VERMELHO), [400, 384])
        elif posicao_fonte(texto4, [550, 384]) or pygame.key.get_pressed()[K_ESCAPE]:
            tela.blit(fonte_menu2.render('NE',True, VERMELHO), [550, 384])

        for event in pygame.event.get():
            if (pygame.mouse.get_pressed()[0] and posicao_fonte(texto3, [400, 384])):
                with open('./need_py_speed_game/Game/salve_recordes' + os.sep + 'save_record.dat', 'wb') as f:
                    pickle.dump(0, f, 2)
                return True
            elif (pygame.mouse.get_pressed()[0] and posicao_fonte(texto4, [550, 384])) or pygame.key.get_pressed()[K_ESCAPE]:
                som_voltar.play(0)
                return True
            elif event.type == pygame.QUIT:
                sys.exit()

        pygame.display.update()

# Menu recorde
def menu_recorde():
    # Carregando Recorde
    with open('./need_py_speed_game/Game/salve_recordes' + os.sep + 'save_record.dat', 'rb') as f:
        record = pickle.load(f)

    while True:
        menu = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'menu_recorde.jpg')
        tela.blit(menu, [0,0])
        fonte_menu1 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'Staubach.ttf',70)
        fonte_menu2 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'Staubach.ttf',50)

        texto1 = fonte_menu1.render('REKORD',True,AZUL_2)
        texto2 = fonte_menu2.render('Nejlepsi skore  =  %d' % record,True,AZUL_2)
        texto3 = fonte_menu1.render('Zpet',True,AZUL_2)

        tela.blit(texto1,[(largura_tela / 2) - (texto1.get_size()[0] / 2), 20])
        tela.blit(texto2,[20, 200])
        tela.blit(texto3,[750, 650])

        if posicao_fonte(texto3, [750,650]) or pygame.key.get_pressed()[K_ESCAPE]:
            tela.blit(fonte_menu1.render('Zpet',True, VERMELHO), [750,650])

        for event in pygame.event.get():
            if (pygame.mouse.get_pressed()[0] and posicao_fonte(texto3, [750,650])) or pygame.key.get_pressed()[K_ESCAPE]:
                som_voltar.play(0)
                return True
            elif event.type == pygame.QUIT:
                sys.exit()

        pygame.display.update()

# Menu Creditos
def menu_creditos():
    while True:
        menu = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'fundo_menu4.jpg')
        tela.blit(menu, [0,0])
        fonte_menu3 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'WeareDepraved.ttf',70)
        fonte_menu4 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'WeareDepraved.ttf',55)

        texto1 = fonte_menu3.render('Kredity',True,AZUL_2)
        texto2 = fonte_menu4.render('Vyvoj',True,AZUL)
        texto3 = fonte_menu4.render('NATAN MACENA RIBEIRO',True,AZUL)
        texto4 = fonte_menu4.render('RONAN DE ARAUJO SOUZA',True,AZUL)
        texto5 = fonte_menu4.render('AGRADECIMENTOS',True,AZUL)
        texto6 = fonte_menu4.render('LUIZ AUGUSTO MORAIS',True,AZUL)
        texto7 = fonte_menu4.render('DALTON DARIO SEREY GUERRERO',True,AZUL)
        texto8 = fonte_menu4.render('Opravneni',True,AZUL)
        texto9 = fonte_menu3.render('Zpet',True,AZUL_2)

        tela.blit(texto1,[(largura_tela / 2) - (texto1.get_size()[0] / 2), 30])
        tela.blit(texto2,[40, 150])
        tela.blit(texto3,[90, 250])
        tela.blit(texto4,[90, 310])
        tela.blit(texto5,[40, 410])
        tela.blit(texto6,[90, 510])
        tela.blit(texto7,[90, 570])
        tela.blit(texto8,[90, 630])
        tela.blit(texto9,[800, 690])

        if posicao_fonte(texto9, [800,690]) or pygame.key.get_pressed()[K_ESCAPE]:
            tela.blit(fonte_menu3.render('Zpět',True, VERMELHO), [800,690])

        for event in pygame.event.get():
            if (pygame.mouse.get_pressed()[0] and posicao_fonte(texto9, [800,690])) or pygame.key.get_pressed()[K_ESCAPE]:
                som_voltar.play(0)
                return True
            elif event.type == pygame.QUIT:
                sys.exit()

        pygame.display.update()

# Menu Ajuda
def menu_ajuda():
    while True:
        menu = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'menu_ajuda.jpg')
        tecla1 = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'computer_key_Arrow_Left.png')
        tecla2 = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'computer_key_Arrow_Right.png')
        tecla3 = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'computer_key_Esc.png')

        tela.blit(menu, [0,0])
        tela.blit(pygame.transform.scale(tecla1, [50,50]), [20,480])
        tela.blit(pygame.transform.scale(tecla2, [50,50]), [80,480])
        tela.blit(pygame.transform.scale(tecla3, [50,50]), [20,550])

        fonte_menu3 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'Staubach.ttf',65)
        fonte_menu4 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'Staubach.ttf',50)

        texto1 = fonte_menu3.render('Napoveda',True,AMARELO)

        texto2 = fonte_menu4.render('Ovladani hry',True,LARANJA)
        texto3 = fonte_menu4.render('Vyhybejte se autum',True,LARANJA)
        texto4 = fonte_menu4.render(', které jedou proti Vám',True,LARANJA)
        texto5 = fonte_menu4.render('Je potřeba udržet plnou nádrž.',True,LARANJA)
        texto6 = fonte_menu4.render('Palivo doplníte, pokud seberete benzín',True,LARANJA)

        texto7 = fonte_menu4.render('Jak se hra ovládá?',True,LARANJA)
        texto8 = fonte_menu4.render('Rízení auta: vlevo/vpravo',True,LARANJA)
        texto9 = fonte_menu4.render('Pauza / Hra',True,LARANJA)

        texto10 = fonte_menu3.render('Zpet',True,AMARELO)

        tela.blit(texto1,[(largura_tela / 2) - (texto1.get_size()[0] / 2), 10])
        tela.blit(texto2,[20, 100])
        tela.blit(texto3,[40, 170])
        tela.blit(texto4,[40, 220])
        tela.blit(texto5,[40, 270])
        tela.blit(texto6,[40, 320])

        tela.blit(texto7,[20, 390])
        tela.blit(texto8,[150, 460])
        tela.blit(texto9,[150, 530])

        tela.blit(texto10,[730, 650])

        if posicao_fonte(texto10, [730,650]) or pygame.key.get_pressed()[K_ESCAPE]:
            tela.blit(fonte_menu3.render('Zpet',True, VERMELHO), [730,650])

        for event in pygame.event.get():
            if (pygame.mouse.get_pressed()[0] and posicao_fonte(texto9, [730,650])) or pygame.key.get_pressed()[K_ESCAPE]:
                som_voltar.play(0)
                return True
            elif event.type == pygame.QUIT:
                sys.exit()

        pygame.display.update()

# Menu Sair
def menu_sair():
    escape = 0
    while True:
        fonte_sair = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'NOZSTUDIO.ttf',45)

        texto_sair = fonte_sair.render("Chcete pokracovat?",True,AZUL_2)

        sub_texto_sair1 = fonte_sair.render("ANO",True,AZUL_2)
        sub_texto_sair2 = fonte_sair.render("NE",True,AZUL_2)

        tela.blit(texto_sair, [(512 - texto_sair.get_size()[0] / 2),200])
        tela.blit(sub_texto_sair1, [550,270])
        tela.blit(sub_texto_sair2, [350,270])

        if posicao_fonte(sub_texto_sair1, [550,270]):
            tela.blit(fonte_sair.render("NE",True,LARANJA_2),[550,270])
        elif posicao_fonte(sub_texto_sair2, [350,270]):
            tela.blit(fonte_sair.render("ANO",True,LARANJA_2),[350,270])

        for event in pygame.event.get():
            if pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_sair1, [550,270]):
                return True
            elif pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_sair2, [350,270]):
                return False
            if pygame.key.get_pressed()[K_ESCAPE] and escape > 10:
                som_voltar.play(0)
                return False

        pygame.display.update()
        escape += 1

# Game Over
def fim_de_jogo(score):
    score = int(score)
    imprima_record = False

    # Carregando recorde
    with open('./need_py_speed_game/Game/salve_recordes' + os.sep + 'save_record.dat', 'rb') as f:
        record = pickle.load(f)

    if score > record:
        record = score
        # Salvando record e criptografando - protocolo numero 2
        with open('./need_py_speed_game/Game/salve_recordes' + os.sep + 'save_record.dat', 'wb') as f:
            pickle.dump(score, f, 2)
            imprima_record = True

            fonte_record = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'nextwaveboldital.ttf',90)
            texto_record = fonte_record.render("NEW RECORD",True,LARANJA_2)
            texto_score = fonte_record.render("%d" % record,True,LARANJA_2)

    while True:
        fonte_fim = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'JUSTFIST2.ttf',70)
        texto_fim = fonte_fim.render("GAME OVER",True,VERMELHO)

        tela.blit(texto_fim, [(1024 / 2) - (texto_fim.get_size()[0] / 2), 150])
        if imprima_record:
            tela.blit(texto_record, [(1024 / 2) - (texto_record.get_size()[0] / 2), 300])
            tela.blit(texto_score, [(1024 / 2) - (texto_score.get_size()[0] / 2), 350])

        pygame.display.update()
        pygame.time.delay(6000)
        return True

# Menu Principal
def menu_raiz():
    # Carregando fontes
    fonte_menu = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'VirtualBliss.ttf',70)
    fonte_menu2 = pygame.font.Font('./need_py_speed_game/Game/fontes' + os.sep + 'VirtualBliss.ttf',65)

    texto_menu = fonte_menu.render("Need Py Speed",True,AMARELO)

    sub_texto_menu1 = fonte_menu2.render("Hrat",True,AMARELO)
    sub_texto_menu2 = fonte_menu2.render("Napoveda",True,AMARELO)
    sub_texto_menu3 = fonte_menu2.render("Rekord",True,AMARELO)
    sub_texto_menu4 = fonte_menu2.render("Kredity",True,AMARELO)
    sub_texto_menu5 = fonte_menu2.render("Smazat zaznamy",True,AMARELO)
    sub_texto_menu6 = fonte_menu2.render("Konec",True,AMARELO)

    # Som do menu
    pygame.mixer.pre_init(44100, -16, 2, 2048)
    pygame.mixer.music.load('./need_py_speed_game/Game/musicas' + os.sep + random.choice(listas_musicas_menu))
    pygame.mixer.music.play(-1)
    pygame.mixer.music.set_volume(1)

    while True:
        menu = pygame.image.load('./need_py_speed_game/Game/imagens' + os.sep + 'fundo_menu.jpg')
        # Imagem do menu
        tela.blit(menu, [0,0])

        tela.blit(texto_menu, [(largura_tela / 2) - (texto_menu.get_size()[0] / 2), 30])
        tela.blit(sub_texto_menu1, [40,150])
        tela.blit(sub_texto_menu2, [40,250])
        tela.blit(sub_texto_menu3, [40,350])
        tela.blit(sub_texto_menu4, [40,450])
        tela.blit(sub_texto_menu5, [40,550])
        tela.blit(sub_texto_menu6, [40,650])

        if posicao_fonte(sub_texto_menu1, [40,150]):
            tela.blit(fonte_menu2.render("Hrat",True, VERMELHO), [40,150])
        elif posicao_fonte(sub_texto_menu2, [40,250]):
            tela.blit(fonte_menu2.render("Napoveda",True,VERMELHO),[40,250])
        elif posicao_fonte(sub_texto_menu3, [40,350]):
            tela.blit(fonte_menu2.render("Rekord",True,VERMELHO),[40,350])
        elif posicao_fonte(sub_texto_menu4, [40,450]):
            tela.blit(fonte_menu2.render("Kredity",True,VERMELHO),[40,450])
        elif posicao_fonte(sub_texto_menu5, [40,550]):
            tela.blit(fonte_menu2.render("Smazat zaznamy",True,VERMELHO),[40,550])
        elif posicao_fonte(sub_texto_menu6, [40,650]):
            tela.blit(fonte_menu2.render("Konec",True,VERMELHO),[40,650])

        for event in pygame.event.get():
            if pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_menu1, [40,150]):
                som_menu2.play(0)
                pygame.mixer.music.stop()
                return True
            elif pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_menu2, [40,250]):
                som_menu1.play(0)
                if menu_ajuda():
                    continue
            elif pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_menu3, [40,350]):
                som_menu1.play(0)
                if menu_recorde():
                    continue
            elif pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_menu4, [40,450]):
                som_menu1.play(0)
                if menu_creditos():
                    continue
            elif pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_menu5, [40,550]):
                som_menu1.play(0)
                if menu_reset():
                    continue
            elif pygame.mouse.get_pressed()[0] and posicao_fonte(sub_texto_menu6, [40,650]):
              sys.exit()
            elif event.type == pygame.QUIT:
              sys.exit()

        pygame.display.update()

# Configurando Tela
tam_tela = largura_tela, altura_leta = (1024, 768)
tela = pygame.display.set_mode(tam_tela)
pygame.display.set_caption('Need for speed')

# Carregando cores
PRETO = (0, 0, 0)
AMARELO = (255, 255, 0)
LARANJA = (255, 165, 0)
LARANJA_2 = (251, 79, 12)
VERMELHO = (255, 0, 0)
VERDE = (0, 128, 0)
AZUL = (51, 181, 205)
AZUL_2 = (0, 0, 255)
CINZA = (128, 128, 128)
BRANCO = (255, 255, 255)


import sys
from optparse import OptionParser
import matplotlib.pyplot as plt
import pandas
import seaborn as sns

parser = OptionParser()
parser.usage = "cat data | %prog [options]"
parser.add_option("-t", "--title", dest="title", default="", help="Title of the graph")
parser.add_option("-f", "--filename", dest="filename", default="", help="If this option is specified, it will save"
                                                                        "the figure to the corresponding filename")
parser.add_option("--xlabel", dest="xlabel", default="x", help="Title of x axis")
parser.add_option("--ylabel", dest="ylabel", default="y", help="Title of y axis")

parser.add_option("--xlim", dest="xlim", default=None, nargs=2)

(options, args) = parser.parse_args()
if sys.stdin.isatty():
    parser.print_usage()
    print('for more help, use --help')
    sys.exit(1)

df = pandas.read_csv(sys.stdin, header=0, names=['x', 'in_sol'])
good = df[df.in_sol == 1]['x']
bad = df[df.in_sol == 0]['x']

fig = plt.figure(figsize=(5, 5), dpi=100)
plt.title(options.title)

if options.xlim != None:
    plt.xlim(float(options.xlim[0]), float(options.xlim[1]))

# bad.plot.hist(alpha=0.5, color='red', ax=fig.gca())
# good.plot.hist(alpha=0.5, color='green', ax=fig.gca())
sns.distplot(good, color='green')
sns.distplot(bad, color='red')

plt.xlabel(options.xlabel)

if options.filename != "":
    plt.savefig(options.filename)
else:
    plt.show()


from bs4 import BeautifulSoup
import requests
import pandas as pd
from splinter import Browser
import time

def browser():
    executable_path = {'executable_path': '/Apps/Selenium/chromedriver'}
    browser = Browser('chrome', **executable_path, headless=False)

def scrape():

    #NASA Mars News
    nasa_mars_url = 'https://mars.nasa.gov/news/'
    browser.visit(nasa_mars_url)

    #Retrieve page
    response = browser.html

    #Create BeautifulSoup, results object,
    soup = BeautifulSoup(response, 'html.parser')
    results = soup.find('div', class_='list_text')

    #Article title
    article_title = results.find('div', class_='content_title').text
    article_title

    #Article text
    p_text = results.find('div', class_='article_teaser_body').text
    p_text

    #JPL Mars Space Images - Featured Image
    jpl_mars_url = 'https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars'
    browser.visit(jpl_mars_url)
    response = browser.html
    soup = BeautifulSoup(response, 'html.parser')

    jpl_results = soup.find('img', class_='thumb')['src']
    featured_image_url= 'https://www.jpl.nasa.gov' + jpl_results
    featured_image_url

    #Mars Weather
    mars_weather = 'https://twitter.com/marswxreport?lang=en'
    browser.visit(mars_weather)
    response = browser.html
    soup = BeautifulSoup(response, 'html.parser')

    tweet_results = soup.find('div', class_='css-901oao css-16my406 r-1qd0xha r-ad9z0x r-bcqeeo r-qvutc0')
    tweet_results

    #Mars Facts

    mars_facts_url = 'https://space-facts.com/mars/'
    tables = pd.read_html(mars_facts_url)
    df_tables = pd.DataFrame(tables)
    html_table = df_tables.to_html()
    html_table

    #Mars Hemispheres
    mars_hemispheres_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(mars_hemispheres_url)
    response = browser.html
    soup = BeautifulSoup(response, 'html.parser')

    hemisphere_image_urls = []

    mars_hemi_results = soup.find("div", class_ = "result-list" )
    results = mars_hemi_results.find_all("div", class_="item")

    for result in results:
        title = mars_hemi_results.find("h3").text
        links = mars_hemi_results.find("a")["href"]
        image_url = "https://astrogeology.usgs.gov/" + links
        html = browser.html
        soup=BeautifulSoup(html, "html.parser")
        hemisphere_image_urls.append({"title": title, "img_url": image_url})

    hemisphere_image_urls


import cv2
import numpy as np
import tensorflow as tf, sys
import time
import pickle
from sklearn.svm import SVC, LinearSVC
import tensorflow.python.platform
from tensorflow.python.platform import gfile
from sklearn.svm import SVC, LinearSVC
import matplotlib.pyplot as plt
from sklearn.externals import joblib

#fonction create graph + extraction

def create_graph():
    with gfile.FastGFile('retrained_graph_smartphones.pb', 'rb') as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
        _ = tf.import_graph_def(graph_def, name='')

create_graph()

def extract_features(list_images,label):
    nb_features = 2048
    features = np.empty((len(list_images),nb_features))
    labels = []

    with tf.Session() as sess:

        next_to_last_tensor = sess.graph.get_tensor_by_name('pool_3:0')

        for ind, image in enumerate(list_images):
            if (ind%100 == 0):
                print('Processing %s...' % (image))

            if not gfile.Exists(image):
                tf.logging.fatal('File does not exist %s', image)

            image_data = gfile.FastGFile(image, 'rb').read()
            predictions = sess.run(next_to_last_tensor,
            {'DecodeJpeg/contents:0': image_data})
            features[ind,:] = np.squeeze(predictions)
            labels.append(label)

        return features, labels

#load training model
clf = joblib.load('svm.pkl')

#open camera
font = cv2.FONT_HERSHEY_SIMPLEX
flux= cv2.VideoCapture(0)
counter = 0

#Prise de photo
_ ,frame = flux.read()

list_prediction = []
image = []
utc = time.time()

while (True):
    _ ,img = flux.read()
    cv2.rectangle(frame,(170,50),(510,450),(0,205,0),2)
    cv2.imshow('Image', frame)

    key = cv2.waitKey(1)
    _ ,frame = flux.read()

    now = time.time()
    #if key == ord('s'):

    for i in range(5):
        time.sleep(0.5)
        while now - utc >= 0.5 and counter <5:
            _ ,img_predict = flux.read()
            img_predict = img_predict[50:450, 170:510]
            path = 'test' + "_" + str(counter) + ".jpg"
            cv2.imwrite(path, img_predict)
            counter += 1
            utc = now
            image.append([path])
            print(image)

    '''if key == ord('s'):
        image = []
        for i in range(5):

            _ ,img_predict = flux.read()
            img_predict = img_predict[50:450, 170:510]
            now = time.time()
            if now-utc >= 2:
                print(now-utc)
                path = 'test' + "_" + str(counter) + ".jpg"
                cv2.imwrite(path, img_predict)
                counter += 1
                utc = now
                image.append([path])
            print(image)'''

   # counter = 0

    '''if key == ord("q") or counter == 5:
        break
        #cv2.putText(img_predict,str(y_pred),(0,25), font, 1 , (0,0,0), 2, cv2.LINE_AA)

        #cv2.imshow('Prediction', img_predict)'''

    if key == 0x0d:
        for img in image:

            features,labels = extract_features(img,'test')

            X = features
            y_pred = clf.predict(X)

            list_prediction.append(y_pred[0])

        print(list_prediction)

        ring1=0
        ring2=0
        ring3=0
        ring4=0
        ring5=0
        ring6=0
        ring7=0
        ring8=0

        for prediction in list_prediction:

            if prediction =='Huawei_P9':

                ring1 += 1

            if prediction == 'Iphone_7':

                ring2 += 1

            if prediction == 'Meizu_m3s':

                ring3 += 1

            if prediction == 'Samsung_S4':

                ring4 += 1

            if prediction == 'Samsung_S6_Edge':

                ring5 += 1

            if prediction == 'Iphone_5':

                ring6 += 1

            if prediction == 'Huawei_P8_Lite':

                ring7 += 1

            if prediction == 'autre':

                ring8 += 1

        last_prediction =''

        def final_prediction(ring,produit):

             if ring == max(rings):

                 last_prediction = produit

                 print('Final prediction :' + last_prediction )

                 if sum(rings) !=  0:

                     confidence = ring/(sum(rings))*100

                     print(confidence)

                     if confidence > 50 :

                         cv2.putText(img_predict,last_prediction,(0,25), font, 1 , (0,0,0), 2, cv2.LINE_AA)

                         #cv2.putText(img_predict,str(confidence) + '%' ,(0,380), font, 1 , (0,0,0), 2, cv2.LINE_AA)

                         cv2.imshow('Prediction', img_predict)

                     else:

                         print('La reconnaissance ne peut aboutir, merci de recommencer !')

        rings= [ring1, ring2, ring3, ring4, ring5,ring6,ring7,ring8]

        print(rings)

        final_prediction(ring1,'Huawei_P9')
        final_prediction(ring2,'Iphone_7' )
        final_prediction(ring3,'Meizu_m3s')
        final_prediction(ring4,'Samsung_S4')
        final_prediction(ring5,'Samsung_S6_Edge')
        final_prediction(ring6,'Iphone_5')
        final_prediction(ring7,'Huawei_P8_Lite')
        final_prediction(ring8,'autre')

        list_prediction=[]

    if key == 0x1b:

        cv2.destroyAllWindows()

        flux.release()
        break


"""ontrendclothing URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from product.views import productviews

urlpatterns = [
    path('', productviews.login),
    path('showproduct', productviews.showproduct),
    path('addproduct', productviews.addproduct),
    path('addtype', productviews.addtype),
    path('addvendor', productviews.addvendor),
    path('editproduct/<int:id>', productviews.editproduct),
    path('updateproduct/<int:id>', productviews.updateproduct),
    path('deleteproduct/<int:id>', productviews.deleteproduct),
    path('search', productviews.search),
    path('login', productviews.login),
    path('entry', productviews.entry),
    path('login', productviews.login),
]


from datetime import datetime
import random
import os
import re

colors = ['31', '32', '33', '34', '35', '36']
gray = '\033[37m'

env = os.getenv('DEBUG')

def noop(*args):
  return

def Debug(name):
  if env is None:
    return noop

  escaped_env = re.escape(env)
  escaped_env = escaped_env.replace('\\*', '.*?')
  escaped_env = escaped_env.replace(',', '|')
  pattern = re.compile('^(' + escaped_env + ')$')

  if not pattern.match(name):
    return noop

  color = '\033[' + colors[random.randint(0, len(colors) - 1)] + 'm'

  def debug(format, *args):
    print(gray + now() + color + name + ' ' + format % args)

  return debug

def now():
  t = datetime.now()
  return str(t.hour) + ':' + str(t.minute) + ':' + str(t.second) + ' ' + str(
      int(t.microsecond / 1000)) + 'ms '


import torch
from model import Model
from custom_data_helper import DataHelper
import numpy as np
import tqdm
import sys, random
import argparse
import time, datetime
import os
from custom_pmi import cal_PMI
from sklearn.metrics import mean_squared_error,mean_absolute_error
from sklearn.metrics import precision_score,accuracy_score
from sklearn.metrics import classification_report,confusion_matrix
import traceback
import sys,os
print(os.path.abspath(os.path.join('..')))
sys.path.append(os.path.abspath(os.path.join('..')))
from UtilFunctions import createDirIfNotExist

NUM_ITER_EVAL = 100
EARLY_STOP_EPOCH = 25

def edges_mapping(vocab_len, content, ngram):
    count = 1
    mapping = np.zeros(shape=(vocab_len, vocab_len), dtype=np.int32)
    for doc in content:
        for i, src in enumerate(doc):
            for dst_id in range(max(0, i-ngram), min(len(doc), i+ngram+1)):
                dst = doc[dst_id]

                if mapping[src, dst] == 0:
                    mapping[src, dst] = count
                    count += 1

    for word in range(vocab_len):
        mapping[word, word] = count
        count += 1

    return count, mapping

def get_time_dif(start_time):
    """获取已使用时间"""
    end_time = time.time()
    time_dif = end_time - start_time
    return datetime.timedelta(seconds=int(round(time_dif)))

def dev(model, dataset):
    data_helper = DataHelper(dataset, mode='dev')

    total_pred = 0
    correct = 0
    iter = 0
    for content, label, _ in data_helper.batch_iter(batch_size=64, num_epoch=1):
        iter += 1
        model.eval()

        logits = model(content)
        pred = torch.argmax(logits, dim=1)

        correct_pred = torch.sum(pred == label)

        correct += correct_pred
        total_pred += len(content)

    # total_pred = float(total_pred)
    # correct = correct.float()

    isRunOK=False
    try:
        if not isinstance(total_pred,float):
            total_pred = float(total_pred)
        if not isinstance(correct,float):
            correct = correct.float()
        isRunOK=True
    except:
        traceback.print_exc()
        isRunOK=False

    if not isRunOK:
        if not isinstance(total_pred,int):
            total_pred = int(total_pred)
        if not isinstance(correct,int):
            correct = correct.int()

    # print(torch.div(correct, total_pred))
    if total_pred ==0:
        return 0
    return torch.div(correct, total_pred)

    # print(torch.div(correct, total_pred))
    return torch.div(correct, total_pred)

def test(model_name, dataset,dictLabel):
    model = torch.load(os.path.join('.', model_name + '.pkl'))

    data_helper = DataHelper(dataset, mode='test')

    total_pred = 0
    correct = 0
    iter = 0
    list_pred=[]
    list_label = []
    for content, label, _ in data_helper.batch_iter(batch_size=64, num_epoch=1):
        iter += 1
        model.eval()

        logits = model(content)
        pred = torch.argmax(logits, dim=1)
        #print('label and pred {}\n{}'.format(label,pred))
        for i in range(0,len(label)):
            list_pred.append(int(dictLabel[int(pred[i])]))
            list_label.append(int(dictLabel[int(label[i])]))

        correct_pred = torch.sum(pred == label)

        correct += correct_pred
        total_pred += len(content)

    total_pred = float(total_pred)
    correct = correct.float()
    # print(torch.div(correct, total_pred))
    return torch.div(correct, total_pred).to('cpu'),list_label,list_pred

def train(ngram, name, bar, drop_out, dataset, is_cuda=False, edges=True,window=20):
    start_time = time.time()
    tupLst=[]
    print('load data helper.')
    data_helper = DataHelper(dataset, mode='train')
    f1=open(data_helper.current_set,'r')
    arrContent=f1.read().strip().split('\n')
    tupLst.append(len(arrContent))
    f1.close()
    print('len aaa {}'.format(len(arrContent)))

    if os.path.exists(os.path.join('.', name+'.pkl')) and name != 'temp_model':
        print('load model from file.')
        model = torch.load(os.path.join('.', name+'.pkl'))
    else:
        print('new model.')
        if name == 'temp_model':
            name = 'temp_model_%s' % dataset
        # edges_num, edges_matrix = edges_mapping(len(data_helper.vocab), data_helper.content, ngram)
        edges_weights, edges_mappings, count = cal_PMI(dataset=dataset,window_size=window)
        tupLst.append(len(data_helper.vocab))
        tupLst.append(count)

        model = Model(class_num=len(data_helper.labels_str), hidden_size_node=200,
                      vocab=data_helper.vocab, n_gram=ngram, drop_out=drop_out, edges_matrix=edges_mappings, edges_num=count,
                      trainable_edges=edges, pmi=edges_weights, cuda=is_cuda)

    print(model)
    if is_cuda:
        print('cuda')
        model.cuda()
    loss_func = torch.nn.CrossEntropyLoss()

    optim = torch.optim.Adam(model.parameters(), weight_decay=1e-6)

    iter = 0
    if bar:
        pbar = tqdm.tqdm(total=NUM_ITER_EVAL)
    best_acc = 0.0
    last_best_epoch = 0
    start_time = time.time()
    total_loss = 0.0
    total_correct = 0
    total = 0
    for content, label, epoch in data_helper.batch_iter(batch_size=32, num_epoch=200):
        improved = ''
        model.train()

        logits = model(content)
        loss = loss_func(logits, label)

        pred = torch.argmax(logits, dim=1)

        correct = torch.sum(pred == label)

        total_correct += correct
        total += len(label)

        total_loss += loss.item()

        optim.zero_grad()
        loss.backward()
        optim.step()

        iter += 1
        if bar:
            pbar.update()
        if iter % NUM_ITER_EVAL == 0:
            if bar:
                pbar.close()

            val_acc = dev(model, dataset=dataset)
            if val_acc > best_acc:
                best_acc = val_acc
                last_best_epoch = epoch
                improved = '*'

                torch.save(model, name + '.pkl', pickle_protocol=4)
            elif best_acc == 0:
                torch.save(model, name + '.pkl', pickle_protocol=4)

            if epoch - last_best_epoch >= EARLY_STOP_EPOCH:
                end_time = time.time()
                running_time = end_time - start_time
                tupLst.append(running_time)
                return name,tupLst
            msg = 'Epoch: {0:>6} Iter: {1:>6}, Train Loss: {5:>7.2}, Train Acc: {6:>7.2%}' \
                  + 'Val Acc: {2:>7.2%}, Time: {3}{4}' \
                  # + ' Time: {5} {6}'

            print(msg.format(epoch, iter, val_acc, get_time_dif(start_time), improved, total_loss/ NUM_ITER_EVAL,
                             float(total_correct) / float(total)))

            total_loss = 0.0
            total_correct = 0
            total = 0
            if bar:
                pbar = tqdm.tqdm(total=NUM_ITER_EVAL)
    end_time = time.time()
    running_time=end_time-start_time
    tupLst.append(running_time)
    # tup=tuple(tupLst)
    return name,tupLst

def word_eval():
    print('load model from file.')
    data_helper = DataHelper('r8')
    edges_num, edges_matrix = edges_mapping(len(data_helper.vocab), data_helper.content, 1)
    model = torch.load(os.path.join('word_eval_1.pkl'))

    edges_weights = model.seq_edge_w.weight.to('cpu').detach().numpy()

    core_word = 'billion'
    core_index = data_helper.vocab.index(core_word)

    results = {}
    for i in range(len(data_helper.vocab)):
        word = data_helper.vocab[i]
        n_word = edges_matrix[i, core_index]
        # n_word = edges_matrix[i, i]
        if n_word != 0:
            results[word] = edges_weights[n_word][0]

    sort_results = sorted(results.items(), key=lambda d: d[1])

    print(sort_results)

if __name__ == '__main__':

    fopDataset = '../../../../dataPapers/dataTextLevelPaper/'
    fopCsv='../../dataset/'
    # fpResultSEE = fopDataset + "/resultSEE.txt"
    fonResult='resultSEE_sen_verbNoun/'
    createDirIfNotExist(fopDataset+fonResult)
    fpResultSEEShort = fopDataset+fonResult + "short.txt"
    fpResultSEEDetails = fopDataset+fonResult + "details.txt"
    fpResultSEEAna = fopDataset+fonResult + "analyze.txt"

    list_dir = os.listdir(fopCsv)  # Convert to lower case
    list_dir = sorted(list_dir)

    o2 = open(fpResultSEEShort, 'w')
    o2.write('')
    o2.close()
    o2 = open(fpResultSEEDetails, 'w')
    o2.write('')
    o2.close()
    o2 = open(fpResultSEEAna, 'w')
    o2.write('')
    o2.close()
    lstResultOverProjects = []
    lstStrResultOverProjects = []
    index=0
    arrWindow = [2, 5, 10, 20, 50, 100]
    for win in arrWindow:
        fff = open(fpResultSEEDetails, 'a')
        fff.write('\n\n\nwindow_size = {}\n\n'.format(win))
        fff.close()

        fff = open(fpResultSEEShort, 'a')
        fff.write('\n\n\nwindow_size = {}\n\n'.format(win))
        fff.close()

        fff = open(fpResultSEEAna, 'a')
        fff.write('\n\n\nwindow_size = {}\n\n'.format(win))
        fff.close()

        for filename in list_dir:
            if not filename.endswith('.csv'):
                continue
            index=index+1
            '''
            if(index<=15):
                continue
            '''
            fnSystem = filename.replace('.csv', '')
            fnSystem='verbNoun_'+fnSystem
            # if not (fnSystem == 'jirasoftware' or  fnSystem == 'clover'):
            #     continue
            fpLabel = fopDataset + fnSystem + "/test_label.txt"
            fpPred = fopDataset + fnSystem + "/test_pred.txt"

            fpTextLabel = fopDataset + fnSystem + '/label.txt'
            try:
                parser = argparse.ArgumentParser()
                parser.add_argument('--ngram', required=False, type=int, default=1, help='ngram number')
                parser.add_argument('--name', required=False, type=str, default='temp_model', help='project name')
                parser.add_argument('--bar', required=False, type=int, default=0, help='show bar')
                parser.add_argument('--dropout', required=False, type=float, default=0.5, help='dropout rate')
                parser.add_argument('--dataset', required=False, type=str,default=fnSystem, help='dataset')
                parser.add_argument('--edges', required=False, type=int, default=1, help='trainable edges')
                parser.add_argument('--rand', required=False, type=int, default=7, help='rand_seed')

                args = parser.parse_args()

                print('ngram: %d' % args.ngram)
                print('project_name: %s' % args.name)
                print('dataset: %s' % args.dataset)
                print('trainable_edges: %s' % args.edges)
                # #
                SEED = args.rand
                torch.manual_seed(SEED)
                torch.cuda.manual_seed(SEED)
                np.random.seed(SEED)
                random.seed(SEED)

                if args.bar == 1:
                    bar = True
                else:
                    bar = False

                if args.edges == 1:
                    edges = True
                    print('trainable edges')
                else:
                    edges = False

                dictLabel = {}
                fff = open(fpTextLabel, 'r')
                lUn = fff.read().split('\n')
                fff.close()

                for i in range(0, len(lUn)):
                    dictLabel[i] = lUn[i].strip()

                model, tupLst = train(args.ngram, args.name, bar, args.dropout, dataset=args.dataset, is_cuda=True, edges=edges,window=win)
                #model='temp_model_'+fnSystem
                result,lLabel,lPred=test(model, args.dataset,dictLabel)
                # print('test acc: ', result.numpy())
                # maeAccuracy = mean_absolute_error(lLabel, lPred)
                y_test = lLabel
                predicted = lPred
                classAccuracy = accuracy_score(y_test, predicted)

                print('{}\t{:.2f}'.format(fnSystem, classAccuracy))

                o2 = open(fpResultSEEDetails, 'a')
                o2.write(fnSystem + '\n')
                o2.write('Result for GCN TextLevel \n')
                # o2.write('MAE {}\nMQE {}\n\n\n'.format(maeAccuracy,mqeAccuracy))

                # o2.write(str(sum(cross_val) / float(len(cross_val))) + '\n')
                o2.write(str(confusion_matrix(y_test, predicted)) + '\n')
                o2.write(str(classification_report(y_test, predicted)) + '\n\n\n')
                o2.close()
                lstResultOverProjects.append(classAccuracy)
                lstStrResultOverProjects.append('{}\t{}'.format(fnSystem, classAccuracy))

                fff=open(fpLabel,'w')
                fff.write('\n'.join(map(str,lLabel)))
                fff.close()
                fff=open(fpPred,'w')
                fff.write('\n'.join(map(str,lPred)))
                fff.close()
                fff = open(fpResultSEEShort, 'a')
                fff.write('{}\t{}\n'.format(fnSystem, classAccuracy))
                fff.close()
                fff = open(fpResultSEEAna, 'a')
                fff.write('{}\t{}\n'.format(fnSystem, '\t'.join(map(str,tupLst))))
                fff.close()
            except Exception as e:
                fff = open(fpResultSEEShort, 'a')
                fff.write('{}\t{}\n'.format(fnSystem, 'Error'))
                fff.close()
                fff = open(fpResultSEEDetails, 'a')
                fff.write('{}\t{}\n'.format(fnSystem, 'Error {}'.format(str(e))))
                fff.close()
                fff = open(fpResultSEEAna, 'a')
                fff.write('{}\t{}\n'.format(fnSystem, 'Error'))
                fff.close()
                traceback.print_exc()

    averageAccuracy = np.average(lstResultOverProjects)
    strAvg='{}\t{}'.format('Avg', averageAccuracy)
    f1 = open(fpResultSEEShort, 'a')
    f1.write(strAvg)
    f1.close()


import torch
from metrics import NDCG_binary_at_k_batch, Recall_at_k_batch

def get_args(args):
    class dotdict(dict):
        """dot.notation access to dictionary attributes"""
        __getattr__ = dict.get
        __setattr__ = dict.__setitem__
        __delattr__ = dict.__delitem__

    args = dotdict(vars(args))

    args.device = 'cpu' if args.gpu == -1 else 'cuda:{}'.format(args.gpu)
    args.torchType = torch.float32

    ## Data and training parameters
    args.train_batch_size = 500
    args.val_batch_size = 2000
    args.n_epoches = 200

    args.print_info_ = 1

    ## Transition parameters (only for our vae)
    args.gamma = 0.1  # Stepsize
    args.alpha = 0.5  # For partial momentum refresh
    args.use_barker = True
    args.use_partialref = True

    args.annealing = True if args.annealing == 'True' else False
    args.learnable_reverse = True if args.learnable_reverse == 'True' else False

    if args.annealing:
        args.total_anneal_steps = 200000
        args.anneal_cap = 0.2
    else:
        args.total_anneal_steps = 0
        args.anneal_cap = 1.

    ## Metric
#     args.metric = NDCG_binary_at_k_batch
    args.metric = Recall_at_k_batch

    if args.model == 'MultiDAE':
        args.l2_coeff = 0.01 / args.train_batch_size
    else:
        args.l2_coeff = 0.

    return args


#!/usr/bin/env python

import matplotlib.pylab as plt
import numpy as np
# from matplotlib import gridspec
# from matplotlib import colors
# from matplotlib.colors import LinearSegmentedColormap

def reshape_1D_to_2D(N_LOS,N_R, input_array):
    output_matrix = np.zeros([N_LOS, N_R])
    idx_work = 0
    if (N_LOS*N_R == input_array.shape[0]):
        for LL in xrange(N_LOS):
            for RR in xrange(N_R):
                output_matrix[LL,RR] = input_array[idx_work]
                idx_work += 1
    else:
        print("N_LOSxN_R does not match input array shape")
    return output_matrix

# path = '/tokp/work/osam/ECRad/ECRad_25781_ECO_ed1.mat'

# /tokp/work/osam/ECRad

# ECRad_25781_ECO_ed1.mat
import scipy.io
mat = scipy.io.loadmat('/tokp/work/osam/ECRad/ECRad_25781_ECO_ed2.mat')
# mat.keys()

mat['time']
mat['Te'].shape
mat['Trad'].shape
mat['R_warm'].shape
N_LOS = 16
N_R = 8
tt = 0
Trad = reshape_1D_to_2D(N_LOS, N_R, mat['Trad'][tt])
R = reshape_1D_to_2D(N_LOS, N_R, mat['R_warm'][tt])
z = reshape_1D_to_2D(N_LOS, N_R, mat['z_warm'][tt])

plt.contourf(R, z, Trad, 100, cmap ="jet")
plt.show()


# -*- encoding: utf-8 -*-

"""
bug备注：
    致命bug:
        1.第160行：print(id)大概率多出几位
        2.视频【BV1Cp4y1X7AF】调试时出现-3楼的情况
"""

# Here put the improt lib
import json
import re
import time
import hashlib
import requests
from difflib import SequenceMatcher

def logo():
    print('                     //')
    print('         \\\         //')
    print('          \\\       //')
    print('    ##3333333333333333333333##')
    print('    ## 22222222222222222222 ##    ________   ___   ___        ___   ________   ___   ___        ___')
    print('    ## hh                hh ##   |\   __  \ |\  \ |\  \      |\  \ |\   __  \ |\  \ |\  \      |\  \\')
    print('    ## hh    //    \\\    hh ##   \ \  \|\ /_\ \  \\\ \  \     \ \  \\\ \  \|\ /_\ \  \\\ \  \     \ \  \\')
    print('    ## hh   //      \\\   hh ##    \ \   __  \\\ \  \\\ \  \     \ \  \\\ \   __  \\\ \  \\\ \  \     \ \  \\')
    print('    ## hh                hh ##     \ \  \|\  \\\ \  \\\ \  \____ \ \  \\\ \  \|\  \\\ \  \\\ \  \____ \ \  \\')
    print('    ## hh      wwww      hh ##      \ \_______\\\ \__\\\ \_______\\\ \__\\\ \_______\\\ \__\\\ \_______\\\ \__\\')
    print('    ## hh                hh ##       \|_______| \|__| \|_______| \|__| \|_______| \|__| \|_______| \|__|')
    print('    ## 22222222222222222222 ##        【exit】退出')
    print('    ##3333333333333333333333##        【达拉崩吧】说明')
    print('\n')

def explain():
    print('Instruction Book')
    print('奖号依据：程序运行时刻的微博热搜词top1=>MD5加密=>奖号')
    print('奖券号：楼层数+昵称+id=>MD5加密=>奖券号')
    print('抽奖时取奖券号最接近奖号的一位')

def getAv(code):  # 获取视频av号
    url = f'https://www.bilibili.com/video/{video_code}'
    response = requests.get(url)
    response.encoding = 'utf-8'
    html = response.text
    av_code = re.findall(
        r'<meta data-vue-meta="true" property="og:url" content="https://www.bilibili.com/video/av(.*?)/">', html)[0]
    return av_code

def getData(av_code, page_num=1, sort=0): #评论区username&id
    url = f'https://api.bilibili.com/x/v2/reply?callback=jQuery1720266519964639113_1586919135667&pn={page_num}&type=1&oid={av_code}&sort={sort}&_=1586919138435'
    # url = f'https://api.bilibili.com/x/v2/reply?callback=jQuery1720266519964639113_1586919135667&pn=1&type=1&oid=540177402&sort=0&_=1586919138435'
    # pn 页码
    # oid 视频id
    # sort == 0 按时间排序
    # sort == 2 按热度排序
    response = requests.get(url)
    response.encoding = 'utf-8'
    html = json.loads(response.text)
    page_data = html['data']['page']
    floor = page_data['count']
    pageNum = page_data['count'] // page_data['size'] + 1
    # print(pprint.pprint(html['data']['replies']))

    cmMap = {}  # 第xxl楼：{'usr_name':xxxxx,'id':xxxxx}
    for num in range(1, pageNum + 1):
        dir_url = f'https://api.bilibili.com/x/v2/reply?callback=jQuery1720266519964639113_1586919135667&pn={num}&type=1&oid={av_code}&sort={sort}&_=1586919138435'
        # dir_url = f'https://api.bilibili.com/x/v2/reply?callback=jQuery1720266519964639113_1586919135667&pn={num}&type=1&oid=540177402&sort=0&_=1586919138435'
        response_d = requests.get(dir_url)
        response_d.encoding = 'utf-8'
        html_d = json.loads(response_d.text)
        for i in html_d['data']['replies']:
            user_data = {'user_name': i['member']
                         ['uname'], 'id': i['member']['mid']}
            # print(i['member']['mid'] + ':' + i['member']['uname'])
            # print(user_data)
            cmMap[str(floor)] = user_data
            floor -= 1
        # print(num)
        # print(dir_url)
        time.sleep(0.5)
    #去重复
    re_list=[]
    for key,value in cmMap.items():
        for k,v in cmMap.items():
            if k in re_list:
                continue
            if key != k and value == v:
                re_list.append(k)
        # print(re_list)
    del_list = re_list[:-1]
    # print(del_list)
    for i in del_list:
        del cmMap[i]
    # print(cmMap)
    return cmMap

def  weiboHot(): #微博热词
    url = 'https://s.weibo.com/top/summary?cate=realtimehot'
    reponse = requests.get(url)
    reponse.encoding = 'utf-8'
    html = reponse.text
    hot_tbody = re.findall(r'
(.*?)
',html,re.S)[0]
    hot_top = re.findall(r'
(.*?)
',hot_tbody,re.S)[1]
    hot_word = re.findall(r'target="_blank">(.*?)',hot_top,re.S)[0]
    return hot_word

def hashMd5(args): #哈希算法
    hash_md5 = hashlib.md5()
    hashMd5 = hash_md5.update(args.encode('utf-8'))#hash.hexdigest（）
    hash_code = hash_md5.hexdigest()
    return hash_code

def similarity(a, b):
    return SequenceMatcher(None, a, b).ratio()

if __name__ == "__main__":
    logo()
    while True:
        video_code = input('BV号:')
        if video_code=='达拉崩吧':
            explain()
        elif video_code=='exit':
            break
        elif len(video_code)==12:
            max_num = 0
            hot_word = weiboHot()
            hash_code_hot = hashMd5(hot_word)

            print(time.asctime( time.localtime(time.time())))

            print('微博热搜top1:'+ hot_word)

            print('哈希值:' + hash_code_hot)

            av_code = getAv(video_code)

            cm_dict = getData(av_code)

            for key,value in cm_dict.items():
                # print(key,value)
                userName = value['user_name']
                uid = value['id']
                cm_word = key + userName + uid
                # print(cm_word)
                hash_code_cm = hashMd5(cm_word)
                # print(hash_code_cm)
                # print(f'\r{hash_code_cm}' , end = '')

                if similarity(hash_code_hot,hash_code_cm) > max_num:
                    max_num = similarity(hash_code_hot,hash_code_cm)
                    lucky_code = hash_code_cm
                    lucky_name = userName
                    lucky_id = uid
                    # print('\n*************************')
                    # print(lucky_code + '--'+ str(max_num) +'--')
                    # print('*************************')
                    # print(f'\r{lucky_code}|相似度:{max_num}', end = '')
                    # print(f'\r***LUCKY MAN***')
                    print('\r第%s楼  NAME:%s  ID:%s' %(key,lucky_name,lucky_id),end='')
                    # print(lucky_name,lucky_id)
                # print(f'\r{cm_word}' , end = '')
                time.sleep(0.05)
            # print('\r')
            print('\n',lucky_name,lucky_id)
        else:
            print('BV号有误')


# -*- coding: utf-8 -*-
import scrapy
import re
import hashlib
from datetime import datetime
from amazones.items import AmazonesItem
from mysqlconn import mysqldbclass

headers = {
            'authority':'www.amazon.es',
    'method':'GET',
    'path':'/s?k=xyz',
    'scheme':'https',
    'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
    'accept-encoding':'gzip, deflate, br',
    'accept-language':'en-GB,en;q=0.9,en-US;q=0.8,ar;q=0.7,es;q=0.6,ru;q=0.5,ja;q=0.4',
    'cache-control':'no-cache',
    'pragma':'no-cache',
    'sec-fetch-mode':'navigate',
    'sec-fetch-site':'none',
    'sec-fetch-user':'?1',
    'upgrade-insecure-requests':'1',
    'user-agent':'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
        }

class amazonespider(scrapy.Spider):
    name = "amazonesoffers"

    def start_requests(self):
        self.dbclass = mysqldbclass()
        asinscodes, self.sellersDict = self.dbclass.mysqldbextract('priceRecord')
        for asin in asinscodes:
            offerurl = 'https://www.amazon.es/gp/offer-listing/' + asin
            yield scrapy.Request(offerurl, callback=self.parse, headers = headers, meta = {'asin': asin})

    def parse(self, response):
        def url_hash(url):
            if not url.startswith('http'):
                return False
            url = url.encode('utf-8')
            return hashlib.md5(url).hexdigest()
        item = AmazonesItem()
        asin = response.meta['asin']
        yield scrapy.Request('https://www.amazon.es/dp/' + asin , callback=self.parse_images, headers = headers, meta = {'asin': asin})
        item['crw_slr_prd_id'] = asin
        item['crw_product_name'] = response.xpath('//title[contains(.,"Amazon.es: Opciones de compra")]/text()').extract_first().replace('Amazon.es: Opciones de compra: ','')
        item['crw_product_brand'] = response.xpath('//div [@id="olpProductByline"]/text()').extract_first()
        item['crw_product_brand'] = re.sub(r'\sde ','',item['crw_product_brand'])
        item['crw_image_url'] = response.xpath('//*[@id="olpProductImage"]//img/@src').extract_first()
        # extracting all offers
        offerslist = response.xpath('//*[@class="a-row a-spacing-mini olpOffer"]')
        for offer in offerslist:
            item['crw_pvr_price'] = ''
            item['crw_sale_price'] = offer.xpath('.//*[@class="a-offscreen"]/text()').extract_first().split()[-1].replace(',','.')
            try:
                item['shippingprice'] = offer.xpath('.//*[@class="olpShippingPrice"]/text()').extract_first().split()[-1].replace(',','.')
            except:
                item['shippingprice'] = offer.xpath('.//*[@class="olpShippingPrice"]/text()').extract_first()
            item['Condition'] = offer.xpath('.//*[@class="a-size-medium olpCondition a-text-bold"]/text()').extract_first()
            item['sellername'] = offer.xpath('.//*[@class="a-spacing-none olpSellerName"]//a/text() | .//*[@class="a-spacing-none olpSellerName"]/img/@alt').extract_first()
            if item['sellername'] == 'Amazon.es':
                item['sellerurl'] = 'Amazon.es'
            else:
                item['sellerurl'] = 'https://www.amazon.es' + offer.xpath('.//*[@class="a-spacing-none olpSellerName"]//a/@href').extract_first()
            if item['sellername'] == 'Amazon.es':
                item['crw_product_url'] = 'https://www.amazon.es/dp/' + asin + '?m='  + 'A1AT7YVPFBWXBL'
            else:
                item['crw_product_url'] = 'https://www.amazon.es/dp/' + asin + '?m='  + item['sellerurl'].split('seller=')[-1]
            item['crw_product_url_hash'] = url_hash(item['crw_product_url'])
            if item['sellername'] == 'Amazon.es':
                item['slr_mkt_id'] = 'A1RKKUPIHCS9HS'
            else:
                item['slr_mkt_id'] = item['sellerurl'].split('seller=')[-1]

            item['creation_date'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            item['offerpage'] = response.request.url
            item['recordtype'] = 'priceRecord'
            item['crw_mkt_id'] = 3
            item['process_name'] = self.name.upper()
            if item['slr_mkt_id'] in self.sellersDict:
                #this seller is in our DB already
                print ("this seller is in our DB already", item['slr_mkt_id'])
                item['crw_seller_id'] = self.sellersDict[item['slr_mkt_id']]
            else:
                print ("this seller is NOT in our DB!", item['slr_mkt_id'])
                item['crw_seller_id'] = self.dbclass.seller_api(item['slr_mkt_id'],item['sellername'],item['sellerurl'])
                self.sellersDict.update({item['slr_mkt_id']:item['crw_seller_id']})

            #cleaning extracted text from extra whitespaces
            for key, value in item.items():
                try:
                    item[key] = re.sub(r'\n+', ' ', value)
                    item[key] = re.sub(r'\t+', ' ', value)
                    item[key] = re.sub(r'\s+', ' ', value)
                    item[key] = item[key].strip()
                except:
                    pass

            yield item
        #crawling next pages
        nextpage = response.xpath('//*[@class="a-last"]/a/@href').extract_first()
        if nextpage:
            yield scrapy.Request( 'https://www.amazon.es' + nextpage,callback=self.parse, headers = headers, meta = {'asin': asin})

    def parse_images(self, response):
        #add ean, sku to amazon products
        # dimension
        # weight

        asin = response.request.url.split('/')[-1]
        skudraft = response.xpath('//b[contains(text(),"Referencia del fabricante")]/following-sibling::text()').extract_first()
        eandraft = response.xpath('//b[contains(text(),"de modelo del producto")]/following-sibling::text()').extract_first()
        print ('skudraft')
        print (skudraft)
        print ('eandraft')
        print (eandraft)
        print ('*'*50)
        print ('&'*50)
        if eandraft:
            try:
                eanint = int(eandraft.strip())
                if len(eandraft.strip()) == 13:
                    ean =eandraft.strip()
                else:
                    if skudraft is None:
                        sku = eandraft.strip()
                        ean = None
                    else:
                        ean = eandraft.strip()
            except:

                if skudraft is None:
                    sku = eandraft.strip()
                    ean = None
                else:
                    ean = eandraft.strip()
        else:
            ean = None

        if skudraft:
            try:
                skuint = int(skudraft.strip())
                if len(skudraft.strip()) == 13:
                    ean = skudraft.strip()
                    if sku is not None:
                        sku = None
                else:
                    sku = skudraft.strip()
            except:
                sku = skudraft.strip()
        else:
            if 'sku' not in locals():
                sku = None
        print (eandraft,'eandraft')
        print (ean,'ean')
        print (skudraft, 'skudraft')
        print (sku, 'sku')
        print ('*'*50)
        print ('&'*50)
        dimensionsdraft = response.xpath('//b[contains(text(),"Dimensiones del producto")]/following-sibling::text()').extract_first()
        try:
            dimensions = dimensionsdraft.split(' ; ')[0]
        except:
            dimensions = None
        try:
            weight = dimensionsdraft.split(' ; ')[-1]
        except:
            weight = None
        if weight:
            if 'x' in weight:
                dimensions = weight
                weight = None
        script = response.xpath('//*[@id="imageBlock_feature_div"]/script/text()').extract_first()
        images = re.findall('hiRes":"([^"]*)',script)
        if len(images) > 0:
            images = ';'.join(images)
            self.dbclass.save_images(asin, images,sku,ean,dimensions,weight)
        else:
            images = re.findall('large":"([^"]*)',script)
            if len(images) > 0:
                images = ';'.join(images)
                self.dbclass.save_images(asin, images,sku,ean,dimensions,weight)

Scrapy project with mysql and mongodb pipeline/amazones/spiders/offers.py


import os
import sys
import zipfile

try:
    import requests
    import clint
except ImportError:
    sys.exit("Convergence tests require `requests` and `clint`, please install using pip or conda")

def download_zip_with_progress_bar(url):
    r = requests.get(url, stream=True)
    path = url.rsplit('/', 1)[-1]
    with open(path, 'wb') as f:
        total_length = int(r.headers.get('content-length'))
        for chunk in clint.textui.progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1):
            if chunk:
                f.write(chunk)
                f.flush()

def unzip(meshzip, folder_name, rename_folder):
    '''Unzip a zip file

    Arguments:
    ----------
    meshzip      : file we want to unzip
    folder_name  : str, name of the folder after extracting.
                  We want this to modify it when download, to follow pygbe
                  problem folder design for running.
    rename_folder: str, name of the folder were we want to store the download.
                   We rename the folder_name, to match config files.
    '''
    with zipfile.ZipFile(meshzip, 'r') as myzip:
        myzip.extractall(path='./tmp')

    print('Unzipping meshes to folder \'geometry/\'...')
    os.rename(os.path.join('tmp', folder_name), rename_folder)
    os.rmdir('tmp')
    print('Removing zip file...')
    os.remove(meshzip)

def check_mesh(mesh_file, folder_name, rename_folder, size):
    ''' Check if there's a geometry folder present in the directory.

    Arguments:
    ----------
    mesh_file    : str, mesh_file url
    folder_name  : str, name of the folder after extracting.
                   We want to pass this to be used in unzip function, where
                   we rename it.
    rename_folder: str, name of the folder were we want to store the download.
                   We rename the folder_name, to match config files.
    size         : str, estimate size of file
    '''
    if not os.path.isdir(rename_folder):
        dl_check = input('The meshes for convergence tests don\'t appear to '
                         'be loaded. Would you like to download them from '
                             'Zenodo?(' + size + ')(y/n): ')
        if dl_check == 'y':
            download_zip_with_progress_bar(mesh_file)
            unzip(mesh_file.split('/')[-1], folder_name, rename_folder)

            print('Done!')


import pygame
import time
import random as rd

pygame.init()

food_point=2
width = 600
height = 400
h1=470
dis = pygame.display.set_mode((width,h1))
pygame.display.set_caption("Snake Game")

def gen_food():
    global foodx,foody,food_point
    dis.fill(black)
    for a in range(width):
        pygame.draw.rect(dis,red,[a,height,snakeblock,snakeblock])
    font = pygame.font.SysFont("comicsansms",35)
    val = font.render("Score = "+str(snake_length-1), True, red)
    dis.blit(val,[snakeblock,height+snakeblock])
    if snake_length%10==1:
        food_point = 5
    else:
        food_point = 1
    foodx = round(rd.randrange(0,width-snakeblock)/snakeblock)*snakeblock
    foody = round(rd.randrange(0,height-snakeblock)/snakeblock)*snakeblock

def snake():
    global foodx,foody,snake_list,food_point
    for x1 in snake_list:
        pygame.draw.rect(dis,orange,[x1[0],x1[1],snakeblock,snakeblock])
    if food_point>1:
        font = pygame.font.SysFont("comicsansms",35)
        val = font.render("Bonus!!!", True, red)
        dis.blit(val,[width*3/7,height+snakeblock])
        pygame.draw.rect(dis,green,[foodx-snakeblock,foody-snakeblock,snakeblock*3,snakeblock*3])
    else:
        pygame.draw.rect(dis,green,[foodx,foody,snakeblock,snakeblock])

red = (255,0,0)
green = (0,255,0)
blue = (0,0,255)
black = (0,0,0)
orange = (240,50,0)
gameover=False
snakeblock = 10
snake_list = []
snake_length=1
x=width/2
x_change=0
y=height/2
y_change=0
dis.fill(black)
foodx=0
foody=0
clock = pygame.time.Clock()
speed= 10
gameend=False

while not gameover:
    while gameend:
        x_change=0
        y_change=0
        dis.fill(black)
        for a in range(width):
            pygame.draw.rect(dis,red,[a,height,snakeblock,snakeblock])
        score=snake_length-1
        font = pygame.font.SysFont("comicsansms",35)
        val = font.render("Score = "+str(score)+". Press 'r' to retry", True, red)
        dis.blit(val,[width/7,height+snakeblock])
        pygame.display.update()
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                gameend=False
                gameover = True
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_r:
                    gen_food()
                    x_change=-snakeblock
                    gameend=False
                    dis.fill(black)
                    x=width/2
                    y=height/2
                    snake_length=1
                    snake_list = []
                    gen_food()
                    for a in range(width):
                         pygame.draw.rect(dis,red,[a,height,snakeblock,snakeblock])

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            gameover = True
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_s:
                print("space pressed")
                dis.fill(black)
                gen_food()
                for a in range(width):
                    pygame.draw.rect(dis,red,[a,height,snakeblock,snakeblock])
                x_change=-snakeblock
            if event.key==pygame.K_DOWN and x_change!=0:
                print("down pressed")
                x_change = 0
                y_change = snakeblock
            if event.key == pygame.K_UP and x_change!=0:
                print("up pressed")
                x_change = 0
                y_change = -snakeblock
            if event.key == pygame.K_RIGHT and y_change!=0:
                print("right pressed")
                x_change = snakeblock
                y_change = 0
            if event.key == pygame.K_LEFT and y_change!=0:
                print("left pressed")
                x_change = -snakeblock
                y_change = 0

    head=[]
    x=x+x_change
    y=y+y_change
    x%=width
    y=y%height
    head.append(x)
    head.append(y)
    snake_list.append(head)
    if x>=width or x<0 or y>height or y<0:
        gameend=True
    if len(snake_list) > snake_length:
        #print("reduce",len(snake_list))
        pygame.draw.rect(dis,black,[snake_list[0][0],snake_list[0][1],snakeblock,snakeblock])
        del snake_list[0]
    if x_change==0 and y_change==0:
        font = pygame.font.SysFont("comicsansms",35)
        val = font.render("Press 's' to start", True, red)
        dis.blit(val,[width*2/7,height+snakeblock])
        pygame.display.update()
        continue;

    snake()
    for x1 in snake_list[0:-1]:
        if x1[0]==x and x1[1]==y:
            print("here")
            gameend=True

    if x==foodx and y==foody:
        snake_length+=food_point
        gen_food()

    clock.tick(speed)
    pygame.display.update()

pygame.quit()


import numpy as np
import os
import csv
import operator
from keras.layers.merge import dot
from keras.models import Model
from keras.layers import Input, Dense, Masking, Dropout, LSTM, Bidirectional, Activation, Concatenate, Merge
from keras.layers import TimeDistributed
from keras.layers import AveragePooling1D
from keras.layers import Flatten
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Bidirectional
from keras.models import load_model
from keras.optimizers import SGD, Adam, RMSprop
import matplotlib.pyplot as plt
from sklearn import svm, datasets
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from keras.utils import np_utils
import itertools
np.seterr(divide='ignore', invalid='ignore')

def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm):

    plt.figure(figsize=(4,7))

    #NOT NORMALIZED
    print('Confusion matrix, without normalization')
    print(cm)
    plt.subplot(2, 1, 1)
    plt.imshow(cm, interpolation='nearest', cmap=cmap.Blues)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

    #NORMALIZED
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    print("Normalized confusion matrix")
    print(cm)
    plt.subplot(2, 1, 2)
    plt.imshow(cm, interpolation='nearest', cmap=cmap.Blues)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

    return plt, cm

def computeConfMatrix(allPredictionClasses, expected, dirRes, nameFileResult, flagPlotGraph):
    labelLimit = 170
    allfile = 1098#(labelLimit*4)#1098

    expected = np.argmax(expected, axis=1)
    cmUA = confusion_matrix(expected, allPredictionClasses)
    plt, cmWA = plot_confusion_matrix(cmUA, ['joy','ang','sad','neu'], title=nameFileResult+'-CM')

    accurancyUA = (cmUA[0][0] + cmUA[1][1] + cmUA[2][2] + cmUA[3][3])/(allfile)
    print('Accurancy UA: ',accurancyUA)
    accurancyWA = (cmWA[0][0] + cmWA[1][1] + cmWA[2][2] + cmWA[3][3])/4
    print('Accurancy WA: ',accurancyWA)

    accurancy = max(accurancyUA, accurancyWA)

    OutputImgPath = os.path.join(dirRes, nameFileResult+'-Acc_'+str(accurancy)+'-CM.png')
    plt.savefig(OutputImgPath)
    if flagPlotGraph:
        plt.show()

def statistics(Y, yhat, correctCounter, predEmoCounter):
    index, value = max(enumerate(Y[0]), key=operator.itemgetter(1))
    Pindex, Pvalue = max(enumerate(yhat[0]), key=operator.itemgetter(1))
    '''print('index: ', index, 'value: ', value)
    print('index: ', Pindex, 'value: ', Pvalue)'''

    #UPDATE CORRECT COUNTER
    if index == Pindex:
        correctCounter[index] += 1

    #UPDATE PREDICTED EMO COUNTER
    predEmoCounter[Pindex] += 1

    return correctCounter, predEmoCounter

def saveCsv(currentFile, csvOutputFilePath):
    csvOutputFilePath = os.path.join(csvOutputFilePath + '.csv')
    try:
        os.remove(csvOutputFilePath)
    except OSError:
        pass

    with open(csvOutputFilePath, "w", newline='') as f:
        writer = csv.writer(f)
        writer.writerows(np.asarray(currentFile))
    f.close()

def saveTxt(currentFile, txtOutputFilePath):
    txtOutputFilePath = os.path.join(txtOutputFilePath + '.txt')
    try:
        os.remove(txtOutputFilePath)
    except OSError:
        pass

    with open(txtOutputFilePath, 'w') as file:
        for item in currentFile:
            file.write(str(item)+'\n')
    file.close()

def readFeatures(DirRoot, labelLimit):
    listA = [ item for item in os.listdir(DirRoot) if os.path.isfile(os.path.join(DirRoot, item)) ]
    allFileFeature = []
    allFileName = []

    i = 0
    #READ encoded audio Features
    for file in listA:
        allFileName.append(file)
        datareader = csv.reader(open(os.path.join(DirRoot, file), 'r'))
        data = []
        for row in datareader:
            data.append([float(val) for val in row])
        Y = np.array([np.array(xi) for xi in data])

        #Append all files feature in an unique array
        allFileFeature.append(Y)

        if i > labelLimit-1:
            break
        else:
            i += 1

    allFileFeature = np.asarray(allFileFeature)

    return allFileFeature, allFileName

def organizeFeaturesV2(dirAudio, dirText, dirLabel, labelLimit):

    joyAudioFeature, joyFileName = readFeatures(os.path.join(dirAudio, 'joy'), labelLimit)
    angAudioFeature, angFileName = readFeatures(os.path.join(dirAudio, 'ang'), labelLimit)
    sadAudioFeature, sadFileName = readFeatures(os.path.join(dirAudio, 'sad'), labelLimit)
    neuAudioFeature, neuFileName = readFeatures(os.path.join(dirAudio, 'neu'), labelLimit)
    joyTextFeature, joyFileName = readFeatures(os.path.join(dirText, 'joy'), labelLimit)
    angTextFeature, angFileName = readFeatures(os.path.join(dirText, 'ang'), labelLimit)
    sadTextFeature, sadFileName = readFeatures(os.path.join(dirText, 'sad'), labelLimit)
    neuTextFeature, neuFileName = readFeatures(os.path.join(dirText, 'neu'), labelLimit)
    joyLabels, joyFileName = readFeatures(os.path.join(dirLabel, 'joy'), labelLimit)
    angLabels, angFileName = readFeatures(os.path.join(dirLabel, 'ang'), labelLimit)
    sadLabels, sadFileName = readFeatures(os.path.join(dirLabel, 'sad'), labelLimit)
    neuLabels, neuFileName = readFeatures(os.path.join(dirLabel, 'neu'), labelLimit)
    '''print(allAudioFeature.shape)
    print(allTextFeature.shape)
    print(allLabels.shape)'''

    #BUILD SHUFFLED FEATURE FILES FOR TRAINING
    allAudioFeature = []
    allTextFeature = []
    allFileName = []
    allLabels = []
    i = 0
    while i < labelLimit:
        if i < len(joyAudioFeature):
            allAudioFeature.append(joyAudioFeature[i])
            allTextFeature.append(joyTextFeature[i])
            allFileName.append(joyFileName[i])
            allLabels.append(joyLabels[i])

        if i < len(angAudioFeature):
            allAudioFeature.append(angAudioFeature[i])
            allTextFeature.append(angTextFeature[i])
            allFileName.append(angFileName[i])
            allLabels.append(angLabels[i])

        if i < len(sadAudioFeature):
            allAudioFeature.append(sadAudioFeature[i])
            allTextFeature.append(sadTextFeature[i])
            allFileName.append(sadFileName[i])
            allLabels.append(sadLabels[i])

        if i < len(neuAudioFeature):
            allAudioFeature.append(neuAudioFeature[i])
            allTextFeature.append(neuTextFeature[i])
            allFileName.append(neuFileName[i])
            allLabels.append(neuLabels[i])

        i +=1

    return allAudioFeature, allTextFeature, allFileName, allLabels

def organizeFeatures(dirAudio, dirText, dirLabel, labelLimit):

    joyAudioFeature, joyFileName = readFeatures(os.path.join(dirAudio, 'joy'), labelLimit)
    angAudioFeature, angFileName = readFeatures(os.path.join(dirAudio, 'ang'), labelLimit)
    sadAudioFeature, sadFileName = readFeatures(os.path.join(dirAudio, 'sad'), labelLimit)
    neuAudioFeature, neuFileName = readFeatures(os.path.join(dirAudio, 'neu'), labelLimit)
    joyTextFeature, allFileName = readFeatures(os.path.join(dirText, 'joy'), labelLimit)
    angTextFeature, angFileName = readFeatures(os.path.join(dirText, 'ang'), labelLimit)
    sadTextFeature, sadFileName = readFeatures(os.path.join(dirText, 'sad'), labelLimit)
    neuTextFeature, neuFileName = readFeatures(os.path.join(dirText, 'neu'), labelLimit)
    joyLabels, joyFileName = readFeatures(os.path.join(dirLabel, 'joy'), labelLimit)
    angLabels, angFileName = readFeatures(os.path.join(dirLabel, 'ang'), labelLimit)
    sadLabels, sadFileName = readFeatures(os.path.join(dirLabel, 'sad'), labelLimit)
    neuLabels, neuFileName = readFeatures(os.path.join(dirLabel, 'neu'), labelLimit)
    '''print(allAudioFeature.shape)
    print(allTextFeature.shape)
    print(allLabels.shape)'''

    #BUILD SHUFFLED FEATURE FILES FOR TRAINING
    allAudioFeature = []
    allTextFeature = []
    allFileName = []
    allLabels = []
    i = 0
    while i < labelLimit:
        allAudioFeature.append(joyAudioFeature[i])
        allAudioFeature.append(angAudioFeature[i])
        allAudioFeature.append(sadAudioFeature[i])
        allAudioFeature.append(neuAudioFeature[i])

        allTextFeature.append(joyTextFeature[i])
        allTextFeature.append(angTextFeature[i])
        allTextFeature.append(sadTextFeature[i])
        allTextFeature.append(neuTextFeature[i])

        allFileName.append(joyFileName[i])
        allFileName.append(angFileName[i])
        allFileName.append(sadFileName[i])
        allFileName.append(neuFileName[i])

        allLabels.append(joyLabels[i])
        allLabels.append(angLabels[i])
        allLabels.append(sadLabels[i])
        allLabels.append(neuLabels[i])

        i +=1

    return allAudioFeature, allTextFeature, allFileName, allLabels

def buildBLTSM(maxTimestepAudio, numFeaturesAudio, maxTimestepText, numFeaturesText):

    nb_lstm_cells = 128
    nb_classes = 4
    nb_hidden_units = 512 #128

    #MODEL AUDIO WITH ATTENTION
    #Input attention
    input_attention = Input(shape=(nb_lstm_cells * 2,))
    u = Dense(nb_lstm_cells * 2, activation='softmax')(input_attention)
    #Input Audio and Text
    input_featureAudio = Input(shape=(maxTimestepAudio, numFeaturesAudio))
    input_featureText = Input(shape=(maxTimestepText, numFeaturesText))
    #Both model parallel structure
    x1 = Masking(mask_value=0.)(input_featureText)
    x1 = Dense(nb_hidden_units, activation='relu')(x1)
    x1 = Dropout(0.5)(x1)
    x1 = Dense(nb_hidden_units, activation='relu')(x1)
    x1 = Dropout(0.5)(x1)
    y1 = Bidirectional(LSTM(nb_lstm_cells, return_sequences=True, dropout=0.5))(x1)
    x2 = Masking(mask_value=0.)(input_featureAudio)
    x2 = Dense(nb_hidden_units, activation='relu')(x2)
    x2 = Dropout(0.5)(x2)
    x2 = Dense(nb_hidden_units, activation='relu')(x2)
    x2 = Dropout(0.5)(x2)
    y2 = Bidirectional(LSTM(nb_lstm_cells, return_sequences=True, dropout=0.5))(x2)
    #Attention step parallel for both model
    alpha1 = dot([u, y1], axes=-1)  # inner prod.
    alpha1 = Activation('softmax')(alpha1)
    alpha2 = dot([u, y2], axes=-1)  # inner prod.
    alpha2 = Activation('softmax')(alpha2)
    z1 = dot([alpha1, y1], axes=1)
    z2 = dot([alpha2, y2], axes=1)
    #Merge step
    mrg = Merge(mode='concat')([z1,z2])
    '''mrg = Concatenate([z1,z2])'''
    #Dense layer and final output
    refOut = Dense(nb_hidden_units, activation='relu')(mrg)
    output = Dense(nb_classes, activation='softmax')(refOut)

    model = Model(inputs=[input_attention, input_featureAudio, input_featureText], outputs=output)
    model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['categorical_accuracy']) #mean_squared_error #categorical_crossentropy

    return model

def reshapeLSTMInOut(Feat, label, maxTimestep):
    X = []
    X = np.asarray(Feat)
    X = pad_sequences(X, maxlen=maxTimestep, dtype='float32')
    Y = np.asarray(label)
    Y = Y.reshape(len(Y), 4)

    return X, Y

def predictFromModel(model, inputTestAudio, inputTestText, Labels, maxTimestepAudio, maxTimestepText):

    allPrediction = []
    allPredictionClasses = []
    allPredictionClassesMerged = []
    expected = []

    #FORMAT X & Y
    X_Audio, Y = reshapeLSTMInOut(inputTestAudio, Labels, maxTimestepAudio)
    X_Text, Y = reshapeLSTMInOut(inputTestText, Labels, maxTimestepText)

    #PREPARE ATTENTION ARRAY INPUT:  training and test
    nb_attention_param = 256
    attention_init_value = 1.0 / 256
    u_train = np.full((X_Audio.shape[0], nb_attention_param), attention_init_value, dtype=np.float64)

    #PREDICT
    yhat = model.predict([u_train, X_Audio, X_Text])
    for i in range(len(yhat)):
        #print('Expected:', Y[i], 'Predicted', yhat[i])
        Pindex, Pvalue = max(enumerate(yhat[i]), key=operator.itemgetter(1))
        allPredictionClasses.append(Pindex)
        allPrediction.append(yhat[i])
        expected.append(Y[i])

    #EVALUATE THE MODEL
    scores = model.evaluate([u_train, X_Audio, X_Text], Y, verbose=0)
    print('NORMAL EVALUATION %s: %.2f%%' % (model.metrics_names[1], scores[1]*100))

    return allPredictionClasses, allPrediction, expected

if __name__ == '__main__':

    #DEFINE MAIN ROOT
    Computer = 'new'
    #Computer = 'old'
    if Computer == 'new':
        mainRoot = os.path.normpath(r'C:\DATA\POLIMI\----TESI-----\Corpus_Test')
        dirRes = os.path.normpath(r'C:\DATA\POLIMI\----TESI-----\Z_Results\Recent_Results')
    if Computer == 'old':
        mainRoot = os.path.normpath(r'D:\DATA\POLIMI\----TESI-----\Corpus_Test')
        dirRes = os.path.normpath(r'D:\DATA\POLIMI\----TESI-----\Z_Results\Recent_Results')

    #BUILD PATH FOR EACH FEATURE DIR
    dirAudio = os.path.join(mainRoot + '\FeaturesAudio')
    dirText = os.path.join(mainRoot + '\FeaturesText')
    dirLabel = os.path.join(mainRoot + '\LablesEmotion')

    #SET MODELS PATH
    mainRootModel = os.path.join(dirRes, 'RNN_Model_FULL_saved.h5')
    OutputWeightsPath = os.path.join(dirRes, 'weights-improvement-60-0.64.hdf5')

    #DEFINE PARAMETERS
    flagLoadModel = 0 #0=model, 1=weight
    labelLimit = 384 #170 for balanced, 380 for max [joy 299, ang 170, sad 245, neu 384] TOT 1098
    fileLimit = (labelLimit*4) #number of file trained: len(allAudioFeature) or a number
    nameFileResult = 'PredM-FULL-Label_'+str(labelLimit)

    #EXTRACT FEATURES, NAMES, LABELS, AND ORGANIZE THEM IN AN ARRAY
    allAudioFeature, allTextFeature, allFileName, allLabels = organizeFeaturesV2(dirAudio, dirText, dirLabel, labelLimit)

    #FIND MAX TIMESTEP FOR PADDING
    maxTimestepAudio = 290 #setted with training because no test file is longer than 290
    maxTimestepText = 85 #text

    #MODEL SUMMARY
    print('Predict of #file: ', fileLimit)
    print('AUDIO Files with #features: ', allAudioFeature[0].shape[1])
    print('AUDIO Max time step: ',maxTimestepAudio)
    print('TEXT Files with #features: ', allTextFeature[0].shape[1])
    print('TEXT Max time step: ',maxTimestepText)
    print('Predict number of each emotion: ', labelLimit)

    #LOAD MODEL OR WIEGHTS FOR AUDIO AND TEXT MODEL
    if flagLoadModel == 0:
        model = load_model(mainRootModel)
    else:
        model = buildBLTSM(maxTimestepAudio, allAudioFeature[0].shape[1], maxTimestepText, allTextFeature[0].shape[1])
        model.load_weights(OutputWeightsPath)

    #PREDICT
    allPredictionClasses, allPrediction, expected = predictFromModel(model, allAudioFeature, allTextFeature, allLabels, maxTimestepAudio, maxTimestepText)

    #PREDICT & SAVE
    computeConfMatrix(allPredictionClasses, expected, dirRes, nameFileResult, True)

    print('END')


#   Copyright 2018 Samuel Payne sam_payne@byu.edu
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#       http://www.apache.org/licenses/LICENSE-2.0
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import pandas as pd
import numpy as np
import os
import warnings
import datetime
import logging
from gtfparse import read_gtf

from cptac.dataset import Dataset
from cptac.dataframe_tools import *
from cptac.exceptions import FailedReindexWarning, PublicationEmbargoWarning, ReindexMapError
from cptac.utils import get_boxnote_text
from .mssmclinical import MssmClinical

class WashuPdac(Dataset):

    def __init__(self, no_internet, version):
        """Load all of the bcmbrca dataframes as values in the self._data dict variable, with names as keys, and format them properly.

        Parameters:
        version (str, optional): The version number to load, or the string "latest" to just load the latest building. Default is "latest".
        no_internet (bool, optional): Whether to skip the index update step because it requires an internet connection. This will be skipped automatically if there is no internet at all, but you may want to manually skip it if you have a spotty internet connection. Default is False.
        """
        #ignore logging messages
        logger = logging.getLogger()
        logger.setLevel(logging.CRITICAL)

        # Set some needed variables, and pass them to the parent Dataset class __init__ function

        # This keeps a record of all versions that the code is equipped to handle. That way, if there's a new data release but they didn't update their package, it won't try to parse the new data version it isn't equipped to handle.
        valid_versions = ["1.0"]

        data_files = {
            "1.0": [
                "PDA_discovery.dnp.annotated.exonic.maf.gz",
                "PDA_NAT_RNA-Seq_Expr_WashU_FPKM.tsv.gz",
                "PDA_tumor_RNA-Seq_Expr_WashU_FPKM.tsv.gz",
                "PDA_mature_miRNA_combined.tsv",
                "PDA_precursor_miRNA_combined.tsv",
                "PDA_total_miRNA_combined.tsv",
                "PDA_xCell.txt",
                "CIBERSORT.Output_Abs_PDA.txt",
                "PDA.gene_level.from_seg.filtered.tsv",
                "gencode.v22.annotation.gtf.gz",
                "CPTAC_pancan_RNA_tumor_purity_ESTIMATE_WashU.tsv.gz",
                "README_miRNA",
                "README_CIBERSORT",
                "README_xCell",
                "README_somatic_mutation_WXS",
                "README_gene_expression",
                "README.boxnote",
                "README_ESTIMATE_WashU"
            ]
        }

        # Call the parent class __init__ function
        super().__init__(cancer_type="washupdac", version=version, valid_versions=valid_versions, data_files=data_files, no_internet=no_internet)

        # get clinical df (used to slice out cancer specific patient_IDs in tumor_purity file)
        mssmclin = MssmClinical(no_internet=no_internet, version=version, filter_type='pancanpdac') #_get_version - pancandataset
        clinical_df = mssmclin.get_clinical()

        # Load the data into dataframes in the self._data dict
        loading_msg = f"Loading {self.get_cancer_type()} v{self.version()}"
        for file_path in self._data_files_paths: # Loops through files variable

            # Print a loading message. We add a dot every time, so the user knows it's not frozen.
            loading_msg = loading_msg + "."
            print(loading_msg, end='\r')

            path_elements = file_path.split(os.sep) # Get a list of the levels of the path
            file_name = path_elements[-1] # The last element will be the name of the file. We'll use this to identify files for parsing in the if/elif statements below

            if file_name == "PDA_discovery.dnp.annotated.exonic.maf.gz": # Note that we use the "file_name" variable to identify files. That way we don't have to use the whole path.
                df = pd.read_csv(file_path, sep='\t')
                # Rename the columns we want to keep to the appropriate names
                df = pd.read_csv(file_path, sep='\t')
                df['Patient_ID'] = df.loc[:, 'Tumor_Sample_Barcode']
                df = df.rename(columns={
                         "Hugo_Symbol":"Gene",
                         "Gene":"Gene_Database_ID",
                         "Variant_Classification":"Mutation",
                         "HGVSp_Short":"Location"})

                df = df.set_index("Patient_ID")
                df = df[ ['Gene'] + ["Mutation"] + ["Location"] + [ col for col in df.columns if col not in ["Gene","Mutation","Location"] ] ]
                df.index = df.index.str.replace(r"_T", "", regex=True)
                self._data["somatic_mutation"] = df

            if file_name == "PDA_NAT_RNA-Seq_Expr_WashU_FPKM.tsv.gz":
                df_norm = pd.read_csv(file_path, sep='\t')
                #change names to universal package names
                df_norm = df_norm.rename(columns={"gene_name": "Name","gene_id": "Database_ID"})
                df_norm = df_norm.set_index(["Name", "Database_ID"])
                df_norm = df_norm.sort_index()
                df_norm = df_norm.T #transpose
                df_norm.index.name = "Patient_ID"
                df_norm.index = df_norm.index.str.replace(r"-A", ".N", regex=True) #remove label for tumor samples
                self._helper_tables["transcriptomics_norm"] = df_norm

            if file_name == "PDA_tumor_RNA-Seq_Expr_WashU_FPKM.tsv.gz":
                df = pd.read_csv(file_path, sep='\t')
                #change names to universal package names
                df = df.rename(columns={"gene_name": "Name","gene_id": "Database_ID"})
                df = df.set_index(["Name", "Database_ID"])
                df = df.sort_index()
                df = df.T #transpose
                df.index.name = "Patient_ID"
                df.index = df.index.str.replace(r"-T", "", regex=True) #remove label for tumor samples
                self._helper_tables["transcriptomics_tumor"] = df

            elif 'miRNA_combined' in file_name:
                miRNA_type = file_name.split('_')[1] # get type of miRNA data (precursor, mature, or total)
                if miRNA_type == 'mature':
                    df = pd.read_csv(file_path, delimiter = '\t', index_col = ['Name', 'ID','Alias', 'Derives_from'])
                else:
                    df = pd.read_csv(file_path, delimiter = '\t', index_col = ['Name', 'ID','Alias'])
                df = df.transpose()
                df = average_replicates(df, common = '\.\d$') # average duplicates for C3L-02617 and C3N-02727
                df.index = df.index.str.replace('\.T$','', regex = True)
                df.index = df.index.str.replace('\.A$','.N', regex = True)
                df.index.name = 'Patient_ID'
                # Sort
                normal = df.loc[df.index.str.contains('\.N$', regex =True)]
                normal = normal.sort_values(by=["Patient_ID"])
                tumor = df.loc[~ df.index.str.contains('\.N$', regex =True)]
                tumor = tumor.sort_values(by=["Patient_ID"])
                all_df = tumor.append(normal)
                self._data[miRNA_type+'_miRNA'] = all_df

            elif file_name == "PDA_xCell.txt":
                df = pd.read_csv(file_path, sep = '\t', index_col = 0)
                df = df.transpose()
                df.columns.name = 'Name'
                df.index.name = 'Patient_ID'
                df.index = df.index.str.replace(r'-T$', '', regex=True) # remove label for tumor samples
                df.index = df.index.str.replace(r'-A$', '.N', regex=True) # change label for normal samples
                self._data["xcell"] = df

            elif file_name == "CIBERSORT.Output_Abs_PDA.txt":
                df = pd.read_csv(file_path, sep = '\t', index_col = 0)
                df.index.name = 'Patient_ID'
                df.columns.name = 'Name'
                df.index = df.index.str.replace(r'-T$', '', regex=True)
                df.index = df.index.str.replace(r'-A$', '.N', regex=True)
                self._data["cibersort"] = df

            elif file_name == "PDA.gene_level.from_seg.filtered.tsv":
                df = pd.read_csv(file_path, sep="\t")
                df = df.rename(columns={"Gene": "Name"})
                df = df.set_index("Name")
                self._data["CNV"] = df

            elif file_name == "gencode.v22.annotation.gtf.gz":
                df = read_gtf(file_path)
                df = df[["gene_name","gene_id"]]
                df = df.drop_duplicates()
                df = df.rename(columns={"gene_name": "Name","gene_id": "Database_ID"})
                df = df.set_index("Name")
                self._helper_tables["CNV_gene_ids"] = df

            elif file_name == "CPTAC_pancan_RNA_tumor_purity_ESTIMATE_WashU.tsv.gz":
                df = pd.read_csv(file_path, sep = "\t", na_values = 'NA')
                df.Sample_ID = df.Sample_ID.str.replace(r'-T', '', regex=True) # only tumor samples in file
                df = df.set_index('Sample_ID')
                df.index.name = 'Patient_ID'
                # Use list of patient_ids to slice out cancers
                patient_ids = clinical_df.index.to_list()
                df = df.loc[df.index.isin(patient_ids)]
                self._data["tumor_purity"] = df

            elif file_name == "README_miRNA":
                with open(file_path, 'r') as reader:
                    self._readme_files["readme_miRNA"] = reader.read()

            elif file_name == "README_CIBERSORT":
                with open(file_path, 'r') as reader:
                    self._readme_files["readme_cibersort"] = reader.read()

            elif file_name == "README_xCell":
                with open(file_path, 'r') as reader:
                    self._readme_files["readme_xcell"] = reader.read()

            elif file_name == "README_somatic_mutation_WXS":
                with open(file_path, 'r') as reader:
                    self._readme_files["readme_somatic_mutation"] = reader.read()

            elif file_name == "README_gene_expression":
                with open(file_path, 'r') as reader:
                    self._readme_files["readme_transcriptomics"] = reader.read()

            elif file_name == "README.boxnote":
                self._readme_files["readme_cnv"] = get_boxnote_text(file_path)

            elif file_name == "README_ESTIMATE_WashU":
                with open(file_path, 'r') as reader:
                    self._readme_files["readme_tumor_purity"] = reader.read()

        print(' ' * len(loading_msg), end='\r') # Erase the loading message
        formatting_msg = f"Formatting {self.get_cancer_type()} dataframes..."
        print(formatting_msg, end='\r')

        # combine and create transcriptomic dataframe
        rna_tumor = self._helper_tables.get("transcriptomics_tumor")
        rna_normal = self._helper_tables.get("transcriptomics_norm") # Normal entries are already marked with 'N' on the end of the ID
        rna_combined = rna_tumor.append(rna_normal)
        self._data["transcriptomics"] = rna_combined

        # CNV
        cnv = self._data["CNV"]
        gene_ids = self._helper_tables["CNV_gene_ids"]
        df = cnv.join(gene_ids,how = "left") #merge in gene_ids
        df = df.reset_index()
        df = df.set_index(["Name", "Database_ID"]) #create multi-index
        df = df.T
        df.index.name = 'Patient_ID'
        self._data["CNV"] = df

        self._data = sort_all_rows_pancan(self._data) # Sort IDs (tumor first then normal)

        print(" " * len(formatting_msg), end='\r') # Erase the formatting message