import json
from aiogram.types import CallbackQuery,FSInputFile, ReplyKeyboardRemove, Message
from aiogram import Bot
from keyboards.reply.reply_keyboards import get_reply_keyboard
from states.states import orderStates
from keyboards.reply.reply_keyboards import get_location_keyboard, get_number_keyboard
from loader import bot, goods, tech_chat
from utils.callbackdata import *
from aiogram.fsm.context import FSMContext
from keyboards.inline.inline_keyboards import *
from .message import check_favs
from connectors.db_funct import create_order,del_from_fav,create_user,add_to_cart,check_cartTable, clean_cart_db,clean_cart_product,add_to_fav,check_favTable
import pandas as pd
from geopy.geocoders import Nominatim
geolocator = Nominatim(user_agent="geoapiExercises")
import datetime
import ast
from pytz import timezone

#ВЫЗОВ ПОДКАТЕГОРИЙ
async def category_call(call: CallbackQuery, bot: Bot,callback_data: Category):
    photo = FSInputFile('photos/NataliyaPhoto.jpg')
    create_user(call.message.chat.id, call.message.chat.first_name)
    df_goods = goods.loc[goods['Категория']==callback_data.name].reset_index(drop=True)
    if (str(goods.loc[goods['Категория']==callback_data.name]['Подкатегория'].unique()[0]))!='nan':
        list_but = goods.loc[goods['Категория']==callback_data.name]['Подкатегория'].unique().tolist()
        ikb_catalogue = subcategory_buttons(list_but,(1,))
    elif (str(goods.loc[goods['Категория']==callback_data.name]['Наименовние по каталогу'].unique()[0]))!='nan':
        ikb_catalogue = product_buttons(df_goods,(1,))
    else:
        await call.answer('Пока нет товаров в данной категории')
        ikb_catalogue = subcategory_buttons(['пусто'],(1,))
        return
    await bot.send_photo(chat_id= call.message.chat.id,
                        photo=photo,
                        caption=callback_data.name,
                        reply_markup=ikb_catalogue)
    text = callback_data.name
    await call.answer(text)

#ВЫЗОВ ТОВАРОВ
async def subcategory_call(call: CallbackQuery, bot: Bot,callback_data: Subcat):
    photo = FSInputFile('photos/NataliyaPhoto.jpg')
    df_goods = goods.loc[goods['Подкатегория']==callback_data.name].reset_index(drop=True)
    ikb_catalogue = product_buttons(df_goods,(1,))
    await bot.send_photo(chat_id= call.message.chat.id,
                        photo=photo,
                        caption=str(call.message.caption) + '\\'+ str(callback_data.name),
                        reply_markup=ikb_catalogue)
    text = callback_data.name
    await call.answer(text)

#ПРОСМОТР ТОВАРА
async def product_call(call: CallbackQuery, bot: Bot,callback_data: Product):
    cat_cap = call.message.caption
    if not cat_cap:
        cat_cap=''
    good = goods.loc[goods['Артикул']==callback_data.id]
    cart_data = pd.DataFrame(check_cartTable(call.message.chat.id), columns=['userid', 'itemid', 'weight','itemsum'])
    if not cart_data.empty:
        cart_exist = True
    else:
        cart_exist = False
    print(good.reset_index(drop=True)['Артикул'][0])
    photo = good.reset_index(drop=True)['Фото товара'][0]
    price = good.reset_index(drop=True)['Цена за 1 кг.'][0]
    title = good.reset_index(drop=True)['Наименовние по каталогу'][0]
    volume = good.reset_index(drop=True)['Объем'][0]
    weight = float(str(good.reset_index(drop=True)['Заказ от'][0]).replace(',','.'))
    sumprice = float(str(weight).replace(',','.')) * float(price)
    ikb_catalogue = product_detail_buttons(good, weight,sumprice,cart_exist)
    await bot.send_photo(chat_id= call.message.chat.id,
                        photo=photo,
                        caption=cat_cap + '\n\n' +title + ' '+ str(price)+ '₽ за '+str(volume),
                        reply_markup=ikb_catalogue)
    await call.answer(title)

#ДОБАВИТЬ ВЕС\ШТ
async def add_button(call: CallbackQuery, callback_data: AddProduct):
    good = goods.loc[goods['Артикул']==callback_data.id]
    cart_data = pd.DataFrame(check_cartTable(call.message.chat.id), columns=['userid', 'itemid', 'weight','itemsum'])
    if not cart_data.empty:
        cart_exist = True
    else:
        cart_exist = False
    step = good.reset_index(drop=True)['Шаг заказа'][0]
    price = good.reset_index(drop=True)['Цена за 1 кг.'][0]
    weight = callback_data.weight + float(step)
    sumprice = callback_data.sumprice + (price*step)
    await call.message.edit_reply_markup(reply_markup=product_detail_buttons(good=good, weight=weight, sumprice=sumprice,cart_exist=cart_exist))

#УБАВИТЬ ВЕС\ШТ
async def delete_button(call: CallbackQuery, callback_data: DelProduct):
    good = goods.loc[goods['Артикул']==callback_data.id]
    cart_data = pd.DataFrame(check_cartTable(call.message.chat.id), columns=['userid', 'itemid', 'weight','itemsum'])
    if not cart_data.empty:
        cart_exist = True
    else:
        cart_exist = False
    step = good.reset_index(drop=True)['Шаг заказа'][0]
    price = good.reset_index(drop=True)['Цена за 1 кг.'][0]
    if callback_data.weight > float(str(good.reset_index(drop=True)['Заказ от'][0]).replace(',','.')):
        weight = callback_data.weight - float(step)
        sumprice = callback_data.sumprice - (price*step)
        await call.message.edit_reply_markup(reply_markup=product_detail_buttons(good=good, weight=weight, sumprice=sumprice, cart_exist=cart_exist))
    else:
        await call.answer('Минимальный заказ от '+ str(good.reset_index(drop=True)['Заказ от'][0])+' ' +good.reset_index(drop=True)['Объем'][0])

###################################
#ДОБАВИТЬ В КОРЗИНУ
async def addcart_button(call: CallbackQuery, callback_data: AddCart):
    create_user(call.message.chat.id, call.message.chat.first_name)
    add_to_cart(call.message.chat.id, callback_data.id, callback_data.weight, callback_data.sumprice)
    cart_exist = True
    good = goods.loc[goods['Артикул']==callback_data.id]
    price = good.reset_index(drop=True)['Цена за 1 кг.'][0]
    weight = float(str(good.reset_index(drop=True)['Заказ от'][0]).replace(',','.'))
    sumprice = float(str(weight).replace(',','.')) * float(price)
    ikb_catalogue = product_detail_buttons(good, weight,sumprice,cart_exist)
    try:
        await call.message.edit_reply_markup(reply_markup=ikb_catalogue)
    except:
        pass
    await call.answer('Товар добавлен в коризну')

#ОЧИСТКА КОРИЗИНЫ
async def clean_cart(call: CallbackQuery):
    cart_data = pd.DataFrame(check_cartTable(call.message.chat.id), columns=['userid', 'itemid', 'weight','itemsum'])
    if not cart_data.empty:
        cart_exist = True
        await bot.send_message(chat_id= call.message.chat.id,
            text='Выберите действие на кнопке ниже👇',
            reply_markup=three_inline_buttons(['Удалить одну позицию','Очистить всё','К корзине'],(1,2,)))
        await call.answer('Очистка корзины')
    else:
        await call.answer('Корзина пуста')

#ОЧИСТКА КОРИЗИНЫ УДАЛЕНИЕ ОДНОЙ ПОЗИЦИИ №1
async def clean_one(call: CallbackQuery):
    cart_data = pd.DataFrame(check_cartTable(call.message.chat.id), columns=['userid', 'itemid', 'weight','itemsum'])
    if not cart_data.empty:
        df2 = cart_data[['itemid','weight','itemsum']].groupby('itemid').sum().reset_index()
        goods_df = goods.loc[goods['Артикул'].isin(df2['itemid'].tolist())].reset_index(drop=True)
        await bot.send_message(chat_id= call.message.chat.id,
        text='Выберите продукт для удаления на кнопках ниже',
        reply_markup=del_cart_product(goods_df,(1,)))
        await call.answer('Удаление одной позиции')
    else:
        await call.answer('Корзина пуста')

#ОЧИСТКА КОРИЗИНЫ УДАЛЕНИЕ ОДНОЙ ПОЗИЦИИ №2
async def clean_two(call: CallbackQuery, callback_data: DelCartProduct):
    clean_cart_product(call.message.chat.id, callback_data.id)
    good = goods.loc[goods['Артикул']==callback_data.id]
    title = good.reset_index(drop=True)['Наименовние по каталогу'][0]
    await bot.send_message(chat_id= call.message.chat.id,
            text='Продукт  "'+ str(title) + '"  удален',
            reply_markup=three_inline_buttons(['Удалить одну позицию','Очистить всё','К корзине', 'Оформить'],(1,2,)))
    await call.answer()

#ОЧИСТКА КОРИЗИНЫ ПОЛНОЕ УДАЛЕНИЕ
async def clean_cart_full(call: CallbackQuery):
    clean_cart_db(call.message.chat.id)
    await bot.send_message(chat_id= call.message.chat.id,
            text='Корзина полностью очищена',
            reply_markup=three_inline_buttons(['К каталогу','К корзине'],(1,)))
    await call.answer('Корзина очищена')

################################
#ДОБАВИТЬ В ИЗБРАННОЕ
async def addfav_button(call: CallbackQuery, callback_data: AddFav):
    create_user(call.message.chat.id, call.message.chat.first_name)
    fav_data = pd.DataFrame(check_favTable(call.message.chat.id), columns=['userid', 'itemid'])
    if not fav_data.empty:
        favs = fav_data.loc[fav_data['itemid']==callback_data.id]
        if not favs.empty:
            await call.answer('Этот товар уже добавлен в ваши избранные')
        else:
            add_to_fav(call.message.chat.id, callback_data.id)
            await call.answer('Товар добавлен в избранное')
    else:
        add_to_fav(call.message.chat.id, callback_data.id)
        await call.answer('Товар добавлен в избранное')

#ОЧИСТКА ИЗБРАННОГО УДАЛЕНИЕ ОДНОЙ ПОЗИЦИИ №1
async def clean_favs(call: CallbackQuery):
    favs_data = pd.DataFrame(check_favTable(call.message.chat.id), columns=['userid', 'itemid'])
    if not favs_data.empty:
        df_goods =goods.loc[goods['Артикул'].isin(favs_data['itemid'].tolist())].reset_index(drop=True)
        ikb_favs = del_favs_product(df_goods,(1,))
        await call.message.answer(text='Выберете продукт для удаления на кнопках ниже👇',
                            reply_markup=ikb_favs)
    else:
        ikb_cart = three_inline_buttons(['К каталогу',],(1,))
        await call.message.answer(text='В избранном пусто, чтобы добавить товар в избранное, перейдите в карточку товара и нажмите "В избранное"',
                            reply_markup=ikb_cart)

#ОЧИСТКА ИЗБРАННОГО УДАЛЕНИЕ ОДНОЙ ПОЗИЦИИ №2
async def clean_favs_two(call: CallbackQuery, callback_data: DelFavsProduct):
    del_from_fav(call.message.chat.id, callback_data.id)
    good = goods.loc[goods['Артикул']==callback_data.id]
    title = good.reset_index(drop=True)['Наименовние по каталогу'][0]
    await bot.send_message(chat_id= call.message.chat.id,
            text='Продукт  "'+ str(title) + '" удален',
            reply_markup=three_inline_buttons(['Избранные','К каталогу','Корзина'],(1,2,)))
    await call.answer()

############################
#ОФОРМЛЕНИЕ

async def neworder_canlcel(call, state: FSMContext):
    if type(call) == CallbackQuery:
        await call.answer()
        call=call.message
    await bot.send_message(chat_id= call.chat.id,
            text='Оформление отменено',
            reply_markup=three_inline_buttons(['Избранные','К каталогу','Корзина'],(1,2,)))
    await state.clear()

async def neworder_shipments(call: CallbackQuery , state: FSMContext):
    shipments_list = goods.loc[( goods['Категория']=='Доставка')].reset_index(drop=True)
    text_msg=''
    for i in range(len(shipments_list)):
        title = shipments_list['Наименовние по каталогу'][i]
        price = shipments_list['Цена за 1 кг.'][i]
        text_msg +='🔹' + title + str('  🫰Цена: ') + str(price) + ';\n'
    await bot.send_message(chat_id= call.message.chat.id,
                  text=f'Варианты доставки:\n{text_msg}Выберите тип доставки на кнопках ниже',
                  reply_markup=shipments_buttons(shipments_list))
    await state.set_state(orderStates.shipment)
    await call.answer()

async def neworder_adress(call: CallbackQuery , state: FSMContext):
    # print(call.data)
    await call.answer()
    ship_product = goods.loc[goods['Артикул']==int(call.data)].reset_index(drop=True)[['Наименовние по каталогу', 'Цена за 1 кг.']]
    cart_data = pd.DataFrame(check_cartTable(call.message.chat.id), columns=['userid', 'itemid', 'weight','itemsum'])
    if not cart_data.empty:
        df2 = cart_data[['itemid','weight','itemsum']].groupby('itemid').sum().reset_index()
        await state.update_data(products = str(df2.to_dict(orient='list')))
        await state.update_data(shipment = str(ship_product['Наименовние по каталогу'][0]) + str(' За ') +str(ship_product['Цена за 1 кг.'][0]))
        text_msg='Ваши товары: \n'
        res_sum = float(ship_product['Цена за 1 кг.'][0])
        for i in range(len(df2)):
            good = goods.loc[goods['Артикул']==df2['itemid'][i]]
            title = good.reset_index(drop=True)['Наименовние по каталогу'][0]
            volume = good.reset_index(drop=True)['Объем'][0]
            text_msg +='🔹' + title + str('  ⚖Вес: ') + str(df2['weight'][i]) +' '+str(volume) + str('  🫰Сумма: ') + str(df2['itemsum'][i]) + ';\n'
            res_sum += df2['itemsum'][i]
        text_msg += '____________________________________\n\n Выбрана доставка {} за {} \n\nСумма с учетом доставки: {}'.format(ship_product['Наименовние по каталогу'][0], ship_product['Цена за 1 кг.'][0], str(res_sum)+' ₽')
        await call.message.answer(text=text_msg)
        if int(call.data) == 520:
            await call.message.answer(text='Супер, наш адрес Такая-то улица дом 4, запиши, чтоб не потерять', reply_markup=get_reply_keyboard(['Записал!']))
            await state.set_state(orderStates.number)
        else:
            kb = get_location_keyboard()
            await bot.send_message(chat_id= call.message.chat.id,
                        text='Отлично!⭐\nКуда доставим?📍\nМожешь нажать на кнопку "Геолокация" чтобы поделиться своим местоположением или просто отправь свой адрес в сообщении🙃\nСкопировать пример: `г. Москва, ул. Чистопрудный бульвар, дом 6, квартира 20`',
                        reply_markup=kb, parse_mode='Markdown')
            await state.set_state(orderStates.number)
    else:
        ikb_cart = three_inline_buttons(['К каталогу',],(1,))
        await call.message.answer(text='Корзина пуста', reply_markup=ikb_cart)

async def neworder_number(msg: Message, state: FSMContext):
    if msg.location:
        location = geolocator.reverse(str(msg.location.latitude)+","+str(msg.location.longitude))
    else:
        location = msg.text
    await state.update_data(adress = location)
    kb = get_number_keyboard()
    await bot.send_message(chat_id= msg.chat.id,
                text='Почти всё!😇\nОсталось указать номер телефона📱\nМожешь нажать на кнопку "Контакт" чтобы поделиться текущим номером, если хочешь указать другой номер для связи просто отправь его в сообщении',
                reply_markup=kb)

    await state.set_state(orderStates.send_order)

async def neworder_send(msg: Message, state: FSMContext):
    if msg.contact:
        phone = msg.contact.phone_number
    else:
        phone = msg.text
    await state.update_data(phone = phone)
    data = await state.get_data()
    today = datetime.datetime.now(timezone('Europe/Moscow')).strftime('%Y-%m-%d %H:%M:%S')
    df_ord_goods = pd.DataFrame(eval(data['products']))
    create_order(msg.chat.username, msg.chat.id, str(data['products']), str(today), str(data['adress']), str(data['phone']), str(data['shipment']))
    order_goods = goods.loc[goods['Артикул'].isin(df_ord_goods['itemid'].tolist())].reset_index(drop=True)[['Наименовние по каталогу', 'Цена за 1 кг.']]
    text_goods = ''
    res_sum=float(data['shipment'].split(' За ')[1])
    for i in range(len(order_goods)):
        text_goods+=str('--') + order_goods['Наименовние по каталогу'][i] + str(' Количество: ')+ str(df_ord_goods['weight'][i]) + str(' Сумма: ') + str(df_ord_goods['itemsum'][i]) + str('\n')
        res_sum+=float(df_ord_goods['itemsum'][i])
    try:
        await bot.send_message(tech_chat, f"✉ | Новый заказ\nОт: {msg.chat.username}\nТовары:\n`{text_goods}`\nДоставка: `{data['shipment']}`\nАдрес: `{data['adress']}`\nТелефон: `{data['phone']}`\nОбщая сумма заказа: `{res_sum}`\n\n📝 Чтобы ответить на вопрос введите `/ответ {msg.chat.id} Ваш ответ`", parse_mode='Markdown')
    except:
        await bot.send_message(tech_chat, f"✉ | Новый заказ\nОт: {msg.chat.username}\nТовары:\n`{text_goods}`\nДоставка: `{data['shipment']}`\nАдрес: `{data['adress']}`\nТелефон: `{data['phone']}`\nОбщая сумма заказа: `{res_sum}`\n\n📝 Чтобы ответить на вопрос введите `/ответ {msg.chat.id} Ваш ответ`")
    await msg.answer(f"Ваш заказ создан и ожидает оплаты\nПеревод по номеру 89991112233 Сбербанк Проверкин П.А, сумма {res_sum}\nМодератор проверит его и свяжется с вами✅", reply_markup=ReplyKeyboardRemove())
    await state.clear()
### КОНЕЦ ФУНКЦИЙ ЗАКАЗА ###

#ОБРАТНО
async def back(call: CallbackQuery, bot: Bot):
    photo = FSInputFile('photos/NataliyaPhoto.jpg')
    if str(call.message.reply_markup.inline_keyboard[0][0]).startswith("text='-'"):
        g_cat = (call.message.caption).split('\n\n')[0].split('\\')
        if len(g_cat)>1:
            df_goods = goods.loc[goods['Подкатегория']==g_cat[1]].reset_index(drop=True)
            ikb_catalogue = product_buttons(df_goods,(1,))
            cap = str(call.message.caption).split('\n\n')[0]
        elif '\n' not in str(call.message.caption):
            await check_favs(call)
            await call.answer('Избранные')
            return
        else:
            df_goods = goods.loc[goods['Категория']==g_cat[0]].reset_index(drop=True)
            ikb_catalogue = product_buttons(df_goods,(1,))
            cap = str(call.message.caption).split('\n\n')[0].split('\\')[0]
    elif len((call.message.caption).split('\\'))>1:
        g = goods.loc[goods['Категория']==str(call.message.caption).split('\\')[0]]['Подкатегория'].unique()
        list_but =g.tolist()
        ikb_catalogue = subcategory_buttons(list_but,(1,))
        cap = str(call.message.caption).split('\\')[0]
    else:
        list_but = goods['Категория'].unique().tolist()[:-2]
        ikb_catalogue = category_buttons(list_but,(2,1,1,1,1,1,1,2,1,1,2,1,))
        cap='Привет! 🙂\nС Вами Наталья и мой новый ассистент - бот 🤖 Market Buyer 👩🏼🌾\nТеперь это Ваш личный помощник для заказа свежих и вкусных продуктов 🍒\n\nСмело наполняйте корзину 🧺 и оформляйте заказы себе и близким, ведь доставка свежих и вкусных продуктов доступна по всей Москве и не только ))'
    await bot.send_photo(chat_id= call.message.chat.id,
                    photo=photo,
                    caption=cap,
                    reply_markup=ikb_catalogue)
    await call.answer(call.message.text)


import numpy as np
import matplotlib.pyplot as plt
import math
from mpl_toolkits.mplot3d import Axes3D
from scipy.spatial.transform import Rotation

class Visualize3D:
    def __init__(self):
        plt.interactive(True)
        fig = plt.figure()
        self.ax = plt.axes(projection="3d")
        self.ax.set_xlim([-5, 5])
        self.ax.set_ylim([-5, 5])
        self.ax.set_zlim([0, 5])
        self.ax.set_xlabel("X Axis")
        self.ax.set_ylabel("Y Axis")
        self.ax.set_zlabel("Z Axis")

    def update(self, poses):
        X, Y, Z, U, V = [[] for _ in range(5)]
        for pose in poses:
            pose = np.array(poses[pose][0])
            trans = pose[:3, 3]
            rot = pose[:3, :3]
            rot = Rotation.from_matrix(rot).as_euler("xyz")
            yaw = rot[2]
            new_x = math.sin(yaw)
            new_y = math.cos(yaw)
            X.append(trans[0])
            Y.append(trans[1])
            Z.append(trans[2])
            U.append(new_x)
            V.append(new_y)

        self.ax.quiver(X, Y, Z, U, V, 0)
        plt.draw()
        plt.pause(0.02)
        self.ax.cla()
        plt.show()


#!/usr/bin/python3
import os

# initialize asebamedulla in background and wait 0.3s to let
# asebamedulla startup
os.system("(asebamedulla ser:name=Thymio-II &) && sleep 0.3")
import matplotlib.pyplot as plt
from time import sleep
import dbus
import dbus.mainloop.glib
from threading import Thread

class Thymio:
    def __init__(self):
        self.proxSensorsVal=[0,0,0,0,0]
        self.aseba = self.setup()

    def move(self):
        count = 1
        left_wheel_velocity = -200
        right_wheel_velocity = 200
        # Get initial sensor values

        while count < 500:
            self.drive(left_wheel_velocity,right_wheel_velocity)
            #robot.sens()
            count += 1

        self.stop()

    def turn(self, l, r):
        left_wheel_velocity = l
        right_wheel_velocity = r
        # Get initial sensor values
        self.drive(left_wheel_velocity,right_wheel_velocity)

    def drive(self, left_wheel_speed, right_wheel_speed):
        #print("Left_wheel_speed: " + str(left_wheel_speed))
        #print("Right_wheel_speed: " + str(right_wheel_speed))

        left_wheel = left_wheel_speed
        right_wheel = right_wheel_speed

        self.aseba.SendEventName("motor.target", [left_wheel, right_wheel])

    def stop(self):
        left_wheel = 0
        right_wheel = 0
        self.aseba.SendEventName("motor.target", [left_wheel, right_wheel])

    def sens(self):

        prox_horizontal = self.aseba.GetVariable("thymio-II", "prox.horizontal")
        print("Sensing:")
        print(prox_horizontal[0])
        print(prox_horizontal[1])
        print(prox_horizontal[2])
        print(prox_horizontal[3])
        print(prox_horizontal[4])

    def get_variables_reply(self, r):
        global proxSensorsVal
        proxSensorsVal=r

    def get_variables_error(self, e):
        print('error:')
        print(str(e))

    def Braitenberg(self):
        #get the values of the sensors
        self.aseba.GetVariable("thymio-II", "prox.horizontal",reply_handler=self.get_variables_reply,error_handler=self.get_variables_error)

        #print the proximity sensors value in the terminal
        print(self.proxSensorsVal[0],self.proxSensorsVal[1],self.proxSensorsVal[2],self.proxSensorsVal[3],self.proxSensorsVal[4])

        #Parameters of the Braitenberg, to give weight to each wheels
        leftWheel=[-0.01,-0.005,-0.0001,0.006,0.015]
        rightWheel=[0.012,+0.007,-0.0002,-0.0055,-0.011]

        #Braitenberg algorithm
        totalLeft=0
        totalRight=0
        for i in range(5):
            totalLeft=totalLeft+(self.proxSensorsVal[i]*leftWheel[i])
            totalRight=totalRight+(self.proxSensorsVal[i]*rightWheel[i])

        #add a constant speed to each wheels so the robot moves always forward
        totalRight=totalRight+50
        totalLeft=totalLeft+50

        #print in terminal the values that is sent to each motor
        print("totalLeft")
        print(totalLeft)
        print("totalRight")
        print(totalRight)

        #send motor value to the robot
        self.aseba.SetVariable("thymio-II", "motor.left.target", [totalLeft])
        self.aseba.SetVariable("thymio-II", "motor.right.target", [totalRight])

        return True

    ############## Bus and aseba setup ######################################

    def setup(self):
        print("Setting up")
        dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
        bus = dbus.SessionBus()
        asebaNetworkObject = bus.get_object('ch.epfl.mobots.Aseba', '/')

        asebaNetwork = dbus.Interface(
            asebaNetworkObject, dbus_interface='ch.epfl.mobots.AsebaNetwork'
        )
        # load the file which is run on the thymio
        asebaNetwork.LoadScripts(
            'thympi.aesl', reply_handler=self.dbusError, error_handler=self.dbusError
        )

        # scanning_thread = Process(target=robot.drive, args=(200,200,))
        return asebaNetwork

    def stopAsebamedulla(self):
        os.system("pkill -n asebamedulla")

    def dbusReply(self):
        # dbus replys can be handled here.
        # Currently ignoring
        pass

    def dbusError(self, e):
        # dbus errors can be handled here.
        # Currently only the error is logged. Maybe interrupt the mainloop here
        print("dbus error: %s" % str(e))

# ------------------ Main -------------------------

if __name__ == '__main__':
    robot = Thymio()
    try:
        while True:
            robot.Braitenberg()
    finally:
        print("Stopping robot")
        exit_now = True
        sleep(5)
        os.system("pkill -n asebamedulla")
        print("asebamodulla killed")


from django.db import models

# Create your models here.
class host(models.Model):
    id = models.IntegerField(primary_key=True)
    ip = models.GenericIPAddressField('主机ip')
    inter_ip = models.CharField('外网ip',null=True,max_length=150)
    hostname = models.CharField('主机名', max_length=50)
    port = models.IntegerField('端口',default=22)
    brand = models.CharField('序列号',max_length=50,null=True)
    cpu = models.CharField('cpu',max_length=50,null=True)
    memory = models.CharField('内存大小',max_length=50,null=True)
    system_version = models.CharField('系统版本',max_length=50,null=True)
    label = models.CharField('主机标签',max_length=50,null=True)
    is_active = models.BooleanField('服务器状态',default=False)
    date_joined = models.DateTimeField('创建时间', auto_now_add=True)
    last_login = models.DateTimeField('修改时间', auto_now=True)

    def __unicode__(self):
        return '%s-%s' % (self.ip, self.hostname)

    class Meta:
        verbose_name = '主机管理'
        verbose_name_plural = '主机管理'


import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import os
from PIL import Image
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import classification_report, confusion_matrix
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow.keras import layers, models
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import seaborn as sns
import random
import pandas as pd
import streamlit as st

def download_images(category, num_images, st_progress, st_text):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
    }
    url = f"https://commons.wikimedia.org/wiki/Category:{category}"
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')

    if not os.path.exists(category):
        os.makedirs(category)

    image_elements = soup.find_all("img")
    image_elements = [img for img in image_elements if int(img.get('width', 0)) > 50 and int(img.get('height', 0)) > 50]

    for i, img in enumerate(image_elements[:num_images]):
        img_url = urljoin(url, img['src'])
        img_response = requests.get(img_url, headers=headers)

        if img_response.status_code == 200:
            img_data = img_response.content
            img_name = os.path.join(category, f"{category}_{i+1}.jpg")

            with open(img_name, 'wb') as f:
                f.write(img_data)

            st_text.text(f"Image {i+1} downloaded and saved as {img_name}")
        else:
            st_text.text(f"Failed to fetch image {i+1}: HTTP Status Code {img_response.status_code}")

    st_text.text(f"{num_images} images from category {category} downloaded successfully!")

def load_images_from_directory(base_path, categories, target_size=(224, 224)):
    images = []
    labels = []

    for category in categories:
        category_path = os.path.join(base_path, category)
        if os.path.isdir(category_path):
            for img_name in os.listdir(category_path):
                if img_name.endswith(".jpg"):
                    img_path = os.path.join(category_path, img_name)
                    img = Image.open(img_path)
                    img = img.resize(target_size)
                    img_array = np.array(img)
                    images.append(img_array)
                    labels.append(category)

    return images, labels

def main():
    st.title("Image Classification App")

    # Image Downloading Section
    st.header("Image Downloading")
    categories_input = st.text_input("Enter categories (comma separated):")
    st.text("Recommended categories: Chickens, Dogs, Fish, Horses, Monkeys")
    num_images = st.number_input("Number of images per category:", min_value=1, value=100)

    if st.button("Download Images"):
        categories = [category.strip() for category in categories_input.split(",") if category]
        if categories:
            with st.spinner('Downloading images...'):
                st_progress = st.progress(0)
                st_text = st.empty()
                for i, category in enumerate(categories, start=1):
                    st_text.text(f"Downloading images for category: {category}")
                    download_images(category, num_images, st_progress, st_text)
                    st_progress.progress(i / len(categories))
                st_text.text("Downloading completed!")
        else:
            st.warning("Please enter at least one category.")

    # Image Loading Section
    st.header("Load Images")
    base_path = "./"  # Set the base directory path directly
    categories_to_load = st.multiselect("Select categories to load:", os.listdir(base_path) if base_path else [])
    loaded_images, loaded_labels = load_images_from_directory(base_path, categories_to_load) if base_path and categories_to_load else ([], [])

    st.text(f"Total images loaded: {len(loaded_images)}")
    if loaded_images:
        st.image(loaded_images[0], caption="Sample Image", width=300)

    # EDA Section
    st.header("Exploratory Data Analysis (EDA)")
    if loaded_images:
        st.subheader("Data Distribution")
        label_series = pd.Series(loaded_labels)
        st.bar_chart(label_series.value_counts())

        st.subheader("Sample Images")
        num_samples = st.slider("Number of sample images to display:", min_value=1, max_value=min(10, len(loaded_images)), value=5)
        sampled_data = random.sample(list(zip(loaded_images, loaded_labels)), num_samples)

        sample_images = [Image.fromarray(img) if isinstance(img, np.ndarray) else img for img, _ in sampled_data]
        sample_labels = [label for _, label in sampled_data]

        for img in sample_images:
            img.format = "PNG"  # Ensure the format attribute is set

        st.image(sample_images, caption=sample_labels, width=100)
    else:
        st.write("No images loaded. Please load images in the 'Load Images' section.")

    # Model Training Section
    st.header("Model Training")
    if loaded_images:
        st.subheader("Training Settings")
        epochs = st.slider("Number of epochs:", min_value=1, max_value=100, value=10)
        batch_size = st.slider("Batch size:", min_value=1, max_value=100, value=32)

        if st.button("Train Model"):
            with st.spinner('Training in progress...'):
                # Ensure all images are RGB and have the same shape
                loaded_images_rgb = [Image.fromarray(img).convert("RGB") if isinstance(img, np.ndarray) else img.convert("RGB") for img in loaded_images]
                X = np.stack([np.array(img) for img in loaded_images_rgb], axis=0)

                # Convert labels to numerical format
                y = LabelEncoder().fit_transform(loaded_labels)
                y = to_categorical(y)

                # Split the data
                X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)

                # Define a simple CNN model
                model = models.Sequential([
                    layers.Conv2D(32, (3, 3), activation='relu', input_shape=(224, 224, 3)),
                    layers.MaxPooling2D((2, 2)),
                    layers.Conv2D(64, (3, 3), activation='relu'),
                    layers.MaxPooling2D((2, 2)),
                    layers.Conv2D(64, (3, 3), activation='relu'),
                    layers.Flatten(),
                    layers.Dense(64, activation='relu'),
                    layers.Dense(y.shape[1], activation='softmax')
                ])
                model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

                # Train the model
                history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, validation_data=(X_val, y_val))

                # Display training results
                st.subheader("Training Results")
                st.line_chart(pd.DataFrame(history.history))
    else:
        st.write("No images loaded. Please load images in the 'Load Images' section.")

if __name__ == "__main__":
    main()


"""Perform search of dataset on IP2."""
from cravattdb.contrib.ip2 import IP2
import config.config as config
import pathlib
import json
import time

SEARCH_PARAMS_PATH = pathlib.Path(config.SEARCH_PARAMS_PATH)

class Search:
    """Perform search of dataset on IP2."""

    def __init__(self, name):
        """Initialize Search with dataset name."""
        self.name = name

    def login(self, username, password):
        """Login to IP2 and keep reference to session."""
        self._ip2 = IP2(
            ip2_url=config.IP2_URL,
            username=username,
            default_project_name=config.PROJECT_NAME
        )
        return self._ip2.login(password)

    def search(self, organism, experiment_type, file_paths):
        """Initiate search on IP2."""
        params = self._get_params(experiment_type)
        # get database by file name
        database = self._ip2.get_database(self._get_database_path(organism)['name'])

        (experiment, job) = self._ip2.search(
            name=self.name,
            file_paths=file_paths,
            search_options={
                'params': params,
                'database': database
            }
        )

        link = self._check_search_status(job, experiment)

        return link

    def _get_params(self, experiment_type):
        with SEARCH_PARAMS_PATH.joinpath('search_params.json').open() as f:
            params_map = json.loads(f.read())

        if experiment_type not in params_map:
            raise KeyError('Search params are not available for this experiment type')

        with SEARCH_PARAMS_PATH.joinpath(params_map[experiment_type]).open() as f:
            params = json.loads(f.read())

        return params

    def _get_database_path(self, organism):
        with SEARCH_PARAMS_PATH.joinpath('databases.json').open() as f:
            database_map = json.loads(f.read())

        if organism not in database_map:
            raise KeyError('There is no database set for this organism')

        return database_map[organism]

    def _check_search_status(self, job, experiment):
        polling_interval = 180

        # wait a bit before we poll for status
        time.sleep(polling_interval)

        while True:
            start = time.clock()

            try:
                job.status()
            except LookupError:
                # job was not found, the job is finished or something went
                # horribly wrong
                return experiment.get_dtaselect()

            work_duration = time.clock() - start
            time.sleep(polling_interval - work_duration)


from telegram import ParseMode, Update
from telegram.ext import CallbackContext
from users.models import Group, User
from colt.models import Game, Section
from .messages import player_list_message
from .statics import (
    ADMIN_ONLY,
    GROUP_ONLY,
    HAS_GAME_ONLY,
    HAS_NOT_GAME_ONLY,
    SPECIFIC_STATUS_ONLY,
)

def checking_all(
    only_admin: bool = None,
        has_game: bool = None,
        game_status: list[Game.GameStatus] = None,
        update_players: bool = False,
) -> None:
    def decorator(func):
        def wrapper(update: Update, context: CallbackContext) -> None:
            group = Group.get_if_is_group(update, context)
            game = None
            if group is None:
                update.message.reply_text(
                    text=GROUP_ONLY,
                )
                return

            user = User.get_user(update, context)

            if only_admin is True and not user.is_admin:
                update.message.reply_text(
                    text=ADMIN_ONLY
                )
                return

            if only_admin is False and user.is_admin:
                update.message.reply_text(
                    text="Permission Denied!"
                )
                return

            opened_games = Game.get_open_games(group)
            has_opened_game = opened_games.exists()
            if has_game is True and not has_opened_game:
                update.message.reply_text(
                    text=HAS_GAME_ONLY
                )
                return

            if has_game is False and has_opened_game:
                update.message.reply_text(
                    text=HAS_NOT_GAME_ONLY,
                )
                return

            if game_status is not None and not has_opened_game:
                update.message.reply_text(
                    text=HAS_GAME_ONLY
                )
                return

            game: Game = opened_games.first()
            if game_status is not None:
                if game.status not in game_status:
                    update.message.reply_text(
                        text=SPECIFIC_STATUS_ONLY
                    )
                    return

            func(update, context, user, group, game=game)

            if game is not None and update_players == True:
                context.bot.send_message(
                    text=player_list_message(
                        game
                    ),
                    parse_mode="Markdown",
                    chat_id=group.group_id
                )

        return wrapper

    return decorator


import re
from pathlib import Path
from typing import Final

import httpx
from pydantic import BaseSettings, SecretStr

class Settings(BaseSettings):
    github_repository: str
    github_ref: str
    github_base_ref: str
    input_github_token: SecretStr

settings: Final[Settings] = Settings()
pyproject_toml: Final[Path] = Path("./pyproject.toml")

version_regex: Final[str] = r"\"(?P<version>[0-9]+\.[0-9]+\.[0-9]+)\""

def get_pyproject_toml_version() -> str:
    content = pyproject_toml.read_text()
    match = re.search(version_regex, content)
    if match is None:
        raise ValueError("Could not find version in pyproject.toml")
    return match.group("version")

def get_master_version() -> str:
    response = httpx.get(f"https://raw.githubusercontent.com/{settings.github_repository}/master/pyproject.toml")
    response.raise_for_status()
    content = response.text
    match = re.search(version_regex, content)
    if match is None:
        raise ValueError(f"[{settings.github_ref}] Could not find version in what_to_eat/__init__.py")
    return match.group("version")

def should_compare_versions() -> bool:
    return settings.github_base_ref == "master"

def is_updated(old: str, new: str) -> bool:
    new = new.split(".")
    old = old.split(".")
    for i in range(3):
        if int(new[i]) > int(old[i]):
            return True
        elif int(new[i]) < int(old[i]):
            return False
    return False

if __name__ == "__main__":
    if should_compare_versions():
        pyproject_toml_version = get_pyproject_toml_version()
        master_version = get_master_version()

        if not is_updated(master_version, pyproject_toml_version):
            raise ValueError(
                f"Version in pyproject.toml ({pyproject_toml_version}) "
                f"must be greater than version in master ({master_version})"
            )


"""
Execute a job on the targeted minions by using a moving window of fixed size `batch`.
"""

import gc

# pylint: enable=import-error,no-name-in-module,redefined-builtin
import logging

import salt.client
import salt.ext.tornado
from salt.cli.batch import batch_get_eauth, batch_get_opts, get_bnum

log = logging.getLogger(__name__)

class BatchAsync:
    """
    Run a job on the targeted minions by using a moving window of fixed size `batch`.

    ``BatchAsync`` is used to execute a job on the targeted minions by keeping
    the number of concurrent running minions to the size of `batch` parameter.

    The control parameters are:
        - batch: number/percentage of concurrent running minions
        - batch_delay: minimum wait time between batches
        - batch_presence_ping_timeout: time to wait for presence pings before starting the batch
        - gather_job_timeout: `find_job` timeout
        - timeout: time to wait before firing a `find_job`

    When the batch stars, a `start` event is fired:
         - tag: salt/batch/<batch-jid>/start
         - data: {
             "available_minions": self.minions,
             "down_minions": targeted_minions - presence_ping_minions
           }

    When the batch ends, an `done` event is fired:
        - tag: salt/batch/<batch-jid>/done
        - data: {
             "available_minions": self.minions,
             "down_minions": targeted_minions - presence_ping_minions
             "done_minions": self.done_minions,
             "timedout_minions": self.timedout_minions
         }
    """

    def __init__(self, parent_opts, jid_gen, clear_load):
        ioloop = salt.ext.tornado.ioloop.IOLoop.current()
        self.local = salt.client.get_local_client(
            parent_opts["conf_file"], io_loop=ioloop
        )
        if "gather_job_timeout" in clear_load["kwargs"]:
            clear_load["gather_job_timeout"] = clear_load["kwargs"].pop(
                "gather_job_timeout"
            )
        else:
            clear_load["gather_job_timeout"] = self.local.opts["gather_job_timeout"]
        self.batch_presence_ping_timeout = clear_load["kwargs"].get(
            "batch_presence_ping_timeout", None
        )
        self.batch_delay = clear_load["kwargs"].get("batch_delay", 1)
        self.opts = batch_get_opts(
            clear_load.pop("tgt"),
            clear_load.pop("fun"),
            clear_load["kwargs"].pop("batch"),
            self.local.opts,
            **clear_load
        )
        self.eauth = batch_get_eauth(clear_load["kwargs"])
        self.metadata = clear_load["kwargs"].get("metadata", {})
        self.minions = set()
        self.targeted_minions = set()
        self.timedout_minions = set()
        self.done_minions = set()
        self.active = set()
        self.initialized = False
        self.jid_gen = jid_gen
        self.ping_jid = jid_gen()
        self.batch_jid = jid_gen()
        self.find_job_jid = jid_gen()
        self.find_job_returned = set()
        self.ended = False
        self.event = salt.utils.event.get_event(
            "master",
            self.opts["sock_dir"],
            self.opts["transport"],
            opts=self.opts,
            listen=True,
            io_loop=ioloop,
            keep_loop=True,
        )
        self.scheduled = False
        self.patterns = set()

    def __set_event_handler(self):
        ping_return_pattern = "salt/job/{}/ret/*".format(self.ping_jid)
        batch_return_pattern = "salt/job/{}/ret/*".format(self.batch_jid)
        self.event.subscribe(ping_return_pattern, match_type="glob")
        self.event.subscribe(batch_return_pattern, match_type="glob")
        self.patterns = {
            (ping_return_pattern, "ping_return"),
            (batch_return_pattern, "batch_run"),
        }
        self.event.set_event_handler(self.__event_handler)

    def __event_handler(self, raw):
        if not self.event:
            return
        try:
            mtag, data = self.event.unpack(raw)
            for (pattern, op) in self.patterns:
                if mtag.startswith(pattern[:-1]):
                    minion = data["id"]
                    if op == "ping_return":
                        self.minions.add(minion)
                        if self.targeted_minions == self.minions:
                            self.event.io_loop.spawn_callback(self.start_batch)
                    elif op == "find_job_return":
                        if data.get("return", None):
                            self.find_job_returned.add(minion)
                    elif op == "batch_run":
                        if minion in self.active:
                            self.active.remove(minion)
                            self.done_minions.add(minion)
                            self.event.io_loop.spawn_callback(self.schedule_next)
        except Exception as ex:
            log.error("Exception occured while processing event: {}".format(ex))

    def _get_next(self):
        to_run = (
            self.minions.difference(self.done_minions)
            .difference(self.active)
            .difference(self.timedout_minions)
        )
        next_batch_size = min(
            len(to_run),  # partial batch (all left)
            self.batch_size - len(self.active),  # full batch or available slots
        )
        return set(list(to_run)[:next_batch_size])

    def check_find_job(self, batch_minions, jid):
        if self.event:
            find_job_return_pattern = "salt/job/{}/ret/*".format(jid)
            self.event.unsubscribe(find_job_return_pattern, match_type="glob")
            self.patterns.remove((find_job_return_pattern, "find_job_return"))

            timedout_minions = batch_minions.difference(
                self.find_job_returned
            ).difference(self.done_minions)
            self.timedout_minions = self.timedout_minions.union(timedout_minions)
            self.active = self.active.difference(self.timedout_minions)
            running = batch_minions.difference(self.done_minions).difference(
                self.timedout_minions
            )

            if timedout_minions:
                self.schedule_next()

            if self.event and running:
                self.find_job_returned = self.find_job_returned.difference(running)
                self.event.io_loop.spawn_callback(self.find_job, running)

    @salt.ext.tornado.gen.coroutine
    def find_job(self, minions):
        if self.event:
            not_done = minions.difference(self.done_minions).difference(
                self.timedout_minions
            )
            try:
                if not_done:
                    jid = self.jid_gen()
                    find_job_return_pattern = "salt/job/{}/ret/*".format(jid)
                    self.patterns.add((find_job_return_pattern, "find_job_return"))
                    self.event.subscribe(find_job_return_pattern, match_type="glob")
                    ret = yield self.local.run_job_async(
                        not_done,
                        "saltutil.find_job",
                        [self.batch_jid],
                        "list",
                        gather_job_timeout=self.opts["gather_job_timeout"],
                        jid=jid,
                        **self.eauth
                    )
                    yield salt.ext.tornado.gen.sleep(self.opts["gather_job_timeout"])
                    if self.event:
                        self.event.io_loop.spawn_callback(
                            self.check_find_job, not_done, jid
                        )
            except Exception as ex:
                log.error(
                    "Exception occured handling batch async: {}. Aborting execution.".format(
                        ex
                    )
                )
                self.close_safe()

    @salt.ext.tornado.gen.coroutine
    def start(self):
        if self.event:
            self.__set_event_handler()
            ping_return = yield self.local.run_job_async(
                self.opts["tgt"],
                "test.ping",
                [],
                self.opts.get(
                    "selected_target_option", self.opts.get("tgt_type", "glob")
                ),
                gather_job_timeout=self.opts["gather_job_timeout"],
                jid=self.ping_jid,
                metadata=self.metadata,
                **self.eauth
            )
            self.targeted_minions = set(ping_return["minions"])
            # start batching even if not all minions respond to ping
            yield salt.ext.tornado.gen.sleep(
                self.batch_presence_ping_timeout or self.opts["gather_job_timeout"]
            )
            if self.event:
                self.event.io_loop.spawn_callback(self.start_batch)

    @salt.ext.tornado.gen.coroutine
    def start_batch(self):
        if not self.initialized:
            self.batch_size = get_bnum(self.opts, self.minions, True)
            self.initialized = True
            data = {
                "available_minions": self.minions,
                "down_minions": self.targeted_minions.difference(self.minions),
                "metadata": self.metadata,
            }
            ret = self.event.fire_event(
                data, "salt/batch/{}/start".format(self.batch_jid)
            )
            if self.event:
                self.event.io_loop.spawn_callback(self.run_next)

    @salt.ext.tornado.gen.coroutine
    def end_batch(self):
        left = self.minions.symmetric_difference(
            self.done_minions.union(self.timedout_minions)
        )
        if not left and not self.ended:
            self.ended = True
            data = {
                "available_minions": self.minions,
                "down_minions": self.targeted_minions.difference(self.minions),
                "done_minions": self.done_minions,
                "timedout_minions": self.timedout_minions,
                "metadata": self.metadata,
            }
            self.event.fire_event(data, "salt/batch/{}/done".format(self.batch_jid))

            # release to the IOLoop to allow the event to be published
            # before closing batch async execution
            yield salt.ext.tornado.gen.sleep(1)
            self.close_safe()

    def close_safe(self):
        for (pattern, label) in self.patterns:
            self.event.unsubscribe(pattern, match_type="glob")
        self.event.remove_event_handler(self.__event_handler)
        self.event = None
        self.local = None
        self.ioloop = None
        del self
        gc.collect()

    @salt.ext.tornado.gen.coroutine
    def schedule_next(self):
        if not self.scheduled:
            self.scheduled = True
            # call later so that we maybe gather more returns
            yield salt.ext.tornado.gen.sleep(self.batch_delay)
            if self.event:
                self.event.io_loop.spawn_callback(self.run_next)

    @salt.ext.tornado.gen.coroutine
    def run_next(self):
        self.scheduled = False
        next_batch = self._get_next()
        if next_batch:
            self.active = self.active.union(next_batch)
            try:
                ret = yield self.local.run_job_async(
                    next_batch,
                    self.opts["fun"],
                    self.opts["arg"],
                    "list",
                    raw=self.opts.get("raw", False),
                    ret=self.opts.get("return", ""),
                    gather_job_timeout=self.opts["gather_job_timeout"],
                    jid=self.batch_jid,
                    metadata=self.metadata,
                )

                yield salt.ext.tornado.gen.sleep(self.opts["timeout"])

                # The batch can be done already at this point, which means no self.event
                if self.event:
                    self.event.io_loop.spawn_callback(self.find_job, set(next_batch))
            except Exception as ex:
                log.error("Error in scheduling next batch: %s. Aborting execution", ex)
                self.active = self.active.difference(next_batch)
                self.close_safe()
        else:
            yield self.end_batch()
        gc.collect()

    def __del__(self):
        self.local = None
        self.event = None
        self.ioloop = None
        gc.collect()


from flask import Flask, render_template,request

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

print("i am learning GIT")
@app.route('/data',methods = ['GET','POST'])
def data():
    user_data = request.form
    print(user_data)

    #return  "Data Captured"
if __name__ == '__main__':
     app.run(debug=True)


# pylint: disable=protected-access, C0413, W0611, C0301
import os
import sys
from pathlib import Path
import unittest
import PIL.Image
sys.path.insert(0, os.getcwd())
from video_and_photo_tools.image_cache import ImageCache
import video_and_photo_tools.pil_image_extensions

class TestImageCache(unittest.TestCase):

    TEST_IMAGE_PATH = Path('.') / 'data/test_data/test_images/photo.jpg'
    IMAGE_CACHE_DIR_PATH = Path('.') / 'data/test_data/test_cache'

    def setUp(self):
        self.image_cache = ImageCache(cache_dir_path=TestImageCache.IMAGE_CACHE_DIR_PATH)
        self.image = PIL.Image.open(TestImageCache.TEST_IMAGE_PATH)
        self.width = 640
        self.image_rgb_640 = self.image.resize_with_width_preserve_aspect(self.width)
        self.image_l_640 = self.image_rgb_640.convert('L')
        self.video_id = 'test_video_id'
        self.frame_num = 500

    def test_setup(self):
        self.assertEqual(self.image_l_640.mode, 'L')
        self.assertEqual(self.image_rgb_640.mode, 'RGB')

    def test_destroy_cache_dir(self):
        self.image_cache._ensure_cache_dir()
        for suffix in ImageCache.VALID_SUFFIXES:
            (TestImageCache.IMAGE_CACHE_DIR_PATH / ('foo.' + suffix)).touch()

        self.assertTrue(os.path.isdir(TestImageCache.IMAGE_CACHE_DIR_PATH))

        for suffix in ImageCache.VALID_SUFFIXES:
            self.assertTrue(os.path.isfile(TestImageCache.IMAGE_CACHE_DIR_PATH / ('foo.' + suffix)))

        self.image_cache.destroy_cache_dir()
        self.assertFalse(os.path.isdir(TestImageCache.IMAGE_CACHE_DIR_PATH))

    def test_store(self):
        self.store_images()

    def store_images(self):
        self.image_cache.clear_cache()
        for suffix in ImageCache.VALID_SUFFIXES:
            self.assertEqual(0, len(list(TestImageCache.IMAGE_CACHE_DIR_PATH.glob('*.' + suffix))))

        for suffix in ImageCache.VALID_SUFFIXES:
            for image in [self.image_rgb_640, self.image_l_640]:
                self.image_cache.store(self.video_id, image, self.frame_num, suffix)
            self.assertEqual(2, len(list(TestImageCache.IMAGE_CACHE_DIR_PATH.glob('*.' + suffix))))

    def test_fetch(self):
        self.image_cache.clear_cache()

        for image_format in ImageCache.VALID_SUFFIXES:
            for mode in ['RGB', 'L']:
                fetched = self.image_cache.fetch(self.video_id,
                                                 self.width,
                                                 mode,
                                                 self.frame_num,
                                                 image_format)
                self.assertIsNone(fetched)

        self.store_images()

        for image_format in ImageCache.VALID_SUFFIXES:
            for mode in ['RGB', 'L']:
                fetched = self.image_cache.fetch(self.video_id,
                                                 self.width,
                                                 mode,
                                                 self.frame_num,
                                                 image_format)
                self.assertIsNotNone(fetched)

if __name__ == '__main__':
    unittest.main()


"""Módulo com todas as funções para a tela Alarmes"""
#######################################################################################################
# Importações
#######################################################################################################
from datetime import datetime

from PyQt5.QtWidgets import QTableWidget, QTableWidgetItem
from PyQt5.QtCore import Qt

from ui_py.ui_gui_final import Ui_MainWindow
from dialogs.alarm import AlarmDialog
from utils.functions.alarm_control import *
#######################################################################################################
# Definição das variáveis globais
#######################################################################################################
UI: Ui_MainWindow
DIALOG: AlarmDialog
SHOW_ALARM_SCREEN = None
CURRENT_ROW = 0

hasAlarm: List[bool] = []
for i in range(0, 129):
    hasAlarm.append(False)
#######################################################################################################
# Funções de Definição
#######################################################################################################
def define_buttons(main_ui: Ui_MainWindow, AlarmDialog: AlarmDialog, show_alarm_screen):
    """
    Define os botões da tela

    :param main_ui: Ui principal da aplicação
    :param AlarmDialog: Dialog de novo alarme
    :param show_alarm_screen: Função que mostra o Dialog de novo alarme
    """
    global UI, DIALOG, SHOW_ALARM_SCREEN
    # define as variaveis globais
    UI = main_ui
    DIALOG = AlarmDialog
    SHOW_ALARM_SCREEN = show_alarm_screen

    # define os botões de cima/baixo
    UI.btn_sobe_alarm.clicked.connect(lambda: row_up(UI.alarm_list_widget))
    UI.btn_desce_alarm.clicked.connect(lambda: row_down(UI.alarm_list_widget))
    UI.btn_sobe_alarm_hist.clicked.connect(lambda: row_up(UI.hist_alarm_list_widget))
    UI.btn_desce_alarm_hist.clicked.connect(lambda: row_down(UI.hist_alarm_list_widget))
#######################################################################################################
# Funções de Controle
#######################################################################################################
def verify_alarms(alarm_time: str, alarm_id: int, alarm_active: bool):
    """
    Verifica se o alarme está ativo e habilita/desabilita ele na tela

    :param alarm_time: String do momento que o alarme aparece
    :param alarm_id: Número do alarme
    :param alarm_active: O sinal que vem do CLP
    """
    listWidget: QTableWidget = UI.alarm_list_widget

    if alarm_active and (not hasAlarm[alarm_id] or listWidget.rowCount() == 0):
        hasAlarm[alarm_id] = True
        define_new_alarm(alarm_time, alarm_id)
    elif not alarm_active and hasAlarm[alarm_id]:
        hasAlarm[alarm_id] = False
        delete_alarm_row(alarm_id)

#######################################################################################################
def define_new_alarm(alarm_time: str, alarm_id: int):
    """
    Adiciona um novo alarme na tela e na variável interna do módulo

    :param alarm_time: String do momento que o alarme aparece
    :param alarm_id: Número do alarme
    """

    listWidget = UI.alarm_list_widget
    histWidget = UI.hist_alarm_list_widget
    dialogsListWidget = DIALOG.ui.alarm_list_widget

    row_num = 0
    hist_row_num = 0
    alarm_msg: str = get_alarm_message(alarm_id)

    set_alarm_list(alarm_id, alarm_time, alarm_msg)

    time_item = QTableWidgetItem()
    msg_item = QTableWidgetItem()
    time_item.setText(str(alarm_time))
    msg_item.setText(alarm_msg)

    hist_time = QTableWidgetItem()
    hist_msg = QTableWidgetItem()
    hist_time.setText(str(alarm_time))
    hist_msg.setText(alarm_msg)

    dialog_time = QTableWidgetItem()
    dialog_msg = QTableWidgetItem()
    dialog_time.setText(str(alarm_time))
    dialog_msg.setText(alarm_msg)

    listWidget.insertRow(row_num)
    listWidget.setItem(row_num, 0, time_item)
    listWidget.setItem(row_num, 1, msg_item)

    histWidget.insertRow(hist_row_num)
    histWidget.setItem(hist_row_num, 0, hist_time)
    histWidget.setItem(hist_row_num, 1, hist_msg)

    dialogsListWidget.insertRow(0)
    dialogsListWidget.setItem(0, 0, dialog_time)
    dialogsListWidget.setItem(0, 1, dialog_msg)
    DIALOG.show_dialog(SHOW_ALARM_SCREEN)
#######################################################################################################
def delete_alarm_row(alarm_id: int):
    """
    Deleta um alarme da tela de Alarmes, mas não da tela Histórico de Alarmes

    :param alarm_id: Número do alarme
    """
    print("Deletando alarme")

    listWidget: QTableWidget = UI.alarm_list_widget

    # itemFound = listWidget.findItems(f"Alarme {alarm_id}:", Qt.MatchContains)
    # listWidget.setCurrentItem(itemFound[0])

    # row_num = itemFound[0].row()
    while listWidget.rowCount() > 0:
        listWidget.removeRow(0)

    delete_alarm_from_list(alarm_id)
#######################################################################################################
def row_down(listWidget: QTableWidget):
    """
    Chamado quando o botão para descer é pressionado

    :param listWidget: Widget da lista de alarmes
    """
    global CURRENT_ROW

    CURRENT_ROW = listWidget.currentRow()
    CURRENT_ROW += 1

    listWidget.setCurrentCell(CURRENT_ROW, 0)
#######################################################################################################
def row_up(listWidget: QTableWidget):
    """
    Chamado quando o botão para subir é pressionado

    :param listWidget: Widget da lista de alarmes
    """
    global CURRENT_ROW

    CURRENT_ROW = listWidget.currentRow()
    CURRENT_ROW -= 1

    listWidget.setCurrentCell(CURRENT_ROW, 0)
#######################################################################################################
# Funções de Atualização
#######################################################################################################
def UpdateAlarms(tag):
    """Recebe a tag do worker e manda para verificação"""
    try:
        alarms_count = 0
        for alarm in tag:
            time_now = str(datetime.now())
            verify_alarms(time_now[0:19], alarms_count, alarm[1])
            alarms_count += 1
    except Exception as e:
        print(f"{e} - {alarms_count} - screens/alarms.py - UpdateAlarms")
#######################################################################################################


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar  7 17:40:12 2017

@author: KimiaNadjahi
"""

import numpy as np
import timeit
import scipy.stats as stats

def rightmost_interval(U, Lambda):
    Z = 1
    X = np.exp(-0.5*Lambda)
    j = 0
    while 1:
        j += 1
        Z -= (j+1)**2 * X**((j+1)**2 - 1)
        if Z > U:
            return 1
        j += 1
        Z += (j+1)**2 * X**((j+1)**2 - 1)
        if Z < U:
            return 0

def leftmost_interval(U, Lambda):
    H = 0.5*np.log(2) + 2.5*np.log(np.pi) - 2.5*np.log(Lambda) - \
        np.pi**2 / (2*Lambda) + 0.5*Lambda
    logU = np.log(U)
    Z = 1
    X = np.exp((-np.pi**2) / (2*Lambda))
    K = Lambda / (np.pi**2)
    j = 0
    while 1:
        j += 1
        Z -= K*X**(j**2 - 1)
        logZ = np.log(Z) if Z > 0 else np.log(-Z)
        if H + np.log(Z) > logU:
            return 1
        j += 1
        Z += (j+1)**2 * X**((j+1)**2 - 1)
        logZ = np.log(Z) if Z > 0 else np.log(-Z)
        if H + np.log(Z) < logU:
            return 0

def mixing_weights_sampling(r2):
    """
    Drawing a sample of mixing weights using rejection sampling with GIG
    """
    r = np.sqrt(r2)
    OK = False
    while not OK:
        Y = np.random.normal()
        Y = Y*Y
        Y = 1 + (Y - np.sqrt(Y * (4*r + Y))) / (2*r)
        U = np.random.uniform()
        if U <= 1/(1+Y):
            Lambda = r/Y
        else:
            Lambda = r*Y
        U = np.random.uniform()
        if Lambda > 4/3:
            OK = rightmost_interval(U, Lambda)
        else:
            OK = leftmost_interval(U, Lambda)
    return Lambda

def auxiliary_gibbs(XX, t, v=100, max_iter=10000, burn_in=5000):
    """
    Auxiliary variable Gibbs sampler
    """
    print("--- Initialization...")
    N, D = XX.shape
    mix_weights = np.ones(N)
    beta_saved = np.zeros((max_iter-burn_in, D))
    # Truncated normal distribution
    t_ones = np.where(t == 1)[0]
    t_zeros = np.where(t == 0)[0]
    Z = np.zeros(N)
    mu, sigma = 0, 1
    for a in t_ones:
        lower, upper = 0, np.inf
        Z[a] = stats.truncnorm.rvs(
            (lower - mu) / sigma, (upper - mu) / sigma, loc=mu, scale=sigma)
    for a in t_zeros:
        lower, upper = -np.inf, 0
        Z[a] = stats.truncnorm.rvs(
            (lower - mu) / sigma, (upper - mu) / sigma, loc=mu, scale=sigma)
    print("--- Initialization: done. Iterating...")
    # Iterating
    for i in range(max_iter):
        if i % 100 == 0:
            print("Iteration %d" % i)
        # Timing after burn in
        if i == burn_in:
            start = timeit.default_timer()
        V = np.linalg.inv(XX.T.dot(np.diag(1/mix_weights)).dot(XX) + np.identity(D)*1/v)
        L = np.linalg.cholesky(V)
        S = V.dot(XX.T)
        B = S.dot(np.diag(1/mix_weights)).dot(Z)
        W = np.zeros(N)
        H = np.zeros(N)
        # Updating Z and B
        for j in range(N):
            z_old = Z[j]
            H[j] = XX[j, :].dot(S[:, j])
            W[j] = H[j] / (mix_weights[j] - H[j])
            m = XX[j].dot(B)
            m -= W[j] * (Z[j] - m)
            q = mix_weights[j] * (W[j] + 1)
            if t[j] == 1:
                lower, upper = 0, np.inf
                std = np.sqrt(q)
                Z[j] = stats.truncnorm.rvs(
                    (lower - m) / std, (upper - m) / std, loc=m, scale=std)
            else:
                lower, upper = -np.inf, 0
                std = np.sqrt(q)
                Z[j] = stats.truncnorm.rvs(
                    (lower - m) / std, (upper - m) / std, loc=m, scale=std)
            B += (Z[j] - z_old) / mix_weights[j] * S[:, j]
        # Drawing new values of beta
        T = np.random.multivariate_normal(np.zeros(D), np.identity(D))
        beta = B + L.dot(T)
        if i >= burn_in:
            beta_saved[i - burn_in] = beta
        # Sampling new mixing weights
        for j in range(N):
            r = Z[j] - XX[j, :].dot(beta)
            mix_weights[j] = mixing_weights_sampling(r**2)  # to time
    print("--- Iterating: done.")
    time = timeit.default_timer() - start
    print("--- Auxiliary Variable Gibbs Sampler finished in {}".format(time))
    return beta_saved, time


# import logging
import threading
import json
from env import api_key, api_key_testnet, api_secret, api_secret_testnet
from binance.spot import Spot as Client
from binance.lib.utils import config_logging
from binance.error import ClientError

# config_logging(logging, logging.DEBUG)

class Cancel(threading.Thread):
    def __init__(self, url, orderbook, symbol, clientOrderId):
        super().__init__()

        try:

            # with lock:
            key = api_key_testnet

            secret = api_secret_testnet

            url = f"https://{url}"

            client = Client(key, secret, base_url=url)

            response = client.cancel_order(
                symbol, origClientOrderId=clientOrderId)

            # orderbook[response['origClientOrderId']] = {
            #     "symbol": response["symbol"], "status": response['status'],
            # }
            orderbook[response['origClientOrderId']] = {"symbol": response["symbol"], "side": response["side"], "price": response["price"],
                                                        "quantity": response["origQty"], "orderId": response["orderId"], "type": response["type"],
                                                        "timeInForce": response["timeInForce"], "status": response['status']
                                                        }
            with open("websocket/project2/cancel_order.json", 'w', encoding='utf-8') as f:
                json.dump(orderbook, f, ensure_ascii=False, indent=2)
            # logging.info(response)
        except ClientError as error:
            # logging.error(
            #     "Found error. status: {}, error code: {}, error message: {}".format(
            #         error.status_code, error.error_code, error.error_message
            #     )
            # )
            print(
                "Found error. status: {}, error code: {}, error message: {}".format(
                    error.status_code, error.error_code, error.error_message
                )
            )
# {'symbol': 'BTCUSDT', 'origClientOrderId': 'trxXLSFk3ZCyv11RlrxgkV', 'orderId': 4359831, 'orderListId': -1, 'clientOrderId': '7TKHWRUCfuGck0BvqGXkut', 'price': '20450.12000000', 'origQty': '0.00212000', 'executedQty': '0.00000000', 'cummulativeQuoteQty': '0.00000000', 'status': 'CANCELED', 'timeInForce': 'GTC', 'type': 'LIMIT', 'side': 'SELL'}


from django.conf.urls import patterns, include, url
from django.contrib import admin
from catalyzer import settings

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'catalyzer.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    (r'^media/(?P<path>.*)$', 'django.views.static.serve',
     {'document_root': settings.MEDIA_ROOT}),

    url(r'^$', 'dicom.views.view_files', name="view_files"),
    url(r'^upload$', 'dicom.views.upload'),

    url(r'^getdata$', 'dicom.views.view_data_api'),
    url(r'^download$', 'dicom.views.download_zip'),

    url(r'^admin/', include(admin.site.urls)),

)


from django.conf import settings
from django.db import models
from django.utils.translation import ugettext_lazy as _
from model_utils import Choices
from model_utils.models import TimeStampedModel

from nc_auth import utils
from nc_core.utils import memcached_method

__all__ = [
    'Token'
]

class TokenManager(models.Manager):
    PREFIX = "AuthToken#"

    @memcached_method(
        key=lambda key: TokenManager.PREFIX + key,
        timeout=settings.CACHES_DURATIONS.get("TOKEN", 60)
    )
    def get_by_key(self, key):
        return self.select_related("user").get(key=key)

class Token(TimeStampedModel):
    PLATFORM_CHOICES = Choices(
        ('web', 'WEB', _('Web')),
        ('ios', 'IOS', _('iOS')),
        ('android', 'ANDROID', _('Android')),
    )

    key = models.CharField(
        verbose_name=_('key'),
        max_length=128,
        primary_key=True,
        default=utils.generate_string,
        editable=False
    )

    platform = models.SlugField(
        verbose_name=_('platform'),
        choices=PLATFORM_CHOICES,
        default=PLATFORM_CHOICES.WEB
    )

    user = models.ForeignKey(
        settings.AUTH_USER_MODEL,
        verbose_name=_('user'),
        related_name='tokens',
        default=None,
        blank=True,
        null=True
    )

    version = models.FloatField(
        verbose_name=_('version'),
        default=1.0
    )

    client = models.CharField(
        verbose_name=_('client'),
        max_length=1024,
        default=''
    )

    objects = TokenManager()

    def __str__(self):
        return "<Token {}:***{}>".format(self.user if self.user else "-", self.key[:4])

    class Meta:
        verbose_name = _('Auth token')
        verbose_name_plural = _('Auth tokens')


# Fine-tuning SentiLARE for sentence-level sentiment classification on SST, MR, IMDB, Yelp-2/5.
# The code is modified based on run_glue.py in pytorch-transformers

from __future__ import absolute_import, division, print_function

import argparse
import glob
import logging
import os
import random
import json
import fcntl

import numpy as np
import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)
from torch.utils.data.distributed import DistributedSampler
from tensorboardX import SummaryWriter
from tqdm import tqdm, trange

from pytorch_transformers import (WEIGHTS_NAME, RobertaTokenizer, RobertaConfig, AdamW, WarmupLinearSchedule)

from modeling_sentilr_roberta import RobertaForSequenceClassification

from sklearn.metrics import accuracy_score, recall_score, f1_score, confusion_matrix
from sent_data_utils_sentilr import convert_examples_to_features_roberta

from sent_data_utils_sentilr import sstProcessor, mrProcessor, imdbProcessor, yelp2Processor, yelp5Processor, iemocapProcessor, meldProcessor
from code.uniter3m.train_emo import clean_chekpoints, write_result_to_tsv

logger = logging.getLogger(__name__)

MODEL_CLASSES = {
    'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
}

processors = {
    "iemocap": iemocapProcessor,
    "meld": meldProcessor,
    "sst": sstProcessor,
    "imdb": imdbProcessor,
    "mr": mrProcessor,
    "yelp2": yelp2Processor,
    "yelp5": yelp5Processor,
}

# SentiLARE's input embeddings include POS embedding, word-level sentiment polarity embedding,
# and sentence-level sentiment polarity embedding (which is set to be unknown during fine-tuning).
is_pos_embedding = True
is_senti_embedding = True
is_polarity_embedding = True

def set_seed(args):
    # Set the random seed
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

def train(args, train_dataset, model, tokenizer):
    # Train the model
    if args.local_rank in [-1, 0]:
        tb_writer = SummaryWriter()

    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size, num_workers = 0)

    if args.max_steps > 0:
        t_total = args.max_steps
        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
    else:
        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs

    # Prepare optimizer and schedule (linear warmup and decay)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total)
    if args.fp16:
        try:
            from apex import amp
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)

    # Multi-gpu training (should be after apex fp16 initialization)
    if args.n_gpu > 1:
        model = torch.nn.DataParallel(model)

    # Distributed training (should be after apex fp16 initialization)
    if args.local_rank != -1:
        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
                                                          output_device=args.local_rank,
                                                          find_unused_parameters=True)

    # Train
    logger.info("***** Running training *****")
    logger.info("  Num examples = %d", len(train_dataset))
    logger.info("  Num Epochs = %d", args.num_train_epochs)
    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
                   args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1))
    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
    logger.info("  Total optimization steps = %d", t_total)

    global_step = 0
    global_epoch = 0
    best_eval_metrix = 0
    best_eval_epoch = 0
    steps2val_results = {}
    tr_loss, logging_loss = 0.0, 0.0
    model.zero_grad()
    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
    set_seed(args)  # Added here for reproducibility (even between python 2 and 3)
    for _ in train_iterator:
        global_epoch += 1
        global_epoch_loss = 0
        global_epoch_steps = 0
        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
        for step, batch in enumerate(epoch_iterator):
            model.train()
            batch = tuple(t.to(args.device) for t in batch)
            inputs = {'input_ids':      batch[0],
                      'attention_mask': batch[1],
                      'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None,  # RoBERTa doesn't use segment_ids
                      'labels':         batch[-1],
                      'pos_tag_ids':    batch[3],
                      'senti_word_ids': batch[4],
                      'polarity_ids':   batch[5]}
            outputs = model(**inputs)
            loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)

            if args.n_gpu > 1:
                loss = loss.mean() # mean() to average on multi-gpu parallel training
            if args.gradient_accumulation_steps > 1:
                loss = loss / args.gradient_accumulation_steps

            if args.fp16:
                with amp.scale_loss(loss, optimizer) as scaled_loss:
                    scaled_loss.backward()
                torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
            else:
                loss.backward()
                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

            global_epoch_loss += loss.mean().item()
            global_epoch_steps += 1
            if (step + 1) % args.gradient_accumulation_steps == 0:
                optimizer.step()
                scheduler.step()  # Update learning rate schedule torch> 1.1
                model.zero_grad()
                global_step += 1

        logger.info('current training epoch: {}'.format(global_epoch))
        logger.info('learning rate: ' + str(scheduler.get_lr()[0]))
        logger.info('training loss: {}'.format(global_epoch_loss/global_epoch_steps))
        # evaluating
        val_log = evaluate(args, model, tokenizer)
        logger.info(f"[Validation] Loss: {val_log['loss']:.2f},"
                                f"\t WA: {val_log['WA']*100:.2f},"
                                f"\t WF1: {val_log['WF1']*100:.2f},"
                                f"\t UA: {val_log['UA']*100:.2f},\n")

        if args.task_name == 'meld':
            select_metrix = 'WF1'
        elif args.task_name == 'msp':
            select_metrix = 'UA'
        elif args.task_name == 'iemocap':
            select_metrix = 'UA'
        else:
            select_metrix = 'WA'
        logger.info(f'task_name {args.task_name} and select_metrix {select_metrix}')
        steps2val_results[global_epoch] = val_log
        # update the current best model based on validation results
        if val_log[select_metrix] > best_eval_metrix:
            best_eval_epoch = global_epoch
            best_eval_metrix = val_log[select_metrix]
            patience = args.patience
            # Save model checkpoint
            output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_epoch))
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
            model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
            model_to_save.save_pretrained(output_dir)
            torch.save(args, os.path.join(output_dir, 'training_args.bin'))
            logger.info("Saving model checkpoint to %s", output_dir)
        else:
            if args.patience > 0 and patience <= 0:
                break

    # select the best result and write to file and remove the other models
    train_iterator.close()
    logger.info('Val: Best eval steps {} found with {} {}'.format(best_eval_epoch, select_metrix, best_eval_metrix))
    logger.info('Val: {}'.format(steps2val_results[best_eval_epoch]))
    steps2val_results['bestepoch'] = best_eval_epoch
    json.dump(steps2val_results, open(os.path.join(args.output_dir, 'step2val_reuslts.json'),'w',encoding='utf-8'))
    write_result_to_tsv(args.results_path, steps2val_results[best_eval_epoch], args.cvNo)
    # remove the others model
    clean_chekpoints(args.output_dir, best_eval_epoch)

    if args.local_rank in [-1, 0]:
        tb_writer.close()

def clean_chekpoints(ckpt_dir, store_epoch):
    # model_step_number.pt
    for checkpoint in os.listdir(ckpt_dir):
        if not checkpoint.startswith('checkpoint-{}'.format(store_epoch)):
            os.system('rm -r {}'.format(os.path.join(ckpt_dir, checkpoint)))

def write_result_to_tsv(file_path, tst_log, cvNo):
    # 1. 使用fcntl对文件加锁,避免多个不同进程同时操作同一个文件
    # 2. 如果不存在先创建一个 output_csv
    if not os.path.exists(file_path):
        print('[Debug] create the {}'.format(file_path))
        open(file_path, 'w').close()  # touch output_csv
    f_in = open(file_path)
    fcntl.flock(f_in.fileno(), fcntl.LOCK_EX) # 加锁
    content = f_in.readlines()
    if len(content) != 12:
        content += ['\n'] * (12-len(content))
    content[cvNo-1] = 'CV{}\t{:.4f}\t{:.4f}\t{:.4f}\n'.format(cvNo, tst_log['WA'], tst_log['WF1'], tst_log['UA'])
    f_out = open(file_path, 'w')
    f_out.writelines(content)
    f_out.close()
    f_in.close()                              # 释放锁

def evaluate(args, model, tokenizer, prefix=""):
    # Do validation
    eval_task = args.task_name
    eval_output_dir = args.output_dir

    # Prepare the validation dataset
    eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)

    if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
        os.makedirs(eval_output_dir)

    args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
    # Note that DistributedSampler samples randomly
    eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size, num_workers = 0)

    # Eval!
    logger.info("***** Running evaluation {} *****".format(prefix))
    logger.info("  Num examples = %d", len(eval_dataset))
    logger.info("  Batch size = %d", args.eval_batch_size)
    eval_loss = 0.0
    nb_eval_steps = 0
    preds = None
    out_label_ids = None

    model.eval()
    for batch in tqdm(eval_dataloader, desc="Evaluating"):
        batch = tuple(t.to(args.device) for t in batch)
        with torch.no_grad():
            inputs = {'input_ids':      batch[0],
                        'attention_mask': batch[1],
                        'token_type_ids': batch[2] if args.model_type in ['bert', 'xlnet'] else None,  # RoBERTa doesn't use segment_ids
                        'pos_tag_ids': batch[3],
                        'senti_word_ids': batch[4],
                        'polarity_ids': batch[5],
                        'labels':         batch[-1]}
            outputs = model(**inputs)
            tmp_eval_loss, logits = outputs[:2]
            eval_loss += tmp_eval_loss.mean().item()
        nb_eval_steps += 1
        if preds is None:
            preds = logits.detach().cpu().numpy()
            out_label_ids = inputs['labels'].detach().cpu().numpy()
        else:
            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
            out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)

    eval_loss = eval_loss / nb_eval_steps
    preds = np.argmax(preds, axis=1)

    acc = accuracy_score(out_label_ids, preds)
    uar = recall_score(out_label_ids, preds, average='macro')
    wf1 = f1_score(out_label_ids, preds, average='weighted')
    f1 = f1_score(out_label_ids, preds, average='macro')
    cm = confusion_matrix(out_label_ids, preds)
    model.train()
    return {'loss': eval_loss, 'WA': acc, 'WF1': wf1, 'UA': uar,  'F1': f1}

def load_and_cache_examples(args, task, tokenizer, evaluate=False, test = False):
    if args.local_rank not in [-1, 0] and not evaluate:
        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache

    # Select the data processor according to the task name
    processor = processors[task]()
    output_mode = "classification"
    # Load data features from cache or dataset file
    cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}'.format(
        'dev' if evaluate else 'test' if test else 'train',
        list(filter(None, args.model_name_or_path.split('/'))).pop(),
        str(args.max_seq_length),
        str(task)))
    if os.path.exists(cached_features_file):
        logger.info("Loading features from cached file %s", cached_features_file)
        features = torch.load(cached_features_file)
    else:
        logger.info("Creating features from dataset file at %s", args.data_dir)
        label_list = processor.get_labels()
        examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_test_examples(args.data_dir) if test else processor.get_train_examples(args.data_dir)
        features = convert_examples_to_features_roberta(examples, label_list, args.max_seq_length, tokenizer, output_mode,
             cls_token_at_end=bool(args.model_type in ['xlnet']),            # xlnet has a cls token at the end
             cls_token=tokenizer.cls_token,
             cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0,
             sep_token=tokenizer.sep_token,
             sep_token_extra=bool(args.model_type in ['roberta']),           # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
             pad_on_left=bool(args.model_type in ['xlnet']),                 # pad on the left for xlnet
             pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
             pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0,
             mode = args.task_name)

        if args.local_rank in [-1, 0]:
            logger.info("Saving features into cached file %s", cached_features_file)
            torch.save(features, cached_features_file)

    if args.local_rank == 0 and not evaluate:
        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache

    # Convert to Tensors and build dataset
    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)
    all_pos_ids = torch.tensor([f.pos_ids for f in features], dtype=torch.long)
    all_senti_ids = torch.tensor([f.senti_ids for f in features], dtype=torch.long)
    all_polarity_ids = torch.tensor([f.polarity_ids for f in features], dtype=torch.long)

    all_label_ids = torch.tensor([f.label_id for f in features], dtype=torch.long)
    dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_pos_ids, all_senti_ids, all_polarity_ids, all_label_ids)
    return dataset

def main():
    parser = argparse.ArgumentParser()

    ## Required parameters
    parser.add_argument("--data_dir", default=None, type=str, required=True,
                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
    parser.add_argument("--model_type", default=None, type=str, required=True,
                        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
                        help="Path to pre-trained model or shortcut name")
    parser.add_argument("--task_name", default=None, type=str, required=True,
                        help="The name of the task to train selected in the list")
    parser.add_argument("--output_dir", default=None, type=str, required=True,
                        help="The output directory where the model predictions and checkpoints will be written.")
    parser.add_argument("--results_path", default=None, type=str, required=True,
                        help="The output results where the model predictions and checkpoints will be written.")
    parser.add_argument("--cvNo", default=1, type=int, required=True,
                        help="which cv.")

    ## Other parameters
    parser.add_argument("--config_name", default="", type=str,
                        help="Pretrained config name or path if not the same as model_name")
    parser.add_argument("--tokenizer_name", default="", type=str,
                        help="Pretrained tokenizer name or path if not the same as model_name")
    parser.add_argument("--cache_dir", default="", type=str,
                        help="Where do you want to store the pre-trained models downloaded from s3")
    parser.add_argument("--max_seq_length", default=128, type=int,
                        help="The maximum total input sequence length after tokenization. Sequences longer "
                             "than this will be truncated, sequences shorter will be padded.")
    parser.add_argument("--do_train", action='store_true',
                        help="Whether to run training.")
    parser.add_argument("--do_eval", action='store_true',
                        help="Whether to run eval on the dev set.")
    parser.add_argument("--evaluate_during_training", action='store_true',
                        help="Rul evaluation during training at each logging step.")
    parser.add_argument("--do_lower_case", action='store_true',
                        help="Set this flag if you are using an uncased model.")

    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int,
                        help="Batch size per GPU/CPU for training.")
    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int,
                        help="Batch size per GPU/CPU for evaluation.")
    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
                        help="Number of updates steps to accumulate before performing a backward/update pass.")
    parser.add_argument("--learning_rate", default=5e-5, type=float,
                        help="The initial learning rate for Adam.")
    parser.add_argument("--weight_decay", default=0.0, type=float,
                        help="Weight deay if we apply some.")
    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
                        help="Epsilon for Adam optimizer.")
    parser.add_argument("--max_grad_norm", default=1.0, type=float,
                        help="Max gradient norm.")
    parser.add_argument("--num_train_epochs", default=3.0, type=float,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--max_steps", default=-1, type=int,
                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
    parser.add_argument("--warmup_steps", default=0, type=int,
                        help="Linear warmup over warmup_steps.")
    parser.add_argument("--patience", default=2, type=int,
                        help="early stop.")

    parser.add_argument('--logging_steps', type=int, default=50,
                        help="Log every X updates steps.")
    parser.add_argument('--save_steps', type=int, default=50,
                        help="Save checkpoint every X updates steps.")
    parser.add_argument("--eval_all_checkpoints", action='store_true',
                        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
    parser.add_argument("--no_cuda", action='store_true',
                        help="Avoid using CUDA when available")
    parser.add_argument('--overwrite_output_dir', action='store_true',
                        help="Overwrite the content of the output directory")
    parser.add_argument('--overwrite_cache', action='store_true',
                        help="Overwrite the cached training and evaluation sets")
    parser.add_argument('--seed', type=int, default=42,
                        help="random seed for initialization")

    parser.add_argument('--fp16', action='store_true',
                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
    parser.add_argument('--fp16_opt_level', type=str, default='O1',
                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
                             "See details at https://nvidia.github.io/apex/amp.html")
    parser.add_argument("--local_rank", type=int, default=-1,
                        help="For distributed training: local_rank")
    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")
    args = parser.parse_args()

    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
        raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))

    # Setup distant debugging if needed
    if args.server_ip and args.server_port:
        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
        import ptvsd
        print("Waiting for debugger attach")
        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
        ptvsd.wait_for_attach()

    # Setup CUDA, GPU & distributed training
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        args.n_gpu = torch.cuda.device_count()
    else:  # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        torch.distributed.init_process_group(backend='nccl')
        args.n_gpu = 1
    args.device = device

    # Setup logging
    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt = '%m/%d/%Y %H:%M:%S',
                        level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
    logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
                    args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)

    # Set seed
    set_seed(args)

    # Prepare task
    args.task_name = args.task_name.lower()
    processor = processors[args.task_name]()
    args.output_mode = "classification"
    label_list = processor.get_labels()
    num_labels = len(label_list)

    # Load pretrained model and tokenizer
    if args.local_rank not in [-1, 0]:
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

    args.model_type = args.model_type.lower()
    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name)
    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case)
    model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config, pos_tag_embedding=is_pos_embedding, senti_embedding=is_senti_embedding, polarity_embedding=is_polarity_embedding)

    if args.local_rank == 0:
        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

    model.to(args.device)
    logger.info("Training/evaluation parameters %s", args)
    train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
    train(args, train_dataset, model, tokenizer)

if __name__ == "__main__":
    main()


import logging
import os
import shutil
import requests
from threading import Thread

class UpdateServer(Thread):
    def __init__(self, config, mc_version, latest):
        # simply read the config setup by the user (or the default values)
        Thread.__init__(self)
        self.mc_version = mc_version
        self.latest = latest
        self.install_location = config[0]
        self.paper_api_url = 'https://papermc.io/api/v1/paper'

    def run(self):
        try:
            # use the previous api calls to get the latest version of paper for MC version
            url = 'https://papermc.io/api/v1/paper/{MCVERSION}/{latest}/download'.format(MCVERSION=self.mc_version,
                                                                                latest=self.latest)
            logging.debug('UPDATER: Downloading new Paper jar file...')
            download = requests.get(url)

            # make a temporary directory to download the new jar into
            if os.path.exists('./temp'):
                pass
            else:
                os.makedirs('./temp', 0o777)

            # download the new jar into the temp directory
            with open('./temp/paper.jar', 'wb') as f:
                f.write(download.content)
            logging.debug('UPDATER: Successfully Downloaded PaperMC jar file...')

            # remove old jar from the primary directory
            os.remove('./paper_server/paper.jar')
            logging.debug('UPDATER: Moving file in place...')
            # move new jar into primary directory
            shutil.move('./temp/paper.jar', '{INSTALL_LOC}/paper.jar'.format(INSTALL_LOC=self.install_location))
            logging.debug('UPDATER: Cleaning up...')
            os.rmdir('./temp')  # removes the temp dir

        except Exception as e:
            print(e)