#!/usr/bin/env python3
import serial
import time
import sys
import yaml

# https://ptelectronics.ru/wp-content/uploads/HF-LPT120-User-Manual-V1.1.120160104.pdf
# https://vestnikmag.ru/osobennosti-primeneniya-wi-fi-modulej-hf-lpb100-hf-lpt100-hf-lpt120/

MAC = "00:00:00:00:00:00"
UID_NULL = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00"
UID = UID_NULL
URL = "socket://dongle.rusklimat.ru:10001?logging=debug"

if len(sys.argv) > 1:
    MAC = sys.argv[1]
else:
    with open("secrets.yaml", "r") as stream:
        try:
            yml = yaml.safe_load(stream)
            mac = yml["device_mac2"]
            if mac:
                MAC = mac
            uid = yml["device_uid2"]
            if uid:
                UID = uid
        except yaml.YAMLError as exc:
            pass
        except KeyError as exc:
            pass

print(f"Using MAC address: {MAC}")

SHOW_AT = False
SHOW_AA = 1

CMD_01_REQ_DEV_TYPE = 0x01
CMD_06_REQ_SAVE_DATA = 0x06
CMD_05_RSP_ERROR = 0x05
CMD_07_REQ_LOAD_DATA = 0x07
CMD_08_REQ_STATE = 0x08
CMD_88_RSP_STATE = 0x88
CMD_0A_REQ_SET = 0x0A

OPER_00_STATE = 0x00
OPER_01_CLOCK = 0x01
OPER_02_TIMER = 0x02
OPER_03_BST = 0x03

MODE_00_OFF = 0
MODE_01_700W = 1
MODE_02_1300W = 2
MODE_03_2000W = 3
MODE_04_NO_FROST = 4

STATE_00_OFF = 0
STATE_01_700W = 1
STATE_02_1300W = 2
STATE_03_2000W = 3
STATE_04_TIMER = 4
STATE_05_NO_FROST = 5

FS_ST = "st"  # state
FS_CT = "ct"  # current_temperature
FS_TT = "tt"  # target_temperature
FS_CH = "ch"  # clock_hours
FS_CM = "cm"  # clock_minutes
FS_TH = "th"  # timer_hours
FS_TM = "tm"  # timer_minutes
FS_UNK = "unk"  # unknown
FS_BST = "bst"  # bst

DEV_STATE = {
    FS_ST: 1,
    FS_CT: 38,
    FS_TT: 62,
    FS_CH: 19,
    FS_CM: 10,
    FS_TH: 0,
    FS_TM: 0,
    FS_UNK: 0,
    FS_BST: 0,
}

if MAC == "00:00:00:00:00:00":
    URL = "loop://?logging=info"
    print("Startnig test mode")
DEBUG = URL.startswith("loop://")
ser = serial.serial_for_url(URL)

def write_at(buf: str):
    """Write AT commant to UART"""
    msg = buf.encode("utf-8") + b"\r\n"
    if SHOW_AT:
        print(f"w >>> {msg}")
    if not DEBUG:
        time.sleep(1)
    ser.write(msg)

def write_aa(buf: str):
    """Write AA commant to UART"""
    buf = bytes.fromhex(buf.replace(" ", "").replace(".", ""))

    if SHOW_AA > 0:
        bbb = buf[2:-1]
        cmd = bbb[0]
        if cmd != 0x88 and cmd != 0x87 and cmd != 0x81:
            bbb = bbb[1:]
            print(f"w >>> {cmd:02X}: {bbb.hex('.')} ({len(bbb)})")

    if SHOW_AA > 1:
        print(f"w >>> {buf.hex('.')} ({len(buf)})")

    if not DEBUG:
        time.sleep(1)
    ser.write(buf)

def calc_crc(data: bytes) -> int:
    crc = 0
    for i in range(len(data)):
        crc = crc + data[i]
    return crc & 0xFF

def write_cmd(cmd: int, data: str):
    data = data.replace(" ", "").replace(".", "")
    size = int((len(data) / 2) + 1)
    data = f"AA{size:02x}{cmd:02x}{data}"
    crc = calc_crc(bytes.fromhex(data))
    write_aa(f"{data}{crc:02x}")

def process_aa(buf: bytes):
    """Process AA commant"""
    global DEV_STATE
    global UID

    if SHOW_AA > 1:
        print(f"r <<< {buf.hex('.')} ({len(buf)})")
    # remove aa, sz and crc
    crc = buf[len(buf) - 1]
    buf = buf[2:-1]
    cmd = buf[0]
    buf = buf[1:]
    if SHOW_AA > 0 and cmd != 0x08 and cmd != 0x01 and cmd != 0x07:
        print(f"r <<< {cmd:02X}: {buf.hex('.')} ({len(buf)})")

    if False:  # just pretty print
        pass
    elif cmd == CMD_01_REQ_DEV_TYPE:
        write_cmd(0x81, "00.00.00.00.00.11")
    elif cmd == CMD_06_REQ_SAVE_DATA:
        UID = buf.hex(".")
        print(f"new UID {UID}")
        write_cmd(0x86, "01")
    elif cmd == CMD_07_REQ_LOAD_DATA:
        write_cmd(0x87, UID)
    elif cmd == CMD_08_REQ_STATE:
        # write_aa("AA 0A 88 00 1B 23 14 13 01 02 00 00 A4")
        write_cmd(
            CMD_88_RSP_STATE,
            f"{DEV_STATE[FS_ST]:02X}"
            + f"{DEV_STATE[FS_CT]:02X}{DEV_STATE[FS_TT]:02X}"
            + f"{DEV_STATE[FS_CH]:02X}{DEV_STATE[FS_CM]:02X}"
            + f"{DEV_STATE[FS_TH]:02X}{DEV_STATE[FS_TM]:02X}"
            + f"{DEV_STATE[FS_UNK]:02X}{DEV_STATE[FS_BST]:02X}",
        )
        # print(DEV_STATE)
    elif cmd == CMD_0A_REQ_SET:
        # no reponse to set command
        sub = buf[0]
        buf = buf[1:]
        if sub == OPER_00_STATE:
            DEV_STATE[FS_ST] = buf[0]
            DEV_STATE[FS_TT] = buf[1]
            mode = DEV_STATE[FS_ST]
            if mode == STATE_00_OFF:
                mode = "off"
            elif mode == MODE_01_700W:
                mode = "700W"
            elif mode == MODE_02_1300W:
                mode = "1300W"
            elif mode == MODE_03_2000W:
                mode = "2000W"
            elif mode == MODE_04_NO_FROST:
                mode = "no frost"
                DEV_STATE[FS_ST] = STATE_05_NO_FROST
            else:
                mode = f"unknown {mode:02X}"
            print(
                f"set mode {mode} ({DEV_STATE[FS_ST]:X}), temperature {DEV_STATE[FS_TT]} °C"
            )
        elif sub == OPER_01_CLOCK:
            print(f"set clock to {buf[0]:02d}:{buf[1]:02d}")
            DEV_STATE[FS_CH] = buf[0]
            DEV_STATE[FS_CM] = buf[1]
        elif sub == OPER_02_TIMER:
            DEV_STATE[FS_TH] = buf[0]
            DEV_STATE[FS_TM] = buf[1]
            mode = buf[2]
            taget_temp = buf[3]
            DEV_STATE[FS_ST] = STATE_04_TIMER
            print(
                f'set timer to {DEV_STATE[FS_TH]:02d}:{DEV_STATE[FS_TM]:02d}, mode {mode} and {taget_temp} °C ({buf.hex(".")})'
            )
        elif sub == OPER_03_BST:
            print(f"set BST to {buf[0]}")
            DEV_STATE[FS_BST] = buf[0]
        else:
            print(f'Unknown set command operation {sub:02X}: {buf.hex(".")}')
    else:
        print(
            f'Unknown AA command: {cmd:02X}, data {buf.hex(".")} ({len(buf)}), CRC: {crc:02X}'
        )

def process_at(buf: str):
    """Process AT commant"""
    if SHOW_AT:
        print(f"r <<< {buf}")
    if buf == "NDBGL=0,0":
        write_at("+ok")
    elif buf == "APPVER":
        write_at("=BAIGE-1.8-20170810")
    elif buf == "WSMAC":
        write_at(f"+ok={MAC.replace(':','')}\r\n")
    else:
        print(f"Unknown AT command: {buf}")

def process_buf(buf):
    """Process input buffer"""
    while len(buf) > 3:
        # print("buf: " + buf.hex("."))

        if buf[0] == 0xAA:
            end = buf[1] + 3
            if len(buf) < end:
                # not enought data len
                break
            process_aa(buf[0:end])
            buf = buf[end:]
            continue

        if buf[0] == ord("A") and buf[1] == ord("T") and buf[2] == ord("+"):
            index_r = buf.find(b"\r\n", 3)
            if index_r < 0:
                # not enought data len
                break
            process_at(buf[3:index_r].decode("utf-8"))
            buf = buf[index_r + 2 :]
            continue

        # not enought data len
        break

    return buf

def process():
    """Main processing cycle"""
    buf = bytes()
    while True:
        available = ser.in_waiting
        if available > 0:
            try:
                buf = process_buf(buf + ser.read(available))
            except serial.SerialException as err:
                print(err)
                break

def test_data():
    """Generate test data for loop interface"""
    ser.write(b"AT+NDBGL=0,0\r\n")
    ser.write(b"AT+NDBGL=0,0\r\n")
    ser.write(b"AT+APPVER\r\n")
    ser.write(b"AT+WSMAC\r\n")
    ser.write(b"\xAA\x01\x07\xB2")
    ser.write(b"AT+APPVER\r\n")
    ser.write(b"AT+WSMAC\r\n")
    ser.write(b"\xAA\x03\x08\x10\x04\xC9")
    ser.write(b"AT+WSMAC\r\n")

    write_cmd(CMD_0A_REQ_SET, "00 00 23")

    write_cmd(CMD_0A_REQ_SET, "00 01 23")
    write_cmd(CMD_0A_REQ_SET, "00 02 23")
    write_cmd(CMD_0A_REQ_SET, "00 03 23")
    write_cmd(CMD_0A_REQ_SET, "00 04 23")
    write_cmd(CMD_0A_REQ_SET, "00 05 23")
    write_cmd(CMD_0A_REQ_SET, "01 1119")
    write_cmd(CMD_0A_REQ_SET, "03 01")
    write_cmd(CMD_08_REQ_STATE, "")

if DEBUG:
    SHOW_AT = True
    SHOW_AA = 2
    test_data()

process()
ser.close()


import torch
import torch.nn as nn

from flwr_baselines.dataloaders.leaf.shakespeare import LEAF_CHARACTERS

class ShakespeareLeafNet(nn.Module):  # type: ignore
    """Create Shakespeare model for LEAF baselines.

    Args:
        chars (str, optional): String of possible characters (letters+digits).
            Defaults to LEAF_CHARACTERS.
        seq_len (int, optional): Length of each sequence. Defaults to 80.
        hidden_size (int, optional): Size of hidden layer. Defaults to 256.
        embedding_dim (int, optional): Dimension of embedding. Defaults to 8.
    """

    def __init__(
        self,
        chars: str = LEAF_CHARACTERS,
        seq_len: int = 80,
        hidden_size: int = 256,
        embedding_dim: int = 8,
    ):
        super().__init__()
        self.dict_size = len(chars)
        self.seq_len = seq_len
        self.hidden_size = hidden_size

        self.encoder = nn.Embedding(self.dict_size, embedding_dim)
        self.lstm = nn.LSTM(
            input_size=embedding_dim,
            hidden_size=hidden_size,
            num_layers=2,
            batch_first=True,  # Notice batch is first dim now
        )
        self.decoder = nn.Linear(self.hidden_size, self.dict_size)

    def forward(self, sentence: torch.Tensor) -> torch.Tensor:
        """Forwards sentence to obtain next character.

        Args:
            sentence (torch.Tensor): Tensor containing indices of characters

        Returns:
            torch.Tensor: Vector encoding position of predicted character
        """
        encoded_seq = self.encoder(sentence)  # (batch, seq_len, embedding_dim)
        _, (h_n, _) = self.lstm(encoded_seq)  # (batch, seq_len, hidden_size)
        pred = self.decoder(h_n[-1])
        return pred

def get_model() -> nn.Module:
    """Returns the LEAF Shakespeare network.

    Returns:
        nn.Module: Implementation of LEAF Shakespeare network
    """
    return ShakespeareLeafNet()


from __future__ import annotations
import itertools
import typing as T

from . import ExtensionModule, ModuleReturnValue, ModuleInfo
from .. import build
from .. import mesonlib
from ..interpreter.type_checking import CT_INPUT_KW
from ..interpreterbase.decorators import KwargInfo, typed_kwargs, typed_pos_args

if T.TYPE_CHECKING:
    from typing_extensions import TypedDict

    from . import ModuleState
    from ..interpreter import Interpreter
    from ..programs import ExternalProgram

    class ProjectKwargs(TypedDict):

        sources: T.List[T.Union[mesonlib.FileOrString, build.GeneratedTypes]]
        constraint_file: T.Union[mesonlib.FileOrString, build.GeneratedTypes]

class IceStormModule(ExtensionModule):

    INFO = ModuleInfo('FPGA/Icestorm', '0.45.0', unstable=True)

    def __init__(self, interpreter: Interpreter) -> None:
        super().__init__(interpreter)
        self.tools: T.Dict[str, T.Union[ExternalProgram, build.Executable]] = {}
        self.methods.update({
            'project': self.project,
        })

    def detect_tools(self, state: ModuleState) -> None:
        self.tools['yosys'] = state.find_program('yosys')
        self.tools['arachne'] = state.find_program('arachne-pnr')
        self.tools['icepack'] = state.find_program('icepack')
        self.tools['iceprog'] = state.find_program('iceprog')
        self.tools['icetime'] = state.find_program('icetime')

    @typed_pos_args('icestorm.project', str,
                    varargs=(str, mesonlib.File, build.CustomTarget, build.CustomTargetIndex,
                             build.GeneratedList))
    @typed_kwargs(
        'icestorm.project',
        CT_INPUT_KW.evolve(name='sources'),
        KwargInfo(
            'constraint_file',
            (str, mesonlib.File, build.CustomTarget, build.CustomTargetIndex, build.GeneratedList),
            required=True,
        )
    )
    def project(self, state: ModuleState,
                args: T.Tuple[str, T.List[T.Union[mesonlib.FileOrString, build.GeneratedTypes]]],
                kwargs: ProjectKwargs) -> ModuleReturnValue:
        if not self.tools:
            self.detect_tools(state)
        proj_name, arg_sources = args
        all_sources = self.interpreter.source_strings_to_files(
            list(itertools.chain(arg_sources, kwargs['sources'])))

        blif_target = build.CustomTarget(
            f'{proj_name}_blif',
            state.subdir,
            state.subproject,
            state.environment,
            [self.tools['yosys'], '-q', '-p', 'synth_ice40 -blif @OUTPUT@', '@INPUT@'],
            all_sources,
            [f'{proj_name}.blif'],
        )

        asc_target = build.CustomTarget(
            f'{proj_name}_asc',
            state.subdir,
            state.subproject,
            state.environment,
            [self.tools['arachne'], '-q', '-d', '1k', '-p', '@INPUT@', '-o', '@OUTPUT@'],
            [kwargs['constraint_file'], blif_target],
            [f'{proj_name}.asc'],
        )

        bin_target = build.CustomTarget(
            f'{proj_name}_bin',
            state.subdir,
            state.subproject,
            state.environment,
            [self.tools['icepack'], '@INPUT@', '@OUTPUT@'],
            [asc_target],
            [f'{proj_name}.bin'],
            build_by_default=True,
        )

        upload_target = build.RunTarget(
            f'{proj_name}-upload',
            [self.tools['iceprog'], bin_target],
            [],
            state.subdir,
            state.subproject,
            state.environment,
        )

        time_target = build.RunTarget(
            f'{proj_name}-time',
            [self.tools['icetime'], bin_target],
            [],
            state.subdir,
            state.subproject,
            state.environment,
        )

        return ModuleReturnValue(
            None,
            [blif_target, asc_target, bin_target, upload_target, time_target])

def initialize(interp: Interpreter) -> IceStormModule:
    return IceStormModule(interp)


import Chemins
from Utils import UTILS_Adaptations
from Utils.UTILS_Traduction import _
import wx
from Ctrl import CTRL_Bouton_image
import GestionDB
from Utils import UTILS_Interface
from Ctrl.CTRL_ObjectListView import FastObjectListView, ColumnDefn, Filter, CTRL_Outils
from Utils import UTILS_Utilisateurs
import wx.lib.colourselect as csel

def ConvertCouleur(couleur=None):
    if couleur == None or len(couleur) == 0 : return None
    couleur = couleur[1:-1].split(",")
    couleur = (int(couleur[0]), int(couleur[1]), int(couleur[2]) )
    return couleur

class Track(object):
    def __init__(self, donnees):
        self.IDlegende = donnees[0]
        self.nom = donnees[1]
        self.couleur = donnees[2]

class ListView(FastObjectListView):
    def __init__(self, *args, **kwds):
        # Récupération des paramètres perso
        self.selectionID = None
        self.selectionTrack = None
        self.criteres = ""
        self.itemSelected = False
        self.popupIndex = -1
        self.listeFiltres = []
        # Initialisation du listCtrl
        self.nom_fichier_liste = __file__
        FastObjectListView.__init__(self, *args, **kwds)
        # Binds perso
        self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.OnItemActivated)
        self.Bind(wx.EVT_CONTEXT_MENU, self.OnContextMenu)

    def OnItemActivated(self,event):
        self.Modifier(None)

    def InitModel(self):
        self.donnees = self.GetTracks()

    def GetTracks(self):
        """ Récupération des données """
        listeID = None
        db = GestionDB.DB()
        req = """SELECT IDlegende, nom, couleur
        FROM menus_legendes;"""
        db.ExecuterReq(req)
        listeDonnees = db.ResultatReq()
        db.Close()

        listeListeView = []
        for item in listeDonnees :
            valide = True
            if listeID != None :
                if item[0] not in listeID :
                    valide = False
            if valide == True :
                track = Track(item)
                listeListeView.append(track)
                if self.selectionID == item[0] :
                    self.selectionTrack = track
        return listeListeView

    def CreationImage(self, couleur=(255, 255, 255)):
        """ Création des images pour le TreeCtrl """
        if 'phoenix' in wx.PlatformInfo:
            bmp = wx.Image(16, 16, True)
            bmp.SetRGB((0, 0, 16, 16), couleur[0], couleur[1], couleur[2])
        else:
            bmp = wx.EmptyImage(16, 16, True)
            bmp.SetRGBRect((0, 0, 16, 16), couleur[0], couleur[1], couleur[2])
        return bmp.ConvertToBitmap()

    def InitObjectListView(self):
        # Couleur en alternance des lignes
        self.oddRowsBackColor = UTILS_Interface.GetValeur("couleur_tres_claire", wx.Colour(240, 251, 237))
        self.evenRowsBackColor = wx.Colour(255, 255, 255)
        self.useExpansionColumn = True

        # ListImages
        for track in self.donnees :
            img = self.CreationImage(ConvertCouleur(track.couleur))
            self.AddNamedImages(str(track.IDlegende), img)

        def GetImage(track):
            return str(track.IDlegende)

        liste_Colonnes = [
            ColumnDefn(_(u"ID"), "center", 50, "IDlegende", typeDonnee="entier", imageGetter=GetImage),
            ColumnDefn(_(u"Nom"), "left", 290, "nom", typeDonnee="texte", isSpaceFilling=True),
        ]

        self.SetColumns(liste_Colonnes)
        self.SetEmptyListMsg(_(u"Aucune légende"))
        self.SetEmptyListMsgFont(wx.FFont(11, wx.DEFAULT, False, "Tekton"))
        self.SetSortColumn(self.columns[1])
        self.SetObjects(self.donnees)

    def MAJ(self, ID=None):
        if ID != None :
            self.selectionID = ID
            self.selectionTrack = None
        else:
            self.selectionID = None
            self.selectionTrack = None
        self.InitModel()
        self.InitObjectListView()
        # Sélection d'un item
        if self.selectionTrack != None :
            self.SelectObject(self.selectionTrack, deselectOthers=True, ensureVisible=True)
        self.selectionID = None
        self.selectionTrack = None
        self._ResizeSpaceFillingColumns()

    def Selection(self):
        return self.GetSelectedObjects()

    def OnContextMenu(self, event):
        """Ouverture du menu contextuel """
        if len(self.Selection()) == 0:
            noSelection = True
        else:
            noSelection = False
            ID = self.Selection()[0].IDlegende

        # Création du menu contextuel
        menuPop = UTILS_Adaptations.Menu()

        # Item Ajouter
        item = wx.MenuItem(menuPop, 10, _(u"Ajouter"))
        bmp = wx.Bitmap(Chemins.GetStaticPath("Images/16x16/Ajouter.png"), wx.BITMAP_TYPE_PNG)
        item.SetBitmap(bmp)
        menuPop.AppendItem(item)
        self.Bind(wx.EVT_MENU, self.Ajouter, id=10)

        menuPop.AppendSeparator()

        # Item Modifier
        item = wx.MenuItem(menuPop, 20, _(u"Modifier"))
        bmp = wx.Bitmap(Chemins.GetStaticPath("Images/16x16/Modifier.png"), wx.BITMAP_TYPE_PNG)
        item.SetBitmap(bmp)
        menuPop.AppendItem(item)
        self.Bind(wx.EVT_MENU, self.Modifier, id=20)
        if noSelection == True : item.Enable(False)

        # Item Supprimer
        item = wx.MenuItem(menuPop, 30, _(u"Supprimer"))
        bmp = wx.Bitmap(Chemins.GetStaticPath("Images/16x16/Supprimer.png"), wx.BITMAP_TYPE_PNG)
        item.SetBitmap(bmp)
        menuPop.AppendItem(item)
        self.Bind(wx.EVT_MENU, self.Supprimer, id=30)
        if noSelection == True : item.Enable(False)

        menuPop.AppendSeparator()

        # Item Apercu avant impression
        item = wx.MenuItem(menuPop, 40, _(u"Aperçu avant impression"))
        bmp = wx.Bitmap(Chemins.GetStaticPath("Images/16x16/Apercu.png"), wx.BITMAP_TYPE_PNG)
        item.SetBitmap(bmp)
        menuPop.AppendItem(item)
        self.Bind(wx.EVT_MENU, self.Apercu, id=60)

        # Item Imprimer
        item = wx.MenuItem(menuPop, 50, _(u"Imprimer"))
        bmp = wx.Bitmap(Chemins.GetStaticPath("Images/16x16/Imprimante.png"), wx.BITMAP_TYPE_PNG)
        item.SetBitmap(bmp)
        menuPop.AppendItem(item)
        self.Bind(wx.EVT_MENU, self.Imprimer, id=70)

        self.PopupMenu(menuPop)
        menuPop.Destroy()

    def Apercu(self, event):
        from Utils import UTILS_Printer
        prt = UTILS_Printer.ObjectListViewPrinter(self, titre=_(u"Liste des légendes de menus"), format="A", orientation=wx.PORTRAIT)
        prt.Preview()

    def Imprimer(self, event):
        from Utils import UTILS_Printer
        prt = UTILS_Printer.ObjectListViewPrinter(self, titre=_(u"Liste des légendes de menus"), format="A", orientation=wx.PORTRAIT)
        prt.Print()

    def Ajouter(self, event):
        if UTILS_Utilisateurs.VerificationDroitsUtilisateurActuel("parametrage_menus_legendes", "creer") == False : return
        # Demande le nom
        dlg = DLG_Saisie(self)
        if dlg.ShowModal() == wx.ID_OK:
            DB = GestionDB.DB()
            listeDonnees = [
                ("nom", dlg.GetNom() ),
                ("couleur", dlg.GetCouleur() ),
                ]
            IDlegende = DB.ReqInsert("menus_legendes", listeDonnees)
            DB.Close()
            self.MAJ(IDlegende)
        dlg.Destroy()

    def Modifier(self, event):
        if UTILS_Utilisateurs.VerificationDroitsUtilisateurActuel("parametrage_menus_legendes", "modifier") == False : return
        if len(self.Selection()) == 0 :
            dlg = wx.MessageDialog(self, _(u"Vous n'avez sélectionné aucune légende à modifier dans la liste"), _(u"Erreur de saisie"), wx.OK | wx.ICON_EXCLAMATION)
            dlg.ShowModal()
            dlg.Destroy()
            return
        track = self.Selection()[0]
        dlg = DLG_Saisie(self, track.nom, track.couleur)
        if dlg.ShowModal() == wx.ID_OK:
            DB = GestionDB.DB()
            listeDonnees = [("nom", dlg.GetNom()), ("couleur", dlg.GetCouleur()),]
            DB.ReqMAJ("menus_legendes", listeDonnees, "IDlegende", track.IDlegende)
            DB.Close()
            self.MAJ(track.IDlegende)
        dlg.Destroy()

    def Supprimer(self, event):
        if UTILS_Utilisateurs.VerificationDroitsUtilisateurActuel("parametrage_menus_legendes", "supprimer") == False : return
        if len(self.Selection()) == 0 :
            dlg = wx.MessageDialog(self, _(u"Vous n'avez sélectionné aucune légende à supprimer dans la liste"), _(u"Erreur de saisie"), wx.OK | wx.ICON_EXCLAMATION)
            dlg.ShowModal()
            dlg.Destroy()
            return
        IDlegende = self.Selection()[0].IDlegende

        dlg = wx.MessageDialog(self, _(u"Souhaitez-vous vraiment supprimer cette légende ?"), _(u"Suppression"), wx.YES_NO|wx.NO_DEFAULT|wx.CANCEL|wx.ICON_INFORMATION)
        if dlg.ShowModal() == wx.ID_YES :
            DB = GestionDB.DB()
            DB.ReqDEL("menus_legendes", "IDlegende", IDlegende)
            DB.Close()
            self.MAJ()
        dlg.Destroy()

# -------------------------------------------------------------------------------------------------------------------------------------------

class DLG_Saisie(wx.Dialog):
    def __init__(self, parent, nom=None, couleur=None):
        wx.Dialog.__init__(self, parent, -1, style=wx.DEFAULT_DIALOG_STYLE|wx.RESIZE_BORDER|wx.MAXIMIZE_BOX|wx.MINIMIZE_BOX)
        self.parent = parent

        self.label_nom = wx.StaticText(self, -1, _(u"Nom :"))
        self.ctrl_nom = wx.TextCtrl(self, -1, "")
        if nom != None :
            self.ctrl_nom.SetValue(nom)

        self.label_couleur = wx.StaticText(self, -1, _(u"Couleur :"))
        self.ctrl_couleur = csel.ColourSelect(self, -1, "", (255, 255, 255), size = (40, 22))
        if couleur != None :
            self.SetCouleur(couleur)

        self.bouton_aide = CTRL_Bouton_image.CTRL(self, texte=_(u"Aide"), cheminImage="Images/32x32/Aide.png")
        self.bouton_ok = CTRL_Bouton_image.CTRL(self, texte=_(u"Ok"), cheminImage="Images/32x32/Valider.png")
        self.bouton_annuler = CTRL_Bouton_image.CTRL(self, id=wx.ID_CANCEL, texte=_(u"Annuler"), cheminImage="Images/32x32/Annuler.png")

        if nom == None :
            self.SetTitle(_(u"Saisie d'une légende"))
        else:
            self.SetTitle(_(u"Modification d'une légende"))
        self.SetMinSize((350, -1))

        grid_sizer_base = wx.FlexGridSizer(rows=3, cols=1, vgap=10, hgap=10)
        grid_sizer_boutons = wx.FlexGridSizer(rows=1, cols=4, vgap=10, hgap=10)
        grid_sizer_contenu = wx.FlexGridSizer(rows=2, cols=2, vgap=10, hgap=10)
        grid_sizer_contenu.Add(self.label_nom, 0, wx.ALIGN_RIGHT|wx.ALIGN_CENTER_VERTICAL, 0)
        grid_sizer_contenu.Add(self.ctrl_nom, 0, wx.EXPAND, 0)
        grid_sizer_contenu.Add(self.label_couleur, 0, wx.ALIGN_RIGHT|wx.ALIGN_CENTER_VERTICAL, 0)
        grid_sizer_contenu.Add(self.ctrl_couleur, 0, 0, 0)
        grid_sizer_contenu.AddGrowableCol(1)
        grid_sizer_base.Add(grid_sizer_contenu, 1, wx.ALL|wx.EXPAND, 10)
        grid_sizer_boutons.Add(self.bouton_aide, 0, 0, 0)
        grid_sizer_boutons.Add((20, 20), 0, wx.EXPAND, 0)
        grid_sizer_boutons.Add(self.bouton_ok, 0, 0, 0)
        grid_sizer_boutons.Add(self.bouton_annuler, 0, 0, 0)
        grid_sizer_boutons.AddGrowableCol(1)
        grid_sizer_base.Add(grid_sizer_boutons, 1, wx.LEFT|wx.RIGHT|wx.BOTTOM|wx.EXPAND, 10)
        self.SetSizer(grid_sizer_base)
        grid_sizer_base.Fit(self)
        grid_sizer_base.AddGrowableCol(0)
        self.Layout()
        self.CenterOnScreen()

        self.Bind(wx.EVT_BUTTON, self.OnBoutonOk, self.bouton_ok)
        self.Bind(wx.EVT_BUTTON, self.OnBoutonAide, self.bouton_aide)

    def OnBoutonAide(self, event):
        from Utils import UTILS_Aide
        UTILS_Aide.Aide("Legendesdemenus")

    def GetNom(self):
        return self.ctrl_nom.GetValue()

    def SetCouleur(self, valeur=None):
        if valeur == None or valeur == "": return
        try:
            valeur = ConvertCouleur(valeur)
            self.ctrl_couleur.SetValue(valeur)
        except:
            pass

    def GetCouleur(self):
        return str(self.ctrl_couleur.GetValue())

    def OnBoutonOk(self, event):
        nom = self.ctrl_nom.GetValue()
        if nom == "" :
            dlg = wx.MessageDialog(self, _(u"Vous n'avez saisi aucun nom de légende !"), _(u"Erreur de saisie"), wx.OK | wx.ICON_EXCLAMATION)
            dlg.ShowModal()
            dlg.Destroy()
            self.ctrl_nom.SetFocus()
            return

        self.EndModal(wx.ID_OK)

# -------------------------------------------------------------------------------------------------------------------------------------------

class MyFrame(wx.Frame):
    def __init__(self, *args, **kwds):
        wx.Frame.__init__(self, *args, **kwds)
        panel = wx.Panel(self, -1, name="test1")
        sizer_1 = wx.BoxSizer(wx.VERTICAL)
        sizer_1.Add(panel, 1, wx.ALL|wx.EXPAND)
        self.SetSizer(sizer_1)
        self.myOlv = ListView(panel, id=-1, name="OL_test", style=wx.LC_REPORT|wx.SUNKEN_BORDER|wx.LC_SINGLE_SEL|wx.LC_HRULES|wx.LC_VRULES)
        self.myOlv.MAJ()
        sizer_2 = wx.BoxSizer(wx.VERTICAL)
        sizer_2.Add(self.myOlv, 1, wx.ALL|wx.EXPAND, 4)
        panel.SetSizer(sizer_2)
        self.Layout()

if __name__ == '__main__':
    app = wx.App(0)
    #wx.InitAllImageHandlers()
    frame_1 = MyFrame(None, -1, "OL TEST")
    app.SetTopWindow(frame_1)
    frame_1.Show()
    app.MainLoop()


import pandas as pd
import sys
import datetime
import os

#Province/State  Country/Region      Last Update  Confirmed  Deaths  Recovered
def parse_coronavirus_data(data_path, input_date):
    df = pd.read_csv(data_path, sep=',', header=0, encoding='utf-8')
    print(df)

    #1. add a column date to be the integration date
    print("add a date column")
    print(input_date)
    date_object =datetime.datetime.strptime(input_date, "%m-%d-%Y")
    print(date_object)
    input_date=date_object.strftime("%Y-%m-%d")
    df.loc[:, 'date']=input_date
    print(df)

    #2. rename the join column
    print("rename columns")
    df.rename({'Province/State':'sub_region_1', 'Country/Region':'country_region_code'}, axis='columns', inplace=True)
    print(df)

    #3. to lowercase
    print("to lower case")
    df['sub_region_1']=df['sub_region_1'].str.lower()
    df['sub_region_1']=df['sub_region_1'].str.strip()
    df['country_region_code']=df['country_region_code'].str.lower()
    df['country_region_code']=df['country_region_code'].str.strip()
    df['date']=df['date'].str.strip()

    states = {
        'AK': 'Alaska',
        'AL': 'Alabama',
        'AR': 'Arkansas',
        'AS': 'American Samoa',
        'AZ': 'Arizona',
        'CA': 'California',
        'CO': 'Colorado',
        'CT': 'Connecticut',
        'DC': 'District of Columbia',
        'DE': 'Delaware',
        'FL': 'Florida',
        'GA': 'Georgia',
        'GU': 'Guam',
        'HI': 'Hawaii',
        'IA': 'Iowa',
        'ID': 'Idaho',
        'IL': 'Illinois',
        'IN': 'Indiana',
        'KS': 'Kansas',
        'KY': 'Kentucky',
        'LA': 'Louisiana',
        'MA': 'Massachusetts',
        'MD': 'Maryland',
        'ME': 'Maine',
        'MI': 'Michigan',
        'MN': 'Minnesota',
        'MO': 'Missouri',
        'MP': 'Northern Mariana Islands',
        'MS': 'Mississippi',
        'MT': 'Montana',
        'NA': 'National',
        'NC': 'North Carolina',
        'ND': 'North Dakota',
        'NE': 'Nebraska',
        'NH': 'New Hampshire',
        'NJ': 'New Jersey',
        'NM': 'New Mexico',
        'NV': 'Nevada',
        'NY': 'New York',
        'OH': 'Ohio',
        'OK': 'Oklahoma',
        'OR': 'Oregon',
        'PA': 'Pennsylvania',
        'PR': 'Puerto Rico',
        'RI': 'Rhode Island',
        'SC': 'South Carolina',
        'SD': 'South Dakota',
        'TN': 'Tennessee',
        'TX': 'Texas',
        'UT': 'Utah',
        'VA': 'Virginia',
        'VI': 'Virgin Islands',
        'VT': 'Vermont',
        'WA': 'Washington',
        'WI': 'Wisconsin',
        'WV': 'West Virginia',
        'WY': 'Wyoming'
    }

    for i, row in df.iterrows():
       cur_state=df.at[i, 'sub_region_1']
       print(cur_state)
       position=str(cur_state).find(',')
       if (position>0):
          state=cur_state[position+1:]
          state=str(state).strip()
          state=str(state).upper()
          print(state)
          if state in states:
              df.at[i, 'sub_region_1']=states[state].lower()

    print(df)

    #4. groupby and aggregation
    print("groupby and aggregation")
    df=df.groupby(['sub_region_1', 'country_region_code', 'date']).agg({'Confirmed':'sum', 'Deaths':'sum', 'Recovered':'sum'})
    print(df)

    return df

#country_region_code,country_region,sub_region_1,sub_region_2,metro_area,iso_3166_2_code,census_fips_code,date,retail_and_recreation_percent_change_from_baseline,grocery_and_pharmacy_percent_change_from_baseline,parks_percent_change_from_baseline,transit_stations_percent_change_from_baseline,workplaces_percent_change_from_baseline,residential_percent_change_from_baseline
def parse_mobility_data(data_path):
    df = pd.read_csv(data_path, sep=',', header=0, encoding='utf-8', low_memory=False)
    print(df)

    #1. aggregate based on country region
    #it includes both aggregated and non aggregated values

    #2. remove county level statistics

    df = df[pd.notnull(df['sub_region_2'])]

    #3  to lowercase
    print("to lowercase for the other table")
    df['sub_region_1']=df['sub_region_1'].str.lower()
    df['sub_region_1']=df['sub_region_1'].str.strip()
    df['country_region_code']=df['country_region_code'].str.lower()
    df['country_region_code']=df['country_region_code'].str.strip()
    df['date']=df['date'].str.strip()

    print(df)
    return df

#download

def download():
    if (os.path.isfile('source-datasets/Global_Mobility_Report.csv')):
        print("file Global_Mobility_Report.csv exists in source-datasets/")
    else:
        os.system('wget https://www.gstatic.com/covid19/mobility/Global_Mobility_Report.csv')
        os.system('mv Global_Mobility_Report.csv source-datasets/')

if __name__ == '__main__':
    input_date = str(sys.argv[1])
    print(input_date)
    download()
    df1=parse_coronavirus_data('source-datasets/'+input_date+'.csv', input_date)
    df1.to_csv('left.csv', encoding='utf-8')
    df2=parse_mobility_data('source-datasets/Global_Mobility_Report.csv')
    df2.to_csv('right.csv', encoding='utf-8')
    #join/merge
    joined=df1.merge(df2, how='inner', on=['sub_region_1','country_region_code','date'])
    print(joined)
    joined.to_csv('target-datasets/'+input_date+'.csv', encoding='utf-8')


# -*- coding: utf-8 -*-
r"""

This command helps to interact with `bib` files in your LaTeX projects.

Examples
^^^^^^^^

::

    papis bibtex                            \
      read new_papers.bib                   \ # Read bib file
      cmd 'papis add --from-doi {doc[doi]}'   # For every entry run the command

I use it for opening some papers for instance

::

    papis bibtex read new_papers.bib open

or to add papers to the bib

::

    papis bibtex          \
      read new_papers.bib \ # Read bib file
      add einstein        \ # Pick a doc with query 'einstein' from library
      add heisenberg      \ # Pick a doc with query 'heisenberg' from library
      save new_papers.bib   # Save in new_papers.bib

or if I update some information in my papis ``yaml`` files then I can do

::

    papis bibtex          \
      read new_papers.bib \ # Read bib file
      update -f           \ # Update what has been read from papis library
      save new_papers.bib   # save everything to new_papers.bib, overwriting

Maybe this is also interesting for you guys!

Vim integration
^^^^^^^^^^^^^^^

Right now, you can easily use it from vim with these simple lines

.. code:: vimscript

    function! PapisBibtexRef()
      let l:temp = tempname()
      echom l:temp
      silent exec "!papis bibtex ref -o ".l:temp
      let l:olda = @a
      let @a = join(readfile(l:temp), ',')
      normal! "ap
      redraw!
      let @a = l:olda
    endfunction

    command! -nargs=0 BibRef call PapisBibtexRef()
    command! -nargs=0 BibOpen exec "!papis bibtex open"

And use like such: |asciicast|

.. |asciicast| image:: https://asciinema.org/a/8KbLQJSVYVYNXHVF3wgcxx5Cp.svg
   :target: https://asciinema.org/a/8KbLQJSVYVYNXHVF3wgcxx5Cp

Cli
^^^
.. click:: papis.commands.add:cli
    :prog: papis add

"""
import os
import re
import papis.api
import papis.cli
import click
import papis.config as config
import papis.utils
import papis.tui.utils
import papis.commands.explore as explore
import papis.commands.add
import papis.commands.open
import papis.commands.edit
import papis.commands.browse
import papis.commands.export
import papis.bibtex
import logging
import colorama

from typing import List, Optional

logger = logging.getLogger('papis:bibtex')

config.register_default_settings({'bibtex': {
    'default-read-bibfile': '',
    'auto-read': '',
    'default-save-bibfile': ''
}})

explorer_mgr = explore.get_explorer_mgr()

@click.group("bibtex", cls=papis.cli.AliasedGroup, chain=True)
@click.help_option('-h', '--help')
@click.option('--noar', '--no-auto-read', 'no_auto_read',
              default=False,
              is_flag=True,
              help="Do not auto read even if the configuration file says it")
@click.pass_context
def cli(ctx: click.Context, no_auto_read: bool) -> None:
    """A papis script to interact wit bibtex files"""
    global explorer_mgr
    ctx.obj = {'documents': []}

    if no_auto_read:
        logger.info('Setting auto-read to False')
        config.set('auto-read', 'False', section='bibtex')

    bibfile = config.get('default-read-bibfile', section='bibtex')
    if (bool(config.getboolean('auto-read', section='bibtex')) and
       bibfile and
       os.path.exists(bibfile)):
        logger.info("auto reading {0}".format(bibfile))
        explorer_mgr['bibtex'].plugin.callback(bibfile)

cli.add_command(explorer_mgr['bibtex'].plugin, 'read')

@cli.command('add')
@papis.cli.query_option()
@click.help_option('-h', '--help')
@papis.cli.all_option()
@click.pass_context
def _add(ctx: click.Context, query: str, _all: bool) -> None:
    """Add a refrence to the bibtex file"""
    docs = papis.api.get_documents_in_lib(search=query)
    if not _all:
        docs = list(papis.api.pick_doc(docs))
    ctx.obj['documents'].extend(docs)

@cli.command('update')
@click.help_option('-h', '--help')
@papis.cli.all_option()
@click.option('--from', '-f', 'fromdb',
              show_default=True,
              help='Update the document from the library',
              default=False, is_flag=True)
@click.option('-t', '--to',
              help='Update the library document from retrieved document',
              show_default=True,
              default=False, is_flag=True)
@click.option('-k', '--keys',
              help='Update only given keys (can be given multiple times)',
              type=str,
              multiple=True)
@click.pass_context
def _update(ctx: click.Context, _all: bool,
            fromdb: bool, to: bool, keys: List[str]) -> None:
    """Update documents from and to the library"""
    docs = click.get_current_context().obj['documents']
    picked_doc = None
    if not _all:
        picked_docs = papis.api.pick_doc(docs)
        if picked_docs is None or picked_docs[0] is None:
            return
        picked_doc = picked_docs[0]
    for j, doc in enumerate(docs):
        if picked_doc and doc["ref"] != picked_doc["ref"]:
            continue
        try:
            libdoc = papis.utils.locate_document_in_lib(doc)
        except IndexError as e:
            logger.info(
                '{c.Fore.YELLOW}{0}:'
                '\n\t{c.Back.RED}{doc: <80.80}{c.Style.RESET_ALL}'
                .format(e, doc=papis.document.describe(doc), c=colorama)
            )
        else:
            if fromdb:
                logger.info(
                    'Updating \n\t{c.Fore.GREEN}'
                    '{c.Back.BLACK}{doc: <80.80}{c.Style.RESET_ALL}'
                    .format(doc=papis.document.describe(doc), c=colorama)
                )
                if keys:
                    docs[j].update(
                        {k: libdoc.get(k) for k in keys if libdoc.has(k)})
                else:
                    docs[j] = libdoc
    click.get_current_context().obj['documents'] = docs

@cli.command('open')
@click.help_option('-h', '--help')
@click.pass_context
def _open(ctx: click.Context) -> None:
    """Open a document in the documents list"""
    docs = ctx.obj['documents']
    docs = papis.api.pick_doc(docs)
    if not docs:
        return
    doc = papis.utils.locate_document_in_lib(docs[0])
    papis.commands.open.run(doc)

@cli.command('edit')
@click.help_option('-h', '--help')
@click.option('-l', '--lib',
              show_default=True,
              help='Edit document in papis library',
              default=False, is_flag=True)
@click.pass_context
def _edit(ctx: click.Context, lib: bool) -> None:
    """edit a document in the documents list"""
    docs = ctx.obj['documents']
    docs = papis.api.pick_doc(docs)
    if not docs:
        return
    doc = papis.utils.locate_document_in_lib(docs[0])
    papis.commands.edit.run(doc)

@cli.command('browse')
@click.help_option('-h', '--help')
@click.option('-k', '--key', default=None, help="doi, url, ...")
@click.pass_context
def _browse(ctx: click.Context, key: Optional[str]) -> None:
    """browse a document in the documents list"""
    docs = papis.api.pick_doc(ctx.obj['documents'])
    if key:
        config.set("browse-key", key)
    if not docs:
        return
    for d in docs:
        papis.commands.browse.run(d)

@cli.command('rm')
@click.help_option('-h', '--help')
@click.pass_context
def _rm(ctx: click.Context) -> None:
    """Remove a document from the documents list"""
    print('Sorry, TODO...')

@cli.command('ref')
@click.help_option('-h', '--help')
@click.option('-o', '--out', help='Output ref to a file', default=None)
@click.pass_context
def _ref(ctx: click.Context, out: Optional[str]) -> None:
    """Print the reference for a document"""
    docs = ctx.obj['documents']
    docs = papis.api.pick_doc(docs)
    if not docs:
        return
    ref = docs[0]["ref"]
    if out:
        with open(out, 'w+') as fd:
            fd.write(ref)
    else:
        print(ref)

@cli.command('save')
@click.help_option('-h', '--help')
@click.argument(
    'bibfile',
    default=lambda: config.get('default-save-bibfile', section='bibtex'),
    required=True, type=click.Path())
@click.option('-f', '--force', default=False, is_flag=True)
@click.pass_context
def _save(ctx: click.Context, bibfile: str, force: bool) -> None:
    """Save the documents imported in bibtex format"""
    docs = ctx.obj['documents']
    if not force:
        c = papis.tui.utils.confirm('Are you sure you want to save?')
        if not c:
            print('Not saving..')
            return
    with open(bibfile, 'w+') as fd:
        logger.info('Saving {1} documents in {0}..'.format(bibfile, len(docs)))
        fd.write(papis.commands.export.run(docs, to_format='bibtex'))

@cli.command('sort')
@click.help_option('-h', '--help')
@click.option('-k', '--key',
              help="Field to order it",
              default=None,
              type=str,
              required=True)
@click.option('-r', '--reverse',
              help="Reverse the order",
              default=False,
              is_flag=True)
@click.pass_context
def _sort(ctx: click.Context, key: Optional[str], reverse: bool) -> None:
    """Sort documents"""
    docs = ctx.obj['documents']
    ctx.obj['documents'] = list(sorted(docs,
                                       key=lambda d: str(d[key]),
                                       reverse=reverse))

@cli.command('unique')
@click.help_option('-h', '--help')
@click.option('-k', '--key',
              help="Field to test for uniqueness, default is ref",
              default="ref",
              type=str)
@click.option('-o',
              help="Output the discarded documents to a file",
              default=None,
              type=str)
@click.pass_context
def _unique(ctx: click.Context, key: str, o: Optional[str]) -> None:
    """Remove repetitions"""
    docs = ctx.obj['documents']
    unique_docs = []
    duplis_docs = []

    while True:
        if not len(docs):
            break
        doc = docs.pop(0)
        unique_docs.append(doc)
        indices = []
        for i, bottle in enumerate(docs):
            if doc.get(key) == bottle.get(key):
                indices.append(i)
                duplis_docs.append(bottle)
                logger.info('{}. repeated {} ⇒ {}'
                            .format(len(duplis_docs), key, doc.get(key)))
        docs = [d for (i, d) in enumerate(docs) if i not in indices]

    logger.info("Unique   : {}".format(len(unique_docs)))
    logger.info("Discarded: {}".format(len(duplis_docs)))

    ctx.obj['documents'] = unique_docs
    if o:
        with open(o, 'w+') as f:
            logger.info('Saving {1} documents in {0}..'
                        .format(o, len(duplis_docs)))
            f.write(papis.commands.export.run(duplis_docs, to_format='bibtex'))

@cli.command('doctor')
@click.help_option('-h', '--help')
@click.option('-k', '--key',
              help="Field to test for uniqueness, default is ref",
              multiple=True,
              default=("doi", "url", "year", "title", "author"),
              type=str)
@click.pass_context
def _doctor(ctx: click.Context, key: List[str]) -> None:
    """
    Check bibfile for correctness, missing keys etc.
        e.g. papis bibtex -k title -k url -k doi

    """
    logger.info("Checking for existence of %s", ", ".join(key))

    failed = [(d, keys) for d, keys in [(d, [k for k in key if not d.has(k)])
                                        for d in ctx.obj['documents']]
              if keys]

    for j, (doc, keys) in enumerate(failed):
        logger.info('{} {c.Back.BLACK}{c.Fore.RED}{doc: <80.80}'
                    '{c.Style.RESET_ALL}'
                    .format(j, doc=papis.document.describe(doc), c=colorama))
        for k in keys:
            logger.info('\tmissing: %s', k)

@cli.command('iscited')
@click.help_option('-h', '--help')
@click.option('-f', '--file', '_files',
              help="Text file to check for references",
              multiple=True, required=True, type=str)
@papis.cli.all_option()
@click.pass_context
def _iscited(ctx: click.Context, _files: List[str], _all: bool) -> None:
    """
    Check which documents are not cited
    e.g. papis bibtex iscited -f main.tex -f chapter-2.tex
    """
    unfound = []

    for f in _files:
        with open(f) as fd:
            text = fd.read()
            for doc in ctx.obj['documents']:
                if not re.search(doc["ref"], text):
                    unfound.append(doc)

    logger.info('%s documents not cited', len(unfound))

    for j, doc in enumerate(unfound):
        logger.info('{} {c.Back.BLACK}{c.Fore.RED}{doc: <80.80}'
                    '{c.Style.RESET_ALL}'
                    .format(j, doc=papis.document.describe(doc), c=colorama))

@cli.command('import')
@click.help_option('-h', '--help')
@click.option('-o', '--out', help="Out folder to export", default=None)
@papis.cli.all_option()
@click.pass_context
def _import(ctx: click.Context, out: Optional[str], _all: bool) -> None:

    """
    Import documents to papis
        e.g. papis bibtex read mybib.bib import
    """
    docs = ctx.obj['documents']

    if not _all:
        docs = papis.api.pick_doc(docs)

    if out is not None:
        logging.info("Setting lib name to %s", out)
        if not os.path.exists(out):
            os.makedirs(out)
        config.set_lib_from_name(out)

    for j, doc in enumerate(docs):
        fileValue = None
        filepaths = []
        for k in ["file", "FILE"]:
            logger.info('{} {c.Back.BLACK}{c.Fore.YELLOW}{doc: <80.80}'
                        '{c.Style.RESET_ALL}'
                        .format(j, doc=papis.document.describe(doc),
                                c=colorama))
            if doc.has(k):
                fileValue = doc[k]
                logger.info("\tkey '%s' exists", k)
                break

        if not fileValue:
            logger.info("\t"
                        "{c.Back.YELLOW}{c.Fore.BLACK}"
                        "no pdf files will be imported"
                        "{c.Style.RESET_ALL}".format(c=colorama))
        else:
            filepaths = [f for f in fileValue.split(":") if os.path.exists(f)]

        if not filepaths and fileValue is not None:
            logger.info("\t"
                        "{c.Back.BLACK}{c.Fore.RED}"
                        "I could not find a valid file in \n"
                        "{value}{c.Style.RESET_ALL}"
                        .format(value=fileValue, c=colorama))
        else:
            logger.info("\tfound %s file(s)", len(filepaths))

        papis.commands.add.run(filepaths, data=doc)


import os
import pickle
import shutil
from unittest import TestCase

import numpy as np
from sklearn.svm import LinearSVC

from immuneML.caching.CacheType import CacheType
from immuneML.data_model.encoded_data.EncodedData import EncodedData
from immuneML.environment.Constants import Constants
from immuneML.environment.EnvironmentSettings import EnvironmentSettings
from immuneML.environment.Label import Label
from immuneML.ml_methods.classifiers.SVC import SVC
from immuneML.util.PathBuilder import PathBuilder

class TestSVC(TestCase):

    def setUp(self) -> None:
        os.environ[Constants.CACHE_TYPE] = CacheType.TEST.name

    def test_fit(self):
        x = np.array([[1, 0, 0], [0, 1, 1], [1, 1, 1], [0, 1, 1]])
        y = {"default": np.array([1, 0, 2, 0])}

        svm = SVC()
        svm.fit(EncodedData(x, y), Label("default", [1, 0, 2]))

    def test_predict(self):
        x = np.array([[1, 0, 0], [0, 1, 1], [1, 1, 1], [0, 1, 1]])
        y = {"test": np.array([1, 0, 2, 0])}

        svm = SVC()
        svm.fit(EncodedData(x, y), Label("test", [1, 0, 2]))

        test_x = np.array([[0, 1, 0], [1, 0, 0]])
        y = svm.predict(EncodedData(test_x), Label("test", [1, 0, 2]))["test"]

        self.assertTrue(len(y) == 2)
        self.assertTrue(y[0] in [0, 1, 2])
        self.assertTrue(y[1] in [0, 1, 2])

    def test_fit_by_cross_validation(self):
        x = EncodedData(np.array([[1, 0, 0], [0, 1, 1], [1, 1, 1], [0, 1, 1], [1, 0, 0], [0, 1, 1], [1, 1, 1], [0, 1, 1]]),
                        {"t1": [1, 0, 2, 0, 1, 0, 2, 0], "t2": [1, 0, 2, 0, 1, 0, 2, 0]})

        svm = SVC(parameter_grid={"penalty": ["l1"], "dual": [False]})
        svm.fit_by_cross_validation(x, number_of_splits=2, label=Label("t1", [1, 0, 2]))

    def test_store(self):
        x = np.array([[1, 0, 0], [0, 1, 1], [1, 1, 1], [0, 1, 1]])
        y = {"default": np.array(['a', "b", "c", "a"])}

        svm = SVC()
        svm.fit(EncodedData(x, y), Label("default", ["a", "b", "c"]))

        path = EnvironmentSettings.root_path / "my_svc/"

        svm.store(path)
        self.assertTrue(os.path.isfile(path / "svc.pickle"))

        with open(path / "svc.pickle", "rb") as file:
            svm2 = pickle.load(file)

        self.assertTrue(isinstance(svm2, LinearSVC))

        shutil.rmtree(path)

    def test_load(self):
        x = np.array([[1, 0, 0], [0, 1, 1], [1, 1, 1], [0, 1, 1]])
        y = {"default": np.array([1, 0, 2, 0])}

        svm = SVC()
        svm.fit(EncodedData(x, y), Label("default", [0, 1, 2]))

        path = EnvironmentSettings.tmp_test_path / "my_svc2/"
        PathBuilder.build(path)

        with open(path / "svc.pickle", "wb") as file:
            pickle.dump(svm.model, file)

        svm2 = SVC()
        svm2.load(path)

        self.assertTrue(isinstance(svm2.model, LinearSVC))

        shutil.rmtree(path)


import numpy as np
import pandas as pd
from Bio import pairwise2
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import connected_components
import pickle

def calculate_identity(seqA, seqB):
    """
    :param seqA: The sequence of amino acid from chain A
    :param seqB: The sequence of amino acid from chain B
    :return: percentage of identity between the sequences
    """
    score = pairwise2.align.globalxx(seqA, seqB, one_alignment_only=True, score_only=True)
    # min_len = min(len(seqA), len(seqB))
    # identity = score / min_len
    max_len = max(len(seqA), len(seqB))
    identity = score / max_len
    return identity

def keep_only_chars(string):
    return ''.join([char for char in string if char.isalpha()])

def listCreationUtil(fullName):
    pdbName = fullName[0:4]
    chainsString = fullName.split('_')[1]
    chainsStringsList = chainsString.split('+')
    chainsNamesList = [keep_only_chars(chainString) for chainString in chainsStringsList]
    pdbNamesWithChainsList = [pdbName + chainName for chainName in chainsNamesList]
    return pdbName, pdbNamesWithChainsList

def createDictionaries(namesList, sizesList, sequenceLists, fullNamesList, pdbNamesWithChainsLists):
    structuresDicts = {}
    for i in range(len(fullNamesList)):
        structureDict = {}
        structureDict['pdbName'] = namesList[i]
        structureDict['size'] = sizesList[i]
        structureDict['sequenceList'] = sequenceLists[i]
        structureDict['pdbNamesWithChainsList'] = pdbNamesWithChainsLists[i]
        structuresDicts[fullNamesList[i]] = structureDict
    return structuresDicts

def listCreation(filename):
    """
    :param filename: PSSM file
    :return: tuple(namesList,sizesList)
    namesList = list of all the chains's name in the file
    sizesList = list of all the chains's number of amino acids in the file
    """
    namesList = []
    fullNamesList = []
    pdbNamesWithChainsLists = []
    sizesList = []
    sequenceLists = [[]]
    file1 = open(filename, 'r')
    lastChainName = ''
    line = file1.readline().split()
    cnt = 0
    seq = ''
    while len(line) != 0:  # end of file
        cnt += 1
        if len(line) == 1:  # in chain header line
            sequenceLists[len(sequenceLists) - 1].append(seq)
            sizesList.append(cnt)
            fullName = line[0][1:]
            fullNamesList.append(fullName)
            pdbName, pdbNamesWithChainsList = listCreationUtil(fullName)
            namesList.append(pdbName)
            pdbNamesWithChainsLists.append(pdbNamesWithChainsList)
            try:
                if len(pdbNamesWithChainsLists) > 1:
                    assert (len(sequenceLists[len(sequenceLists) - 1]) == len(
                        pdbNamesWithChainsLists[len(pdbNamesWithChainsLists) - 2]))
                    assert (sizesList[len(sizesList) - 1]) == sum(
                        [len(seq) for seq in sequenceLists[len(sequenceLists) - 1]])
            except:
                print(pdbNamesWithChainsList)
                print(sequenceLists[len(sequenceLists) - 1])
                raise Exception(pdbName)
            sequenceLists.append([])
            cnt = -1
            seq = ''
            lastChainName = ''
        else:
            if lastChainName != line[0]:  # switching chains
                lastChainName = line[0]
                if len(seq) != 0:
                    sequenceLists[len(sequenceLists) - 1].append(seq)
                seq = ''
            seq = seq + line[2]  # not chain's name
        line = file1.readline().split()
    sizesList.append(cnt)
    sequenceLists[len(sequenceLists) - 1].append(seq)
    sizesList = sizesList[1:]  # first one is redundent
    sequenceLists = sequenceLists[1:]
    file1.close()
    return namesList, sizesList, sequenceLists, fullNamesList, pdbNamesWithChainsLists

def make_cath_df(filename, columns_number):
    """
    :param filename: cath-domain-list file
    :param columns_number: the number of columns to consider with the cath classification not include the cath domain name
    :return: dataframe of all the chains in the file and their cath classification divide to 4 different columns
    """

    df = pd.read_csv(filename, skiprows=16, header=None, delimiter=r"\s+")
    df = df.iloc[:, 0:columns_number + 1]
    cath_columns = ["n" + str(i) for i in range(1, columns_number + 1)]
    df.columns = ['chain'] + cath_columns
    df['chain'] = df['chain'].apply(lambda x: x[0:5])
    return df

def make_cath_df_new(filename, columns_number):
    """
    :param filename: cath-domain-list file
    :param columns_number: the number of columns to consider with the cath classification not include the cath domain name
    :return: dataframe of all the chains in the file and their cath classification divide to 4 different columns
    """
    file = open("cath_b.20230204.txt", 'r')
    lines = file.readlines()
    structuresNames = [line[0:5] for line in lines]
    structuresNumbers = [line[5:7] for line in lines]
    c0 = [line.split(" ")[2].split(".")[0] for line in lines]
    c1 = [line.split(" ")[2].split(".")[1] for line in lines]
    c2 = [line.split(" ")[2].split(".")[2] for line in lines]
    c3 = [line.split(" ")[2].split(".")[3] for line in lines]
    data = {
        'chain': structuresNames,
        'number': structuresNumbers,
        'c0': c0,
        'c1': c1,
        'c2': c2,
        'c3': c3,
    }
    df = pd.DataFrame(data)
    print(df)
    return df

def getAllCathClassificationsForChain(cath_df, chainName, columns_Number):
    df = cath_df[cath_df['chain'] == chainName]
    myList = df.values.tolist()
    onlyClassificationList = [l[2:2 + columns_Number] for l in myList]
    return onlyClassificationList

def addClassificationsForDict(cath_df, structuresDicts, columns_Number):
    for key in structuresDicts.keys():
        structureDict = structuresDicts[key]
        classificationsLists = []
        for i in range(len(structureDict['pdbNamesWithChainsList'])):
            if structureDict['inOrNotInCathList'][i]:
                classificationsLists.append(
                    getAllCathClassificationsForChain(cath_df, structureDict['pdbNamesWithChainsList'][i],
                                                      columns_Number))
            else:
                classificationsLists.append(None)
        structureDict['classificationsLists'] = classificationsLists

def findChainsInCath(cath_df, structuresDicts):
    setOfchainsNames = set(cath_df['chain'])
    for fullpdbName, structureDict in structuresDicts.items():
        inOrNotInCathList = [structureDict['pdbNamesWithChainsList'][j] in setOfchainsNames for j in
                             range(len(structureDict['pdbNamesWithChainsList']))]
        structureDict['inOrNotInCathList'] = inOrNotInCathList

def DivideToStructuresInAndNotInCath(cath_df, structuresDicts):
    setOfchainsNames = set(cath_df['chain'])
    inCath = []
    notInCath = []
    for fullpdbName, structureDict in structuresDicts.items():
        sequencesInCath = []
        pdbNamesWithChainsIn = []
        for i in range(len(structureDict['pdbNamesWithChainsList'])):
            if structureDict['pdbNamesWithChainsList'][i] in setOfchainsNames:
                sequencesInCath.append(structureDict['sequenceList'][i])
                pdbNamesWithChainsIn.append(structureDict['pdbNamesWithChainsList'][i])
        structureDict['sequencesInCath'] = sequencesInCath
        structureDict['pdbNamesWithChainsIn'] = pdbNamesWithChainsIn
        if len(pdbNamesWithChainsIn) >= 1:
            inCath.append(fullpdbName)
        else:
            notInCath.append(fullpdbName)
    return inCath, notInCath

def countInCath(cath_df, structuresDicts):
    setOfchainsNames = set(cath_df['chain'])
    cnt = 0
    inCath = []
    notInCath = []
    for fullpdbName, structureDict in structuresDicts.items():
        sequencesInCath = []
        pdbNamesWithChainsIn = []
        isInCath = False
        for i in range(len(structureDict['pdbNamesWithChainsList'])):
            if structureDict['pdbNamesWithChainsList'][i] in setOfchainsNames:
                isInCath = True
                break
        if isInCath:
            cnt += 1
            inCath.append(fullpdbName)
        else:
            notInCath.append(fullpdbName)
    return inCath, notInCath, cnt

def neighbor_mat(df, nameList, seqList, columns_number):
    """
    :param df: cath data frame as it return from the func make_cath_df
    :param lst: list of chains
    :param columns_number: the number of columns to consider with the cath classification not include the cath domain name
    :return: matrix. mat[i][j] == 1 if there is connection between chain i and chain j
    """
    # generate the graph using CATH.
    cath_columns = ["n" + str(i) for i in range(1, columns_number + 1)]
    not_in_cath = set()
    n = len(nameList)
    mat = np.zeros((n, n))
    for i in range(n):
        for j in range(i + 1, n):
            similarity = df[df['chain'].isin([nameList[i], nameList[j]])]
            if len(similarity['chain'].unique()) == 1:
                if (similarity['chain'].unique()[0] == nameList[i]):
                    not_in_cath.add(nameList[j])
                else:
                    not_in_cath.add(nameList[i])
            else:
                similarity = similarity.groupby(by=cath_columns)
                for name, group in similarity:
                    if len(group['chain'].unique()) == 2:
                        mat[i][j] = mat[j][i] = 1
                        break
    # calculate the sequence identity
    for i in range(n):
        for j in range(i + 1, n):
            if (nameList[i] in not_in_cath):
                score = calculate_identity(seqList[i], seqList[j])
                mat[i][j] = mat[j][i] = 1
    return mat

def comapreClassifications(c1, c2):
    for i in range(len(c1)):
        if c1[i] != c2[i]:
            return False
    return True

def isSimiliarChains(structureDict1, structureDict2):
    connected = False
    for k in range(len(structureDict1['pdbNamesWithChainsList'])):
        for l in range(len(structureDict2['pdbNamesWithChainsList'])):
            if structureDict1['inOrNotInCathList'][k] and structureDict2['inOrNotInCathList'][l]:  # both chains in cath
                allClassifications1 = structureDict1['classificationsLists'][k]
                allClassifications2 = structureDict2['classificationsLists'][l]
                for c1 in allClassifications1:
                    for c2 in allClassifications2:
                        if comapreClassifications(c1, c2):
                            connected = True
            else:  # at least one of the chains not in cath
                if calculate_identity(structureDict1['sequenceList'][k], structureDict2['sequenceList'][l]) > 0.5:
                    connected = True
    return connected

def neighbor_mat_new(structuersDictionaries):
    """
    :param df: cath data frame as it return from the func make_cath_df
    :param lst: list of chains
    :param columns_number: the number of columns to consider with the cath classification not include the cath domain name
    :return: matrix. mat[i][j] == 1 if there is connection between chain i and chain j
    """
    # generate the graph using CATH.
    n = len(structuersDictionaries)
    print(n)
    structuersDictionariesValues = [structuersDictionaries[key] for key in structuersDictionaries.keys()]
    mat = np.zeros((n, n))
    for i in range(n):
        for j in range(i, n):
            structureDict1 = structuersDictionariesValues[i]
            structureDict2 = structuersDictionariesValues[j]
            if isSimiliarChains(structureDict1, structureDict2):
                mat[i][j] = mat[j][i] = 1
    return mat

def createRelatedChainslist(numberOfComponents, labels):
    """
    :param numberOfComponents: number of component = x => 0<=label values<x
    :param labels: labels
    :return: RelatedChainslist: RelatedChainslist[i] = list of all the chain index's which has the label i
    """
    relatedChainsLists = [[] for _ in range(numberOfComponents)]
    for i in range(len(labels)):
        relatedChainsLists[labels[i]].append(i)
    return relatedChainsLists

def createClusterSizesList(relatedChainslists, sizeList):
    """
    :param relatedChainslists:  relatedChainslist[i] = list of all the chain index's which has the label i
    :param relatedChainslists:  sizeList- list of all the chains's size
    :return: list of tuples (clusterIndex,size)
    """
    clusterSizes = []
    for i in range(len(relatedChainslists)):
        my_sum = 0
        for index in relatedChainslists[i]:
            my_sum += sizeList[index]
        clusterSizes.append((i, my_sum))
    return clusterSizes

def divideClusters(clusterSizes):
    """
    :param clusterSizes: list of tuples (clusterIndex,size)
    :return:  sublists,sublistsSum
    divide the list into 5 sublists such that the sum of each cluster sizes in the sublist is as close as possible
    """
    sublists = [[] for i in range(5)]
    sublistsSum = [0 for i in range(5)]
    clusterSizes.sort(reverse=True, key=lambda x: x[1])  # Sort the clusters by size descending order.
    for tup in clusterSizes:
        min_cluster_index = sublistsSum.index(min(sublistsSum))  # find the cluster with the minimal sum
        sublistsSum[min_cluster_index] += tup[1]
        sublists[min_cluster_index].append(tup[0])
    return sublists, sublistsSum

def clusterToChainList(clusterId, relatedChainsLists, nameList):
    """
    :param clusterId: list of chain indexs
    :param relatedChainsLists: relatedChainslist[i] = list of all the chain index's in cluster i
    :param  nameList = list of all the chains's name in the file
    :return: chainList = list of all chain names in the cluster
    """
    cluster = relatedChainsLists[clusterId]  # get the chains in the cluster
    chainList = [nameList[i] for i in cluster]
    return chainList

def sublistsToChainLists(sublists, relatedChainsLists, nameList):
    """
    :param sublists: sublists[i] = all the clusters in sublist i
    :param relatedChainsLists: relatedChainslist[i] = list of all the chain index's in cluster i
    :return: chainLists: ChainLists[i] = list of all the chains in cluster i
    """
    chainLists = [[] for i in range(len(sublists))]
    for i in range(len(sublists)):
        for clusterId in sublists[i]:
            chainLists[i] += clusterToChainList(clusterId, relatedChainsLists, nameList)
    return chainLists

def chainListsToChainIndexDict(chainLists):
    """
    :param chainLists: chainLists[i] = list of all the chains in cluster i
    :return: chainDict: chainDict[chainName] = index of chain cluster(i if chain in ChainLists[i])
    """
    chainDict = {}
    for i in range(len(chainLists)):
        for chainName in chainLists[i]:
            chainDict[chainName] = i
    return chainDict

def dividePSSM(chainDict):
    """
    :param chainDict: chainDict[chainName] = index of chain cluster(i if chain in ChainLists[i])
    create len(chainLists) txt files. the i txt file contains the chains in chainLists[i]
    """
    filesList = [open("PSSMFiles\\PSSM{}.txt".format(i), 'w') for i in range(5)]
    pssmFile = open("FullPssmContent", 'r')
    lines = pssmFile.readlines()
    fillIndex = -1  # fillIndex = i -> we now write to PSSMi.txt
    for line in lines:
        if line[0] == '>':  # header line
            fillIndex = chainDict[line[1:len(line)-1]]
        filesList[fillIndex].write(line)
    for i in range(5):
        filesList[i].close()
    pssmFile.close()

def calclulateIndexesOfScc(homologousLabels, sccNumber):
    indexes = np.where(homologousLabels == sccNumber)
    return indexes[0]

def calculateRatioFromIndexes(matHomologous, indexes):
    selected_matrix = matHomologous[np.ix_(indexes, indexes)]
    avarageNumberOfEdgesForStructure = (np.sum(selected_matrix) / len(indexes))
    ratio = avarageNumberOfEdgesForStructure / len(indexes)
    return ratio

def calculateHomologousRatioForSCC(homologousLabels, matHomologous, sccNumber):
    indexes = calclulateIndexesOfScc(homologousLabels, sccNumber)
    return calculateRatioFromIndexes(matHomologous, indexes)

def calculateRatioForFold(homologousLabels, matHomologous, sublists, foldNum):
    indexesLists = [calclulateIndexesOfScc(homologousLabels, sccNumber) for sccNumber in sublists[foldNum]]
    totalIndexes = np.concatenate(indexesLists)
    ratio = calculateRatioFromIndexes(matHomologous, totalIndexes)
    return ratio

cath_df = make_cath_df_new("cath_b.20230204.txt", 4)
namesList, sizesList, sequenceList, fullNamesList, pdbNamesWithChainsLists = listCreation("FullPssmContent")
structuresDicts = createDictionaries(namesList, sizesList, sequenceList, fullNamesList, pdbNamesWithChainsLists)
inCath, notInCath, cnt = countInCath(cath_df, structuresDicts)

# print(structuresDicts)
print(len(structuresDicts))
print(cnt)
print(inCath)
print(notInCath)
# print(cath_df)
findChainsInCath(cath_df, structuresDicts)
addClassificationsForDict(cath_df, structuresDicts, 4)
matHomologous = neighbor_mat_new(structuresDicts)
graphHomologous = csr_matrix(matHomologous)
homologous_components, homologousLabels = connected_components(csgraph=graphHomologous, directed=False,
                                                               return_labels=True)
print(namesList)
print(sizesList)
print(sum(sizesList))
print(homologous_components)
print(homologousLabels)
print("Done2")
relatedChainsLists = createRelatedChainslist(homologous_components, homologousLabels)
print("Done3")
clusterSizes = createClusterSizesList(relatedChainsLists, sizesList)
print("Done4")
sublists, sublistsSum = divideClusters(clusterSizes)
print("Done5")

print(relatedChainsLists)
print(clusterSizes)
print(sublists)
print(sublistsSum)

chainLists = sublistsToChainLists(sublists, relatedChainsLists, fullNamesList)
chainDict = chainListsToChainIndexDict(chainLists)
print(chainLists)
print(chainDict)
pickleDirPath = 'C:\\Users\\omriy\\UBDAndScanNet\\UBDModel'
with open(pickleDirPath + "\\receptorsFoldsDict.pkl" , "wb") as f:
    # pickle the list to the file
    pickle.dump(chainDict, f)

# print(calculateHomologousRatioForSCC(homologousLabels, matHomologous, 0))
# for i in range(5):
#     print("avarage ratio of fold : ", i + 1)
#     print(calculateRatioForFold(homologousLabels, matHomologous, sublists, i))
# dividePSSM(chainDict)


from django.shortcuts import render, redirect
from django.views.decorators.csrf import csrf_exempt
from django.contrib import messages
from django.contrib.auth.decorators import login_required

from .forms import UserRegistrationForm, UserUpdateForm
from .user_services import create_customer__ordered_plan_list, get_ordered_plan_list

@csrf_exempt
def register(request):
    if request.method == 'POST':
        form = UserRegistrationForm(request.POST)
        if form.is_valid():
            form.save()
            create_customer__ordered_plan_list(form.cleaned_data.get('username'))
            messages.add_message(request, messages.SUCCESS, 'Your account was successfully created!')
            return redirect('login')
    form = UserRegistrationForm()
    context = {
        'form': form
    }
    return render(request, 'users/register.html', context=context)

@login_required
def account(request):
    u_form = UserUpdateForm(instance=request.user)
    if request.method == 'POST':
        u_form = UserUpdateForm(request.POST, instance=request.user)
        if u_form.is_valid():
            u_form.save()
            messages.add_message(request, messages.SUCCESS, 'Your account info was successfully updated!')
            return redirect('account')

    plan_list = get_ordered_plan_list(request.user)
    context = {
        'u_form': u_form,
        'plans_list': plan_list
    }

    return render(request, 'users/account.html', context=context)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

from sklearn.preprocessing import LabelEncoder, MinMaxScaler
import numpy as np
import pandas as pd
import cv2
import os
import h5py
import mahotas
import pickle

# In[2]:

train_labels = [i for i in os.listdir('flwfolder3/jpg') if '.txt' not in i]
train_labels

# In[3]:

train_labels.sort()

# In[4]:

train_labels

# In[5]:

X = []
y = []
for label in train_labels:
    dir = os.path.join('flwfolder2/jpg', label)
    current_label = label
    print('Current Folder: ' + current_label)
    for x in range(1, 81):
        print('Current Image: ' + str(x))
        file = os.path.join(dir, str(x) + '.jpg')
        image = cv2.imread(file)
        try:
            image = cv2.resize(image,(500,500))
            image_hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)
            hist = cv2.calcHist([image], [0, 1, 2], None, [8, 8, 8], [0, 256, 0, 256, 0, 256])
            cv2.normalize(hist, hist)
            histogram_features = hist.flatten()
            image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
            hue_features = cv2.HuMoments(cv2.moments(image_gray)).flatten()
            haralick_features = mahotas.features.haralick(image_gray).mean(axis=0)
            current_features = np.hstack([histogram_features, hue_features, haralick_features])
            X.append(current_features)
            y.append(current_label)
        except:
            break

# In[ ]:

np.shape(X)

# In[ ]:

np.shape(y)

# In[ ]:

le = LabelEncoder()
y = le.fit_transform(y)

# In[ ]:

y

# In[ ]:

mms = MinMaxScaler()
X = mms.fit_transform(X)

# In[ ]:

X

# In[ ]:

h5_X = h5py.File('data/X.h5', 'w')
h5_X.create_dataset('features', data=np.array(X))

# In[ ]:

h5_y = h5py.File('data/y.h5', 'w')
h5_y.create_dataset('targets', data=np.array(y))

# In[ ]:

h5_X.close()
h5_y.close()

# In[ ]:

with open('data/le.h5', 'wb') as f:
    pickle.dump(le, f)

# In[ ]:

with open('data/mms.h5', 'wb') as f:
    pickle.dump(mms, f)

# In[ ]:


import sys
from PyQt5.QtWidgets import (
    QApplication, QMainWindow, QWidget, QMessageBox,
    QGridLayout, QPushButton, QLabel, QLineEdit, QTextEdit,
    QListWidget
)
from PyQt5.QtCore import Qt, pyqtSignal, pyqtSlot #, QString

from trading_agent.pyqt5.kiwoom_api import KiwoomAPI

# from hgoldfish.utils import eventlet
from trading_agent.eventlet_pyqt.hgoldfish.utils import eventlet

class KiwoomAgent(QMainWindow):
    # OnEventConnect = pyqtSignal(int)

    def __init__(self, qtapp, flask_app, socketio):
        super().__init__()
        self.operations = eventlet.GreenletGroup()

        self.qtapp = qtapp
        self.flask_app = flask_app
        self.socketio = socketio
        self.initUI()

        self.api = KiwoomAPI(agent=self)

    def initUI(self):
        frame = QWidget(self)
        self.setCentralWidget(frame)
        grid = QGridLayout()
        grid.setSpacing(10)
        frame.setLayout(grid)

        self.statusBar() #.showMessage('Ready')

        # self.input_edit = QLineEdit(frame)
        self.debug_list = QListWidget(frame)

        login_btn = QPushButton("로그인", frame)
        # current_price_button = QPushButton("종목 현재가", frame)
        # test_btn = QPushButton("테스트", frame)
        quit_btn = QPushButton("종료", frame)

        login_btn.clicked.connect(self.login)
        # current_price_button.clicked.connect(self.get_current_price)
        # test_btn.clicked.connect(self.test)
        quit_btn.clicked.connect(self.quit)

        # grid.addWidget(self.input_edit, 2, 0)
        grid.addWidget(self.debug_list, 4, 0, 8, 0)

        grid.addWidget(login_btn, 13, 0)
        # grid.addWidget(current_price_button, 14, 0)
        # grid.addWidget(test_btn, 15, 0)
        grid.addWidget(quit_btn, 16, 0)

        self.setGeometry(0, 0, 480, 480)  # x, y, w, h
        self.setWindowTitle('Trading Agent for Kiwoom')
        # self.setWindowIcon(QIcon('web.png'))
        self.show()

    def closeEvent(self, event):
        reply = QMessageBox.question(self, 'Message', "Are you sure to quit?", QMessageBox.No|QMessageBox.Yes, QMessageBox.Yes)

        if reply == QMessageBox.Yes:
            self.socketio.stop()
            # event.accept()
        else:
            event.ignore()

    def keyPressEvent(self, e):
        if e.key() == Qt.Key_Escape:
            self.close()

    def quit(self):
        self.socketio.stop()
        # self.qtapp.quit()

    def poweroff(self):
        self.socketio.stop()
        # self.qtapp.quit()

    def get_platform(self):
        platforms = {
            'linux1' : 'Linux',
            'linux2' : 'Linux',
            'darwin' : 'OS X',
            'win32' : 'Windows'
        }
        if sys.platform not in platforms:
            return sys.platform

        return platforms[sys.platform]

    def test(self):
        print('In test()', self, self.socketio)
        self.socketio.emit('SSE', {'Hello': 'TEST'})
        self.socketio.send('Blah Blah', namespace='/')
        # self.socketio.sleep(0)
        # pass
        # self.OnEventConnect.emit(1)

    def login(self):
        if self.api.getConnectState() == 1:
            # self.socketio.emit('authentication', {'status': 'logged_in'})
            return True
        else:
            ret = self.api.commConnect()
            return False # not connected yet

    def getConnectState(self):
        if self.api.getConnectState() == 1:
            tags = ["ACCNO", "USER_ID", "USER_NAME"]
            results = {tag: self.api.getLoginInfo(tag) for tag in tags}
            self.socketio.emit('basic-info', results)
            return True
        else:
            return False

    def handleConnect(self, errCode):
        if errCode == 0:  # 0: No Error, others: Error
            state = self.api.getConnectState()
            if state == 1: # 1: 연결 완료, 0: 미연결
                self.socketio.emit('authentication', {'status': 'logged_in'})

                tags = ["ACCNO", "USER_ID", "USER_NAME"]
                # results = list(map(lambda tag: self.api.getLoginInfo(tag), tags))
                results = {tag: self.api.getLoginInfo(tag) for tag in tags}
                self.socketio.emit('basic-info', results)
                # print('Login Info', results)
                return

        self.socketio.emit('authentication', {'status': 'logged_out'})
        self.statusBar().showMessage('Logged out')

    def check_balance(self, account_no):
        self.api.setInputValue('계좌번호', account_no)
        self.api.setInputValue('비밀번호', '')
        self.api.setInputValue('상장폐지조회구분', '0')
        self.api.setInputValue('비밀번호입력매체구분', '00')

        ret = self.api.commRqData('계좌평가현황요청', 'opw00004', 0, '0001')
        return ret

    def handleCheckBalance(self, trCode, rQName, scrNo, recordName):
        itemNames = ["예수금", "D+2추정예수금", "유가잔고평가액", "예탁자산평가액", "추정예탁자산", "누적손익율"]
        results = {itemName: self.api.getCommData(trCode, rQName, recordName, itemName) for itemName in itemNames}
        self.socketio.emit('balance-info', results)
        # print('계좌평가현황요청', results)
        self.handleGetAssets(trCode, rQName, scrNo, recordName)

    def get_assets(self, account_no):
        self.api.setInputValue('계좌번호', account_no)
        self.api.setInputValue('비밀번호', '')
        self.api.setInputValue('상장폐지조회구분', '0')
        self.api.setInputValue('비밀번호입력매체구분', '00')

        ret = self.api.commRqData('보유주식요청', 'opw00004', 0, '0001')
        return ret

    def handleGetAssets(self, trCode, rQName, scrNo, recordName):
        itemNames = ["종목코드", "종목명", "보유수량", "평균단가", "현재가"]
        results = []

        cnts = self.api.getRepeatCnt(trCode, rQName)
        for idx in range(0, cnts):
            results.append({itemName: self.api.getCommData(trCode, rQName, idx, itemName) for itemName in itemNames})

        if len(results) > 0:
            self.socketio.emit('assets-info', results)
            # print('보유주식요청', results)

    def get_condition_equations(self):
        ret = self.api.getConditionLoad()
        return ret

    def handleGetConditionNameList(self):
        conditionList = self.api.getConditionNameList()
        conditionList = conditionList[:-1].split(";")
        # print('조건검색명', conditionList)

        results = []
        for idx in range(0, len(conditionList)):
            item = conditionList[idx].split("^")
            results.append({"인덱스": item[0], "조건명": item[1]})

        if len(results) > 0:
            self.socketio.emit('equations-info', results)

    def get_current_price(self, stock_code):
        val = self.input_edit.text()
        if val != "":
            stock_code = val
        else:
            stock_code = '005930'

        self.api.setInputValue("종목코드", stock_code)

        # rQName = '주식기본정보'.encode('utf-8')
        # trCode = 'opt10001'
        # prevNext = 0
        # screenNo = '0001'
        ret = self.api.commRqData("주식기본정보", "opt10001", 0, "0001");