import fcntl
import json
import os
import socket
import select
import struct
import sys
import termios
import threading
import time
import glob
import platform
import traceback
import errno
import re
import logging
import logging.handlers

logger = logging.getLogger('candy_board_qws')
logger.setLevel(logging.INFO)
handler = logging.handlers.SysLogHandler(address='/dev/log')
logger.addHandler(handler)
formatter = logging.Formatter('[candy_board_qws] %(module)s.%(funcName)s: %(message)s')
handler.setFormatter(formatter)

# SerialPort class was imported from John Wiseman's
# https://github.com/wiseman/arduino-serial/blob/master/arduinoserial.py

# Map from the numbers to the termios constants (which are pretty much
# the same numbers).

BPS_SYMS = {
    115200:   termios.B115200,
    460800:   4100,
    # 230400:   termios.B230400,
    # 57600:    termios.B57600,
    # 38400:    termios.B38400,
    # 19200:    termios.B19200,
    # 9600:     termios.B9600,
    # 921600:   4103
    }

# Indices into the termios tuple.

IFLAG = 0
OFLAG = 1
CFLAG = 2
LFLAG = 3
ISPEED = 4
OSPEED = 5
CC = 6

def bps_to_termios_sym(bps):
    return BPS_SYMS[bps]

CGREG_STATS = [
    "Unregistered",
    "Registered",
    "Searching",
    "Denied",
    "Unknown",
    "Roaming"
]

# For local debugging:
# import candy_board_qws
# serial = candy_board_qws.SerialPort("/dev/ttyUSB2", 115200)
# server = candy_board_qws.SockServer(
#     "1.0.0",
#     "/var/run/candy-board-service.sock", serial)
# server.debug = True
# server.apn_ls()

# looking for a modem serial port
# import candy_board_qws
# candy_board_qws.SerialPort.resolve_modem_port()

class SerialPort(object):

    def __init__(self, serialport, bps):
        """Takes the string name of the serial port
        (e.g. "/dev/tty.usbserial","COM1") and a baud rate (bps) and
        connects to that port at that speed and 8N1. Opens the port in
        fully raw mode so you can send binary data.
        """
        self.fd = os.open(serialport, os.O_RDWR | os.O_NOCTTY | os.O_NDELAY)
        attrs = termios.tcgetattr(self.fd)
        bps_sym = bps_to_termios_sym(int(bps))
        # Set I/O speed.
        attrs[ISPEED] = bps_sym
        attrs[OSPEED] = bps_sym

        # 8N1
        attrs[CFLAG] &= ~termios.PARENB
        attrs[CFLAG] &= ~termios.CSTOPB
        attrs[CFLAG] &= ~termios.CSIZE
        attrs[CFLAG] |= termios.CS8
        # No flow control
        attrs[CFLAG] &= ~termios.CRTSCTS

        # Turn on READ & ignore contrll lines.
        attrs[CFLAG] |= termios.CREAD | termios.CLOCAL
        # Turn off software flow control.
        attrs[IFLAG] &= ~(termios.IXON | termios.IXOFF | termios.IXANY)

        # Make raw.
        attrs[LFLAG] &= ~(termios.ICANON | termios.ECHO |
                          termios.ECHOE | termios.ISIG)
        attrs[OFLAG] &= ~termios.OPOST

        # It's complicated--See
        # http://unixwiz.net/techtips/termios-vmin-vtime.html
        attrs[CC][termios.VMIN] = 0
        attrs[CC][termios.VTIME] = 20
        termios.tcsetattr(self.fd, termios.TCSANOW, attrs)

        self.ping()

    def available(self):
        return True

    def read_until(self, until):
        buf = ""
        done = False
        cnt = 0
        while not done:
            try:
                n = os.read(self.fd, 1).decode()
            except UnicodeDecodeError:
                n = '.'
            if n == '':
                if cnt > 200:
                    buf = None
                    break
                cnt = cnt + 1
                # FIXME: Maybe worth blocking instead of busy-looping?
                time.sleep(0.01)
                continue
            buf = buf + n
            if n == until:
                done = True
        return buf

    def read_line(self):
        try:
            return self.read_until("\n").strip()
        except OSError:
            return None

    def write(self, str):
        os.write(self.fd, str.encode())

    def write_byte(self, byte):
        os.write(self.fd, byte)

    def close(self):
        try:
            os.close(self.fd)
        except OSError:
            pass

    def ping(self, loop=3):
        ret = None
        for i in (0, loop):
            self.write("AT\r")
            time.sleep(0.1)
            line = self.read_line()
            if line is None:
                time.sleep(0.1)
                continue
            else:
                ret = ''
                while line is not None:
                    ret = ret + line + '\r'
                    line = self.read_line()
                break
        return ret

    @staticmethod
    def resolve_modem_baudrate(p):
        if platform.system() != 'Linux':
            return None

        for bps in BPS_SYMS.keys():
            port = SerialPort.open_serial_port(p, bps)
            if port is None:
                continue
            ret = port.ping()
            if ret is None:
                port.close()
                continue
            if "OK" in ret:
                port.close()
                return bps
            port.close()

        return None

    @staticmethod
    def open_serial_port(p, bps):
        for i in (0, 3):
            port = None
            try:
                port = SerialPort(p, bps)
                return port
            except Exception:
                if port:
                    try:
                        port.close()
                    except Exception:
                        pass
                port = None
                time.sleep(0.1)
                pass
        return None

    @staticmethod
    def resolve_modem_port(bps=115200):
        if platform.system() != 'Linux':
            return None

        for t in [
                    '/dev/QWS.*.MODEM',
                    '/dev/ttySC1'
                ]:
            for p in sorted(glob.glob(t)):
                port = SerialPort.open_serial_port(p, bps)
                if port is None:
                    continue
                ret = port.ping()
                if ret is None:
                    port.close()
                    continue
                if "OK" in ret:
                    port.close()
                    return p
                port.close()

        return None

class LazySerialPort:
    def __init__(self, serialport, bps):
        self.serial = None
        self.serialport = serialport
        self.bps = bps

    def _serial(self):
        if self.serial is None:
            self.serial = SerialPort(self.serialport, self.bps)
        return self.serial

    def available(self):
        try:
            self._serial()
            return True
        except OSError:
            return False

    def read_line(self):
        return self._serial().read_line()

    def write(self, str):
        return self._serial().write(str)

    def write_byte(self, byte):
        return self._serial().write_byte(byte)

    def close(self):
        if self.serial is None:
            return
        try:
            return self.serial.close()
        finally:
            self.serial = None

class SockServer(threading.Thread):
    def __init__(self, version,
                 sock_path="/var/run/candy-board-service.sock", serial=None):
        super(SockServer, self).__init__()
        self.version = version
        self.sock_path = sock_path
        self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        self.serial = serial
        self.debug = False

    def recv(self, connection, size):
        ready, _, _ = select.select([connection], [], [], 5)
        if ready:
            return connection.recv(size)
        else:
            raise IOError("recv Timeout")

    def run(self):
        self.sock.bind(self.sock_path)
        self.sock.listen(128)
        header_packer = struct.Struct("I")
        print("Listening to the socket[%s]...." % self.sock_path)

        while True:
            try:
                logger.debug("Waiting for an incoming request")
                connection, client_address = self.sock.accept()
                connection.setblocking(0)

                # request
                logger.debug("Request has arrived!")
                header = self.recv(connection, header_packer.size)
                size = header_packer.unpack(header)
                logger.debug("Size:%d" % size)
                unpacker_body = struct.Struct("%is" % size)
                cmd_json = self.recv(connection, unpacker_body.size)
                logger.debug("Body:[%s]" % cmd_json)
                cmd = json.loads(cmd_json)

                # response
                logger.debug("Performing a command")
                message = self.perform(cmd)
                logger.debug("Command done!")
                if message:
                    size = len(message)
                else:
                    size = 0
                packed_header = header_packer.pack(size)
                connection.sendall(packed_header)
                logger.debug("Size:%d" % size)
                if size > 0:
                    packer_body = struct.Struct("%is" % size)
                    packed_message = packer_body.pack(message.encode('utf-8'))
                    logger.debug("Body:[%s]" % message)
                    connection.sendall(packed_message)

            except socket.error as e:
                if isinstance(e.args, tuple):
                    if e[0] == errno.EPIPE:
                        continue
                logger.error("Socket Error: %s" %
                                      (''.join(traceback
                                       .format_exception(*sys.exc_info())[-2:])
                                       .strip().replace('\n', ': '))
                                      )

            except Exception:
                logger.error("Unexpected Error: %s" %
                                      (''.join(traceback
                                       .format_exception(*sys.exc_info())[-2:])
                                       .strip().replace('\n', ': '))
                                      )

            finally:
                if 'connection' in locals():
                    connection.close()

    def perform(self, cmd):
        if self.serial is None or self.serial.available() is False:
            return self.error_message("Modem is not ready")
        try:
            if cmd['category'][0] == '_':
                raise AttributeError()
            m = getattr(self.__class__,
                        "%s_%s" % (cmd['category'], cmd['action']))
            return m(self, cmd)
        except AttributeError:
            return self.error_message("Unknown Command")
        except KeyError:
            return self.error_message("Invalid Args")
        except OSError:  # I/O Error
            return self.error_message("Modem is not ready")
        except Exception:
            return self.error_message("Unexpected error: %s" %
                                      (''.join(traceback
                                       .format_exception(*sys.exc_info())[-2:])
                                       .strip().replace('\n', ': '))
                                      )

    def error_message(self, msg):
        return json.dumps({"status": "ERROR", "result": msg})

    def read_line(self):
        line = self.serial.read_line()
        if self.debug:
            print("[modem:IN] => [%s]" % line)
        return line

    def send_at(self, cmd, ok="OK"):
        line = "%s\r" % cmd
        if self.debug:
            print("[modem:OUT] => [%s]" % line)
        self.serial.write(line)
        time.sleep(0.1)
        result = ""
        status = None
        count = 0
        while True:
            line = self.read_line()
            if line is None:
                if status is not None or count > 650:
                    break
                time.sleep(0.1)
                count = count + 1
            elif line == cmd:
                continue
            elif line == ok or \
                    line == "ERROR" or \
                    line == "NO DIALTONE" or \
                    line == "NO CARRIER" or \
                    line.startswith("+CME ERROR"):
                status = line
            elif line is None:
                status = "UNKNOWN"
            elif line.strip() != "":
                result += line + "\n"
        if self.debug:
            print("cmd:[%s] => status:[%s], result:[%s]" %
                  (cmd, status, result))
        return (status, result.strip())

    def _apn_ls(self):
        status, result = self.send_at("AT+CGDCONT?")
        apn_list = []
        id_name_list = []
        if status == "OK":
            try:
                id_name_list = list(map(lambda e: e[10:].split(",")[0] + "," +
                                   re.sub('"', '', e[10:].split(",")[2]),
                                   result.split("\n")))
                status, result = self.send_at("AT$QCPDPP?")
                creds_list = []
                if status == "OK":
                    def to_user(e):
                        if len(e) > 2:
                            return re.sub('"', '', e[2])
                        return ''
                    creds_list = list(map(lambda e: to_user(e),
                                     list(map(lambda e: e[9:].split(","),
                                         result.split("\n")))))
            except IndexError:
                pass
        for i in range(len(id_name_list)):
                id_name = id_name_list[i].split(",")
                apn = {
                    'apn_id': id_name[0],
                    'apn': id_name[1]
                }
                if len(creds_list) > i:
                    apn['user'] = creds_list[i]
                apn_list.append(apn)
        message = {
            'status': status,
            'result': {
                'apns': apn_list
            }
        }
        return message

    def apn_ls(self, cmd={}):
        return json.dumps(self._apn_ls())

    def apn_set(self, cmd={}):
        (name, user_id, password) = (cmd['name'], cmd['user_id'],
                                     cmd['password'])
        if 'type' in cmd:
            t = cmd['type']
            if t == 'ipv4':
                pdp = 'IP'
            elif t == 'ipv6':
                pdp = 'IPV6'
            else:
                pdp = 'IPV4V6'
        else:
            pdp = 'IP'

        apn_id = "1"
        if 'id' in cmd:
            apn_id = cmd['id']
        status, result = self.send_at(("AT+CGDCONT=%s,\"%s\",\"%s\"," +
                                      ",0,0") % (apn_id, pdp, name))
        if status == "OK":
            status, result = self.send_at(("AT$QCPDPP=%s,3,\"%s\",\"%s\"") %
                                          (apn_id, password, user_id))
        message = {
            'status': status,
            'result': result
        }
        return json.dumps(message)

    def _apn_del(self, apn_id):
        # removes QCPDPP as well
        status, result = self.send_at(("AT+CGDCONT=%s") % apn_id)
        message = {
            'status': status,
            'result': result
        }
        return message

    def apn_del(self, cmd={}):
        apn_id = "1"
        if 'id' in cmd:
            apn_id = cmd['id']
        return json.dumps(self._apn_del(apn_id))

    def network_show(self, cmd={}):
        rssi = ""
        network = "UNKNOWN"
        rssi_desc = ""
        operator = "UNKNOWN"
        status, result = self.send_at("AT+CSQ")
        if status == "OK":
            rssi_level = int(result[5:].split(",")[0])
            if rssi_level == 0:
                rssi = "-113"
                rssi_desc = "OR_LESS"
            elif rssi_level == 1:
                rssi = "-111"
            elif rssi_level <= 30:
                rssi = "%i" % (-109 + (rssi_level - 2) * 2)
            elif rssi_level == 31:
                rssi = "-51"
                rssi_desc = "OR_MORE"
            elif rssi_level == 100:
                rssi = "-116"
                rssi_desc = "OR_LESS"
            elif rssi_level == 101:
                rssi = "-115"
            elif rssi_level <= 190:
                rssi = "%i" % (-114 + (rssi_level - 102))
            elif rssi_level == 191:
                rssi = "-25"
                rssi_desc = "OR_MORE"
            else:
                rssi_desc = "NO_SIGANL"
            status, result = self.send_at("AT+COPS?")
            try:
                operator = result.split(',')[2][1:-1]
            except IndexError:
                operator = "N/A"
            registration = {
                "cs": "N/A",
                "ps": "N/A",
                "eps": "N/A"
            }
            status, result = self.send_at("AT+CREG?")
            try:
                cs = int(result.split(",")[1])
                registration["cs"] = CGREG_STATS[cs]
            except IndexError:
                pass
            status, result = self.send_at("AT+CGREG?")
            try:
                ps = int(result.split(",")[1])
                registration["ps"] = CGREG_STATS[ps]
            except IndexError:
                pass
            status, result = self.send_at("AT+CEREG?")
            try:
                eps = int(result.split(",")[1])
                registration["eps"] = CGREG_STATS[eps]
            except IndexError:
                pass
            access = 'N/A'
            band = 'N/A'
            status, result = self.send_at("AT+QNWINFO")
            if status == 'ERROR':
                status, result = self.send_at("AT+QGBAND")
                try:
                    currentband = int(result.split(': ')[1])
                    if currentband == 1:
                        access, band = 'GSM', 'GSM 900'
                    elif currentband == 2:
                        access, band = 'GSM', 'GSM 1800'
                    elif currentband == 4:
                        access, band = 'GSM', 'GSM 850'
                    elif currentband == 8:
                        access, band = 'GSM', 'GSM 1900'
                    elif currentband == 16:
                        access, band = 'WCDMA', 'WCDMA 2100'
                    elif currentband == 32:
                        access, band = 'WCDMA', 'WCDMA 1900'
                    elif currentband == 64:
                        access, band = 'WCDMA', 'WCDMA 850'
                    elif currentband == 128:
                        access, band = 'WCDMA', 'WCDMA 900'
                    elif currentband == 256:
                        access, band = 'WCDMA', 'WCDMA 800'
                except IndexError:
                    pass
            else:
                try:
                    nwinfo = result.split(': ')[1].split(',')
                    access = nwinfo[0].replace('"', '')
                    band = nwinfo[2].replace('"', '')
                except IndexError:
                    pass
        message = {
            'status': status,
            'result': {
                'rssi': rssi,
                'rssiDesc': rssi_desc,
                'network': 'N/A',
                'operator': operator,
                'registration': registration,
                'access': access,
                'band': band
            }
        }
        return json.dumps(message)

    def network_deregister(self, cmd={}):
        status, result = self.send_at("AT+COPS=2")
        message = {
            'status': status,
            'result': result
        }
        return json.dumps(message)

    def network_register(self, cmd={}):
        operator = None
        if 'operator' in cmd:
            mode = '4' if 'auto' in cmd and cmd['auto'] else '1'
            operator = cmd['operator']
        if operator is None or operator == '':
            mode = '0'
        status, result = self.send_at(
            "AT+COPS=%s,2,%s"
            % (mode, operator))
        message = {
            'status': status,
            'result': {
                'mode': mode
            }
        }
        return json.dumps(message)

    def sim_show(self, cmd={}):
        state = "SIM_STATE_ABSENT"
        msisdn = ""
        imsi = ""
        iccid = ""
        status, result = self.send_at("AT+CIMI")
        if status == "OK":
            imsi = result
            state = "SIM_STATE_READY"
            status, result = self.send_at("AT+CNUM")
            if len(result) > 5:
                msisdn = re.sub('"', '', result[6:].split(",")[1])
            else:
                msisdn = ""
            status, result = self.send_at("AT+QCCID")
            if len(result) > 7:
                iccid = result.split(':')[1].strip()
            else:
                iccid = ""
        message = {
            'status': status,
            'result': {
                'msisdn': msisdn,
                'imsi': imsi,
                'iccid': iccid,
                'state': state
            }
        }
        return json.dumps(message)

    def _counter_show(self):
        """
        - Show TX/RX packet counter
        """
        status, result = self.send_at("AT+QGDCNT?")
        tx = '0'
        rx = '0'
        if status == "OK":
            txrx = result.split(':')[1].strip().split(',')
            tx = txrx[0]
            rx = txrx[1]
        message = {
            'status': status,
            'result': {
                'tx': tx,
                'rx': rx
            }
        }
        return message

    def _counter_reset(self):
        """
        - Reset packet counter
        """
        status, result = self.send_at("AT+QGDCNT=0")
        message = {
            'status': status,
            'result': result
        }
        return message

    def _imei_show(self):
        """
        - Show IMEI
        """
        status, result = self.send_at("AT+GSN")
        message = {
            'status': status,
            'result': result
        }
        return message

    def _timestamp_show(self):
        """
        - Show timestamp
        """
        status, result = self.send_at("AT+CCLK?")
        message = {
            'status': status,
            'result': result
        }
        return message

    def _functionality_show(self):
        """
        - Show phone functionality
        """
        status, result = self.send_at("AT+CFUN?")
        func = "Error"
        if status == "OK":
            func = result.split(':')[1].strip()
            if func == "0":
                func = "Minimum"
            elif func == "1":
                func = "Full"
            elif func == "4":
                func = "Disabled"
            else:
                func = "Anomaly"
        message = {
            'status': status,
            'result': {
                'functionality': func
            }
        }
        return message

    def modem_show(self, cmd={}):
        status, result = self.send_at("ATI")
        man = "UNKNOWN"
        mod = "UNKNOWN"
        rev = "UNKNOWN"
        imei = "UNKNOWN"
        func = "UNKNOWN"
        counter = None
        utc = None
        timezone_hrs = None
        if status == "OK":
            info = result.split("\n")
            man = info[0]
            mod = info[1]
            rev = info[2][10:]
            result = self._imei_show()
            if result['status'] == "OK":
                imei = result['result']
            result = self._counter_show()
            if result['status'] == "OK":
                counter = result['result']
            result = self._timestamp_show()
            if result['status'] == "OK":
                datelen = len(result['result'])
                if datelen == 29:
                    utc = result['result'][8:-4]
                    timezone_hrs = float(result['result'][-4:-1]) / 4
                elif datelen == 26:
                    utc = result['result'][8:-1]
                    timezone_hrs = 0.0
            result = self._functionality_show()
            func = result['result']['functionality']
        message = {
            'status': status,
            'result': {
                'manufacturer': man,
                'model': mod,
                'revision': rev,
                'imei': imei,
                'datetime': utc,
                'timezone': timezone_hrs,
                'functionality': func
            }
        }
        if counter:
            message['result']['counter'] = counter
        return json.dumps(message)

    def _parse_opts(self, cmd={}):
        opts = {}
        if 'opts' in cmd:
            try:
                opts = json.loads(cmd['opts'])
            except Exception:
                try:
                    entries = list(map(lambda e: e.split('='),
                                  cmd['opts'].split(',')))
                    for (k, v) in entries:
                        opts[k.strip()] = v.strip()
                except Exception:
                    pass
        return opts

    def modem_reset(self, cmd={}):
        """
        - opts counter=yes
            - Reset packet counter
        - no-opts or counter!=yes
            - Reset packet counter
            - Remove all APN
            - Reset Phone Functionality
        """
        result = 'counter'
        counter_reset_ret = self._counter_reset()
        status = counter_reset_ret['status']
        opts = self._parse_opts(cmd)
        if 'counter' not in opts or opts['counter'] != 'yes':
            result = ''
            apn_ls_ret = self._apn_ls()
            status = apn_ls_ret['status']
            if apn_ls_ret['status'] == "OK":
                apns = apn_ls_ret['result']['apns']
                for apn in apns:
                    self._apn_del(apn['apn_id'])
            status, qnvw_result = self.send_at(
                'AT+QNVW=4548,0,"0000400C00000210"')
            if status != "OK":
                result = qnvw_result
            status, qnvw_result = self.send_at(
                'AT+QNVW=930,0,"2A39380000030003002A3939000003000300233737370'
                '004000400000000000000000000"')
            if status != "OK":
                result = qnvw_result
            self._modem_clck_unlock(cmd)

        message = {
            'status': status,
            'result': result
        }
        return json.dumps(message)

    def modem_off(self, cmd={}):
        """
        PRIVATE COMMAND (not available from CLI)
        - Power off
        """
        status, result = self.send_at("AT+QPOWD", "POWERED DOWN")
        if status == "POWERED DOWN":
            status = "OK"
            result = ""
        message = {
            'status': status,
            'result': result
        }
        return json.dumps(message)

    def modem_init(self, cmd={}):
        """
        PRIVATE COMMAND (not available from CLI)
        - Enable automatic timezone update with NITZ
          to set modem RTC (if NW is capable)
          Enabled by default.
        - Reset Phone Functionality
        - Set baudrate (optional)
        - Reset packet counter (optional)
        """
        tz_update = "N/A"
        counter_reset_ret = "N/A"
        baudrate_ret = "N/A"
        if 'tz_update' not in cmd or cmd['tz_update'] is True:
            status, result = self.send_at("AT+CTZU?")
            if status == "OK":
                tz_update = "OK"
                for at in ["AT+COPS=2", "AT+CTZU=1", "AT+COPS=0"]:
                    status, result = self.send_at(at)
                    if status != "OK":
                        tz_update = "ERROR"
                        break
        if 'counter_reset' in cmd and cmd['counter_reset']:
            counter_reset_ret = self._counter_reset()['status']
        status, result_qnvw = self.send_at(
            'AT+QNVW=4548,0,"0000400C00000210"')
        if status != "OK":
            message = {
                'status': status,
                'result': result_qnvw,
                'cmd': 'AT+QNVW'
            }
            return json.dumps(message)
        self._modem_clck_unlock(cmd)
        if 'baudrate' in cmd:
            baudrate_ret, result = self.send_at("AT+IPR=%s" % cmd['baudrate'])
            if baudrate_ret != "OK":
                message = {
                    'status': baudrate_ret,
                    'result': result,
                    'cmd': 'baudrate'
                }
                return json.dumps(message)
        message = {
            'status': status,
            'result': {
                'counter_reset': counter_reset_ret,
                'baudrate': baudrate_ret
            }
        }
        return json.dumps(message)

    def _modem_clck_unlock(self, cmd={}):
        if 'pu' not in cmd or cmd['pu'] is False:
            self.send_at(
                'AT+CLCK="PU",0,"12341234"')

    def _gnss_config_uc2x(self, cmd={}):
        glonassenable = '0'
        glonassnmeatype = '0'
        if 'all' in cmd and cmd['all']:
            glonassenable = '1'
            glonassnmeatype = '7'

        status, result = self.send_at('AT+QGPSCFG="glonassenable",%s'
                                      % glonassenable)
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'glonassenable'
            }
            return json.dumps(message)
        status, result = self.send_at('AT+QGPSCFG="glonassnmeatype",%s'
                                      % glonassnmeatype)
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'glonassnmeatype'
            }
            return json.dumps(message)

    def _gnss_config_ec2x(self, cmd={}):
        config = '0'
        glonassnmeatype = '0'
        beidounmeatype = '0'
        galileonmeatype = '0'
        if 'all' in cmd and cmd['all']:
            config = '1'
            glonassnmeatype = '7'
            beidounmeatype = '3'
            galileonmeatype = '1'
        elif 'qzss' in cmd and cmd['qzss']:
            config = '2'
            glonassnmeatype = '7'
            beidounmeatype = '3'

        status, result = self.send_at('AT+QGPSCFG="gnssconfig",%s' % config)
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'gnssconfig'
            }
            return json.dumps(message)
        status, result = self.send_at('AT+QGPSCFG="glonassnmeatype",%s'
                                      % glonassnmeatype)
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'glonassnmeatype'
            }
            return json.dumps(message)
        status, result = self.send_at('AT+QGPSCFG="beidounmeatype",%s'
                                      % beidounmeatype)
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'beidounmeatype'
            }
            return json.dumps(message)
        status, result = self.send_at('AT+QGPSCFG="galileonmeatype",%s'
                                      % galileonmeatype)
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'galileonmeatype'
            }
            return json.dumps(message)

    def _gnss_config(self, cmd={}):
        status, result = self.send_at("ATI")
        if status == "OK":
            info = result.split("\n")
            if info[1] == 'UC20':
                return self._gnss_config_uc2x(cmd)
            else:
                return self._gnss_config_ec2x(cmd)

    def gnss_start(self, cmd={}):
        status, result = self.send_at('AT+QGPSCFG="gpsnmeatype",31')
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'gpsnmeatype'
            }
            return json.dumps(message)
        message_json = self._gnss_config(cmd)
        if message_json is not None:
            return message_json
        status, result = self.send_at('AT+QGPSCFG="nmeasrc",1')
        if status != "OK":
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'nmeasrc'
            }
            return json.dumps(message)
        status, result = self.send_at("AT+QGPS=1,30,50,0,1")
        if status == "OK":
            result = ""
        elif status == "+CME ERROR: 504":
            status = "OK"
        else:
            result = status
            status = "ERROR"
        message = {
            'status': status,
            'result': result,
        }
        return json.dumps(message)

    def gnss_status(self, cmd={}):
        status, result = self.send_at("AT+QGPS?")
        if status == "OK":
            gnssstate = result.split(':')[1].strip()
            if gnssstate == "0":
                session = 'stopped'
            elif gnssstate == "1":
                session = 'started'
        else:
            result = status
            status = "ERROR"
            message = {
                'status': status,
                'result': result,
                'cmd': 'QGPS?'
            }
            return json.dumps(message)

        qzss = 'N/A'
        status, result = self.send_at("ATI")
        if status == "OK":
            info = result.split("\n")
            if info[1] != 'UC20':
                status, result = self.send_at('AT+QGPSCFG="gnssconfig"')
                if status != "OK":
                    result = status
                    status = "ERROR"
                    message = {
                        'status': status,
                        'result': result,
                        'cmd': 'gnssconfig'
                    }
                    return json.dumps(message)
                gnssconfig = result.split(':')[1].split(',')[1].strip()
                if gnssconfig == '1' or gnssconfig == '2':
                    qzss = 'enabled'
                else:
                    qzss = 'disabled'

        message = {
            'status': status,
            'result': {
                'session': session,
                'qzss': qzss
            },
        }
        return json.dumps(message)

    def gnss_stop(self, cmd={}):
        status, result = self.send_at("AT+QGPSEND")
        if status == "OK":
            result = ""
        elif status == "+CME ERROR: 505":
            status = "OK"
        else:
            result = status
            status = "ERROR"
        message = {
            'status': status,
            'result': result,
        }
        return json.dumps(message)

    def gnss_locate(self, cmd={}):
        if 'format' in cmd and cmd['format']:
            format = str(cmd['format'])
        else:
            format = '2'
        status, result = self.send_at("AT+QGPSLOC=%s" % (format))
        if status == "OK":
            csv = result.split(':')[1].strip().split(',')
            if len(csv) < 11:
                message = {
                    'status': 'ERROR',
                    'result': 'Temporary I/O Error',
                }
                return json.dumps(message)
            latitude = csv[1]
            if format == '1':
                latitude = '%s %s' % (latitude, csv[2])
                del csv[2]
            elif format == '0':
                latitude = '%s %s' % (latitude[:-1], latitude[-1])
            longitude = csv[2]
            if format == '1':
                longitude = '%s %s' % (longitude, csv[3])
                del csv[3]
            elif format == '0':
                longitude = '%s %s' % (longitude[:-1], longitude[-1])
            altitude = float(csv[4])
            if format == '2':
                latitude = float(latitude)
                longitude = float(longitude)
            result = {
                'timestamp': '20%s-%s-%sT%s:%s:%s.000Z' %
                (
                    csv[9][4:6], csv[9][2:4], csv[9][0:2],
                    csv[0][0:2], csv[0][2:4], csv[0][4:6]
                ),
                'latitude': latitude,
                'longitude': longitude,
                'hdop': float(csv[3]),
                'altitude': altitude,
                'fix': '%sD' % csv[5],
                'cog': float(csv[6]),
                'spkm': float(csv[7]),
                'spkn': float(csv[8]),
                'nsat': int(csv[10])
            }
        else:
            code = status.split(':')
            if len(code) > 1:
                code = code[1].strip()
                status = "ERROR"
            else:
                code = code[0]
            if code == "516":
                result = "Not fixed yet"
            elif code == "502":
                result = "Invalid format"
            elif code == "505":
                result = "Session not started"
            else:
                result = code
        message = {
            'status': status,
            'result': result,
        }
        return json.dumps(message)

    def service_version(self, cmd={}):
        message = {
            'status': 'OK',
            'result': {
                'version': self.version,
            }
        }
        return json.dumps(message)


from django.conf.urls import url
from . import views
urlpatterns = [
	url(r'^$', views.index),
	url(r'^main$', views.loginreg),
	url(r'^register$', views.register),
	url(r'^logout$', views.logout),
	url(r'^login$', views.login),
	url(r'^dashboard$', views.dashboard),
	url(r'^additem$', views.additem),
	url(r'^add$', views.add),
	url(r'^wish_items/(?P<iid>\d+)$', views.showitem),
	url(r'^add/(?P<iid>\d+)$', views.addwish),
	url(r'^delete/(?P<iid>\d+)$', views.delete),
	url(r'^remove/(?P<iid>\d+)$', views.remove)
]


import pymongo
import json

persistent_client = pymongo.MongoClient("10.4.41.47", 27017)
persistent_zone = persistent_client["PersistentLandingZone"]
formated_zone = persistent_client['FormattedZone']
lookup = persistent_zone['lookup'].find({})
idealista = persistent_zone['idealista'].find({})

def get_lookup(lookup):
    for doc in lookup:
        if doc['filename'] == 'lookup_idealista_extended_2023-05-11-11:31:49.csv':
            lookupIdealista = doc['data']
        else:
            lookupOpenBCN = doc['data']
    return lookupIdealista,lookupOpenBCN

def formatted_idealista(idealista, Formated_zone, lookup ):
    propertyCodes = []
    duplicates = 0
    for document in idealista:
        data = document['data'][0]
        if data['propertyCode'] not in propertyCodes:
            propertyCodes.append(data['propertyCode'])
        else:
            duplicates +=1
            continue

        if 'district' in data.keys():
            district = data['district']
            if 'neighborhood' in data.keys():
                neighborhood = data['neighborhood']
                for neighborhoodLookup in lookup:
                    if neighborhoodLookup['neighborhood'] == neighborhood:

                        idN = neighborhoodLookup['neighborhood_id']
                        idD = neighborhoodLookup['district_id']

                        document['data'][0]['neighborhood_id'] = idN
                        document['data'][0]['district_id'] = idD
                        break
                continue

            for districtLookup in lookup:
                if districtLookup['district'] == district:

                    idD = neighborhoodLookup['district_id']

                    document['data'][0]['district_id'] = idD
                    break

    print(f'Removed {duplicates} duplicates')
lookupIdealista,lookupOpenBCN =  get_lookup(lookup)
formatted_idealista(idealista, formated_zone, lookupIdealista)


from acme import environment_loop
from acme import specs
from acme.agents.tf import actors
from acme.testing import fakes
import dm_env
import numpy as np
import sonnet as snt
import tensorflow as tf

from absl.testing import absltest

def _make_fake_env() -> dm_env.Environment:
  env_spec = specs.EnvironmentSpec(
      observations=specs.Array(shape=(10, 5), dtype=np.float32),
      actions=specs.DiscreteArray(num_values=3),
      rewards=specs.Array(shape=(), dtype=np.float32),
      discounts=specs.BoundedArray(
          shape=(), dtype=np.float32, minimum=0., maximum=1.),
  )
  return fakes.Environment(env_spec, episode_length=10)

class ActorTest(absltest.TestCase):

  def test_feedforward(self):
    environment = _make_fake_env()
    env_spec = specs.make_environment_spec(environment)

    network = snt.Sequential([
        snt.Flatten(),
        snt.Linear(env_spec.actions.num_values),
        lambda x: tf.argmax(x, axis=-1, output_type=env_spec.actions.dtype),
    ])

    actor = actors.FeedForwardActor(network)
    loop = environment_loop.EnvironmentLoop(environment, actor)
    loop.run(20)

  def test_recurrent(self):
    environment = _make_fake_env()
    env_spec = specs.make_environment_spec(environment)

    network = snt.DeepRNN([
        snt.Flatten(),
        snt.Linear(env_spec.actions.num_values),
        lambda x: tf.argmax(x, axis=-1, output_type=env_spec.actions.dtype),
    ])

    actor = actors.RecurrentActor(network)
    loop = environment_loop.EnvironmentLoop(environment, actor)
    loop.run(20)

if __name__ == '__main__':
  absltest.main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os

def CHK(return_code, *args):
    """
    Check if the return value of the invoked function returned an error.

    The Qmix DLL makes this pretty easy: All errors are indicated by
    negative return values.

    Parameters
    ----------
    return_code : int
        The code returned from a Qmix DLL function.
    args
        All arguments passed to the function ``funcname``.

    Returns
    -------
    return_code : int
        If no error occurred, the originally passed return value is
        returned.

    Raises
    ------
    RuntimeError
        If the DLL function returned an error code.

    """
    if return_code >= 0:
        return return_code
    else:
        from .error import QmixError  # We import here to avoid circularity with error.py
        e = QmixError(return_code)
        error_string = e.error_string
        msg = (error_string + ", Error number: " + str(e.error_number) +
               ", Error code: " + str(e.error_code))
        raise RuntimeError(msg)

def find_dll(dll_dir, dll_filename):
    """
    Try to find the specified DLL.

    If a DLL directory was given, try to find the DLL there.

    If no DLL directory was given, start looking for the DLL in the following
    order:

    - current working directory
    - global DLL search paths (this also respects the PATH
      environment variable)
    - default Qmix SDK installation directory

    Parameters
    ----------
    dll_dir : string or None
        The directory containing the DLLs. Pass `None` to look in the default locations.
    dll_filename : string
        The name of the DLL to find, including filename extension.

    Returns
    -------
    dll_path : string or None
        The path of the DLL, or `None` if the DLL could not be found.

    """
    if not dll_dir:
        # Check current directory.
        if os.path.exists(dll_filename):
            dll_path = dll_filename
            return dll_path

        # Check DLL search paths.
        import win32api
        try:
            win32api.LoadLibrary(dll_filename)
            dll_path = dll_filename
            return dll_path
        except Exception:
            pass

        # Check default installation directory.
        import appdirs
        dll_dir = appdirs.user_data_dir('QmixSDK', '')
        if os.path.exists(os.path.join(dll_dir, dll_filename)):
            dll_path = os.path.join(dll_dir, dll_filename)
            # Add DLL dir to PATH, otherwise we won't be able to load the DLL.
            os.environ['PATH'] += os.pathsep + dll_dir
            return dll_path

        return None
    else:
        if os.path.exists(os.path.join(dll_dir, dll_filename)):
            dll_path = os.path.join(dll_dir, dll_filename)
            # Add DLL dir to PATH, otherwise we won't be able to load the DLL.
            os.environ['PATH'] += os.pathsep + dll_dir
            return dll_path
        else:
            return None


from flask import Flask, request, render_template
from random import choice, sample

app = Flask(__name__)

predictions = ["You will come across a great fortune soon", "Prepare for a life changing moment", "Your day will be made by a stranger today",
"You might feel a little down today for no apparent reason", "You will spread happiness today"]

@app.route('/')
def index():
    return render_template('index.html')

@app.route("/predictions")
def give_prediction():
    name = request.args.get("name")
    show_prediction = request.args.get('show_prediction')
    random_predictions = choice(predictions)

    return render_template(
        'predictions.html',
        name=name,
        show_prediction=show_prediction,
        random_predictions=random_predictions
    )

# if __name__ == "__main__":
#     app.run(debug=True)


from __future__ import absolute_import, division, print_function

from functools import partial, wraps
from itertools import product
from math import factorial, log, ceil
import operator

import numpy as np
from toolz import compose, partition_all, merge, get

from . import chunk
from .core import _concatenate2, Array, atop, sqrt, lol_tuples
from .numpy_compat import divide
from ..compatibility import getargspec, builtins
from ..base import tokenize
from ..context import _globals
from ..utils import ignoring

def reduction(x, chunk, aggregate, axis=None, keepdims=None, dtype=None,
              split_every=None, combine=None):
    """ General version of reductions

    >>> reduction(my_array, np.sum, np.sum, axis=0, keepdims=False)  # doctest: +SKIP
    """
    if axis is None:
        axis = tuple(range(x.ndim))
    if isinstance(axis, int):
        axis = (axis,)
    axis = tuple(i if i >= 0 else x.ndim + i for i in axis)

    if dtype and 'dtype' in getargspec(chunk).args:
        chunk = partial(chunk, dtype=dtype)
    if dtype and 'dtype' in getargspec(aggregate).args:
        aggregate = partial(aggregate, dtype=dtype)

    # Normalize split_every
    split_every = split_every or _globals.get('split_every', 4)
    if isinstance(split_every, dict):
        split_every = dict((k, split_every.get(k, 2)) for k in axis)
    elif isinstance(split_every, int):
        n = builtins.max(int(split_every ** (1/(len(axis) or 1))), 2)
        split_every = dict.fromkeys(axis, n)
    else:
        split_every = dict((k, v) for (k, v) in enumerate(x.numblocks) if k in axis)

    # Map chunk across all blocks
    inds = tuple(range(x.ndim))
    tmp = atop(partial(chunk, axis=axis, keepdims=True), inds, x, inds)
    tmp._chunks = tuple((1,)*len(c) if i in axis else c for (i, c)
                        in enumerate(tmp.chunks))

    # Reduce across intermediates
    depth = 1
    for i, n in enumerate(tmp.numblocks):
        if i in split_every and split_every[i] != 1:
            depth = int(builtins.max(depth, ceil(log(n, split_every[i]))))
    func = compose(partial(combine or aggregate, axis=axis, keepdims=True),
                   partial(_concatenate2, axes=axis))
    for i in range(depth - 1):
        tmp = partial_reduce(func, tmp, split_every, True, None)
    func = compose(partial(aggregate, axis=axis, keepdims=keepdims),
                   partial(_concatenate2, axes=axis))
    return partial_reduce(func, tmp, split_every, keepdims=keepdims,
                          dtype=dtype)

def partial_reduce(func, x, split_every, keepdims=False, dtype=None, name=None):
    """Partial reduction across multiple axes.

    Parameters
    ----------
    func : function
    x : Array
    split_every : dict
        Maximum reduction block sizes in each dimension.

    Example
    -------
    Reduce across axis 0 and 2, merging a maximum of 1 block in the 0th
    dimension, and 3 blocks in the 2nd dimension:

    >>> partial_reduce(np.min, x, {0: 1, 2: 3})    # doctest: +SKIP
    """
    name = name or 'p_reduce-' + tokenize(func, x, split_every, keepdims, dtype)
    parts = [list(partition_all(split_every.get(i, 1), range(n))) for (i, n)
             in enumerate(x.numblocks)]
    keys = product(*map(range, map(len, parts)))
    out_chunks = [tuple(1 for p in partition_all(split_every[i], c)) if i
                  in split_every else c for (i, c) in enumerate(x.chunks)]
    if not keepdims:
        out_axis = [i for i in range(x.ndim) if i not in split_every]
        getter = lambda k: get(out_axis, k)
        keys = map(getter, keys)
        out_chunks = list(getter(out_chunks))
    dsk = {}
    for k, p in zip(keys, product(*parts)):
        decided = dict((i, j[0]) for (i, j) in enumerate(p) if len(j) == 1)
        dummy = dict(i for i in enumerate(p) if i[0] not in decided)
        g = lol_tuples((x.name,), range(x.ndim), decided, dummy)
        dsk[(name,) + k] = (func, g)
    return Array(merge(dsk, x.dask), name, out_chunks, dtype=dtype)

@wraps(chunk.sum)
def sum(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.empty((1,), dtype=a._dtype).sum().dtype
    else:
        dt = None
    return reduction(a, chunk.sum, chunk.sum, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every)

@wraps(chunk.prod)
def prod(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.empty((1,), dtype=a._dtype).prod().dtype
    else:
        dt = None
    return reduction(a, chunk.prod, chunk.prod, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every)

@wraps(chunk.min)
def min(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.min, chunk.min, axis=axis, keepdims=keepdims,
                     dtype=a._dtype, split_every=split_every)

@wraps(chunk.max)
def max(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.max, chunk.max, axis=axis, keepdims=keepdims,
                     dtype=a._dtype, split_every=split_every)

@wraps(chunk.any)
def any(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.any, chunk.any, axis=axis, keepdims=keepdims,
                     dtype='bool', split_every=split_every)

@wraps(chunk.all)
def all(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.all, chunk.all, axis=axis, keepdims=keepdims,
                     dtype='bool', split_every=split_every)

@wraps(chunk.nansum)
def nansum(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = chunk.nansum(np.empty((1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, chunk.nansum, chunk.sum, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every)

with ignoring(AttributeError):
    @wraps(chunk.nanprod)
    def nanprod(a, axis=None, dtype=None, keepdims=False, split_every=None):
        if dtype is not None:
            dt = dtype
        elif a._dtype is not None:
            dt = np.empty((1,), dtype=a._dtype).nanprod().dtype
        else:
            dt = None
        return reduction(a, chunk.nanprod, chunk.prod, axis=axis,
                         keepdims=keepdims, dtype=dt, split_every=split_every)

@wraps(chunk.nanmin)
def nanmin(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.nanmin, chunk.nanmin, axis=axis,
                     keepdims=keepdims, dtype=a._dtype, split_every=split_every)

@wraps(chunk.nanmax)
def nanmax(a, axis=None, keepdims=False, split_every=None):
    return reduction(a, chunk.nanmax, chunk.nanmax, axis=axis,
                     keepdims=keepdims, dtype=a._dtype, split_every=split_every)

def numel(x, **kwargs):
    """ A reduction to count the number of elements """
    return chunk.sum(np.ones_like(x), **kwargs)

def nannumel(x, **kwargs):
    """ A reduction to count the number of elements """
    return chunk.sum(~np.isnan(x), **kwargs)

def mean_chunk(x, sum=chunk.sum, numel=numel, dtype='f8', **kwargs):
    n = numel(x, dtype=dtype, **kwargs)
    total = sum(x, dtype=dtype, **kwargs)
    result = np.empty(shape=n.shape,
              dtype=[('total', total.dtype), ('n', n.dtype)])
    result['n'] = n
    result['total'] = total
    return result

def mean_combine(pair, sum=chunk.sum, numel=numel, dtype='f8', **kwargs):
    n = sum(pair['n'], **kwargs)
    total = sum(pair['total'], **kwargs)
    result = np.empty(shape=n.shape, dtype=pair.dtype)
    result['n'] = n
    result['total'] = total
    return result

def mean_agg(pair, dtype='f8', **kwargs):
    return divide(pair['total'].sum(dtype=dtype, **kwargs),
                  pair['n'].sum(dtype=dtype, **kwargs), dtype=dtype)

@wraps(chunk.mean)
def mean(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.mean(np.empty(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, mean_chunk, mean_agg, axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every, combine=mean_combine)

def nanmean(a, axis=None, dtype=None, keepdims=False, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.mean(np.empty(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, partial(mean_chunk, sum=chunk.nansum, numel=nannumel),
                     mean_agg, axis=axis, keepdims=keepdims, dtype=dt,
                     split_every=split_every,
                     combine=partial(mean_combine, sum=chunk.nansum, numel=nannumel))

with ignoring(AttributeError):
    nanmean = wraps(chunk.nanmean)(nanmean)

def moment_chunk(A, order=2, sum=chunk.sum, numel=numel, dtype='f8', **kwargs):
    total = sum(A, dtype=dtype, **kwargs)
    n = numel(A, **kwargs)
    u = total/n
    M = np.empty(shape=n.shape + (order - 1,), dtype=dtype)
    for i in range(2, order + 1):
        M[..., i - 2] = sum((A - u)**i, dtype=dtype, **kwargs)
    result = np.empty(shape=n.shape, dtype=[('total', total.dtype),
                                            ('n', n.dtype),
                                            ('M', M.dtype, (order-1,))])
    result['total'] = total
    result['n'] = n
    result['M'] = M
    return result

def _moment_helper(Ms, ns, inner_term, order, sum, kwargs):
    M = Ms[..., order - 2].sum(**kwargs) + sum(ns * inner_term**order, **kwargs)
    for k in range(1, order - 1):
        coeff = factorial(order)/(factorial(k)*factorial(order - k))
        M += coeff * sum(Ms[..., order - k - 2] * inner_term**k, **kwargs)
    return M

def moment_combine(data, order=2, ddof=0, dtype='f8', sum=np.sum, **kwargs):
    kwargs['dtype'] = dtype
    kwargs['keepdims'] = True

    totals = data['total']
    ns = data['n']
    Ms = data['M']
    total = totals.sum(**kwargs)
    n = sum(ns, **kwargs)
    mu = divide(total, n, dtype=dtype)
    inner_term = divide(totals, ns, dtype=dtype) - mu
    M = np.empty(shape=n.shape + (order - 1,), dtype=dtype)

    for o in range(2, order + 1):
        M[..., o - 2] = _moment_helper(Ms, ns, inner_term, o, sum, kwargs)

    result = np.zeros(shape=n.shape, dtype=[('total', total.dtype),
                                            ('n', n.dtype),
                                            ('M', Ms.dtype, (order-1,))])
    result['total'] = total
    result['n'] = n
    result['M'] = M
    return result

def moment_agg(data, order=2, ddof=0, dtype='f8', sum=np.sum, **kwargs):
    totals = data['total']
    ns = data['n']
    Ms = data['M']

    kwargs['dtype'] = dtype
    # To properly handle ndarrays, the original dimensions need to be kept for
    # part of the calculation.
    keepdim_kw = kwargs.copy()
    keepdim_kw['keepdims'] = True

    n = sum(ns, **keepdim_kw)
    mu = divide(totals.sum(**keepdim_kw), n, dtype=dtype)
    inner_term = divide(totals, ns, dtype=dtype) - mu

    M = _moment_helper(Ms, ns, inner_term, order, sum, kwargs)
    return divide(M, sum(n, **kwargs) - ddof, dtype=dtype)

def moment(a, order, axis=None, dtype=None, keepdims=False, ddof=0,
           split_every=None):
    if not isinstance(order, int) or order < 2:
        raise ValueError("Order must be an integer >= 2")
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.var(np.ones(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, partial(moment_chunk, order=order), partial(moment_agg,
                     order=order, ddof=ddof), axis=axis, keepdims=keepdims,
                     dtype=dt, split_every=split_every,
                     combine=partial(moment_combine, order=order))

@wraps(chunk.var)
def var(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.var(np.ones(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, moment_chunk, partial(moment_agg, ddof=ddof), axis=axis,
                     keepdims=keepdims, dtype=dt, split_every=split_every,
                     combine=moment_combine)

def nanvar(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    if dtype is not None:
        dt = dtype
    elif a._dtype is not None:
        dt = np.var(np.ones(shape=(1,), dtype=a._dtype)).dtype
    else:
        dt = None
    return reduction(a, partial(moment_chunk, sum=chunk.nansum, numel=nannumel),
                     partial(moment_agg, sum=np.nansum, ddof=ddof), axis=axis,
                     keepdims=keepdims, dtype=dt, split_every=split_every,
                     combine=partial(moment_combine, sum=np.nansum))

with ignoring(AttributeError):
    nanvar = wraps(chunk.nanvar)(nanvar)

@wraps(chunk.std)
def std(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    result = sqrt(a.var(axis=axis, dtype=dtype, keepdims=keepdims, ddof=ddof,
                        split_every=split_every))
    if dtype and dtype != result.dtype:
        result = result.astype(dtype)
    return result

def nanstd(a, axis=None, dtype=None, keepdims=False, ddof=0, split_every=None):
    result = sqrt(nanvar(a, axis=axis, dtype=dtype, keepdims=keepdims,
                         ddof=ddof, split_every=split_every))
    if dtype and dtype != result.dtype:
        result = result.astype(dtype)
    return result

with ignoring(AttributeError):
    nanstd = wraps(chunk.nanstd)(nanstd)

def vnorm(a, ord=None, axis=None, dtype=None, keepdims=False, split_every=None):
    """ Vector norm

    See np.linalg.norm
    """
    if ord is None or ord == 'fro':
        ord = 2
    if ord == np.inf:
        return max(abs(a), axis=axis, keepdims=keepdims, split_every=split_every)
    elif ord == -np.inf:
        return min(abs(a), axis=axis, keepdims=keepdims, split_every=split_every)
    elif ord == 1:
        return sum(abs(a), axis=axis, dtype=dtype, keepdims=keepdims,
                   split_every=split_every)
    elif ord % 2 == 0:
        return sum(a**ord, axis=axis, dtype=dtype, keepdims=keepdims,
                   split_every=split_every)**(1./ord)
    else:
        return sum(abs(a)**ord, axis=axis, dtype=dtype, keepdims=keepdims,
                   split_every=split_every)**(1./ord)

def _arg_combine(data, axis, argfunc):
    """Merge intermediate results from ``arg_*`` functions"""
    vals = data['vals']
    arg = data['arg']
    ns = data['n']
    args = argfunc(vals, axis=axis)
    offsets = np.roll(np.cumsum(ns, axis=axis), 1, axis)
    offsets[tuple(slice(None) if i != axis else 0 for i in range(ns.ndim))] = 0
    inds = list(reversed(np.meshgrid(*map(np.arange, args.shape), sparse=True)))
    inds.insert(axis, args)

    arg = (arg + offsets)[tuple(inds)]
    vals = vals[tuple(inds)]
    n = ns.sum(axis=axis).take(0, 0)
    return arg, vals, n

def arg_chunk(func, argfunc, x, axis=None, **kwargs):
    axis = axis[0] if isinstance(axis, tuple) else axis
    vals = func(x, axis=axis, keepdims=True)
    arg = argfunc(x, axis=axis, keepdims=True)
    result = np.empty(shape=vals.shape, dtype=[('vals', vals.dtype),
                                               ('arg', arg.dtype),
                                               ('n', 'i8')])
    result['vals'] = vals
    result['arg'] = arg
    result['n'] = x.shape[axis]
    return result

def arg_combine(func, argfunc, data, axis=None, **kwargs):
    axis = axis[0] if isinstance(axis, tuple) else axis
    arg, vals, n = _arg_combine(data, axis, argfunc)
    shape = tuple(s if i != axis else 1 for (i, s) in enumerate(data.shape))
    result = np.empty(shape=shape, dtype=[('vals', vals.dtype),
                                          ('arg', arg.dtype),
                                          ('n', 'i8')])
    result['vals'] = vals.reshape(shape)
    result['arg'] = arg.reshape(shape)
    result['n'] = n
    return result

def arg_agg(func, argfunc, data, axis=None, **kwargs):
    axis = axis[0] if isinstance(axis, tuple) else axis
    return _arg_combine(data, axis, argfunc)[0]

def arg_reduction(func, argfunc):
    chunk = partial(arg_chunk, func, argfunc)
    agg = partial(arg_agg, func, argfunc)
    combine = partial(arg_combine, func, argfunc)
    @wraps(argfunc)
    def _(a, axis=None, split_every=None):
        if axis < 0:
            axis = a.ndim + axis
        return reduction(a, chunk, agg, axis=axis, dtype='i8',
                         split_every=split_every, combine=combine)
    return _

argmin = arg_reduction(chunk.min, chunk.argmin)
argmax = arg_reduction(chunk.max, chunk.argmax)
nanargmin = arg_reduction(chunk.nanmin, chunk.nanargmin)
nanargmax = arg_reduction(chunk.nanmax, chunk.nanargmax)

def cumreduction(func, binop, ident, x, axis, dtype=None):
    """ Generic function for cumulative reduction

    Parameters
    ----------
    func: callable
        Cumulative function like np.cumsum or np.cumprod
    binop: callable
        Associated binary operator like ``np.cumsum->add`` or ``np.cumprod->mul``
    ident: Number
        Associated identity like ``np.cumsum->0`` or ``np.cumprod->1``
    x: dask Array
    axis: int
    dtype: dtype

    Returns
    -------
    dask array

    See also
    --------
    cumsum
    cumprod
    """
    if dtype is None:
        dtype = func(np.empty((0,), dtype=x.dtype)).dtype
    assert isinstance(axis, int)
    m = x.map_blocks(func, axis=axis, dtype=dtype)

    name = '%s-axis=%d-%s' % (func.__name__, axis, tokenize(x, dtype))
    n = x.numblocks[axis]
    full = slice(None, None, None)
    slc = (full,) * axis + (slice(-1, None),) + (full,) * (x.ndim - axis - 1)

    indices = list(product(*[range(nb) if i != axis else [0]
                             for i, nb in enumerate(x.numblocks)]))
    dsk = dict()
    for ind in indices:
        shape = tuple(x.chunks[i][ii] if i != axis else 1
                      for i, ii in enumerate(ind))
        dsk[(name, 'extra') + ind] = (np.full, shape, ident, m.dtype)
        dsk[(name,) + ind] = (m.name,) + ind

    for i in range(1, n):
        last_indices = indices
        indices = list(product(*[range(nb) if ii != axis else [i]
                                 for ii, nb in enumerate(x.numblocks)]))
        for old, ind in zip(last_indices, indices):
            last = (name, 'extra') + old
            this_slice = (name, 'extra') + ind
            dsk[this_slice] = (binop, (name, 'extra') + old,
                                      (operator.getitem, (m.name,) + old, slc))
            dsk[(name,) + ind] = (binop, this_slice, (m.name,) + ind)

    return Array(merge(dsk, m.dask), name, x.chunks, m.dtype)

@wraps(np.cumsum)
def cumsum(x, axis, dtype=None):
    return cumreduction(np.cumsum, operator.add, 0, x, axis, dtype)

@wraps(np.cumprod)
def cumprod(x, axis, dtype=None):
    return cumreduction(np.cumprod, operator.mul, 1, x, axis, dtype)


#!/usr/bin/python
#
# Simple module to control the Sparkfun Serial LCD Modules
#
import serial

#
# LCD Constants
#
LCD_BACKLIGHT_OFF = 128
LCD_BACKLIGHT_MAX = 157
LCD_CHARS = 16
LCD_LINES = 2
LCD_LINESTART = [0, 64, 16, 80] # Cursor position starts for 16 character displays
LCD_MAX_CHARS = LCD_CHARS * LCD_LINES

#
# Control characters/commands
#
LCD_BACLKIGHT_CMD = 0x7C
LCD_CMD = 0xFE
LCD_CMD_CLEAR = 0x01
LCD_CMD_CURSORPOS = 0x80
LCD_CMD_MOVRIGHT = 0x14
LCD_CMD_MOVLEFT = 0x10
LCD_CMD_SCRLRIGHT = 0x1C
LCD_CMD_SCRLLEFT = 0x18

class Controller():
    def __init__(self):
        self.backlight = 1.0
        self.connected = False
        self.serialPort = None

    def connect(self, serialPort, baudRate):
        if self.serialPort is not None and self.serialPort.isOpen():
            self.serialPort.close()

        self.serialPort = serial.Serial(port = serialPort, baudrate = baudRate)
        self.serialPort.close()
        self.serialPort.open()

        if self.serialPort.isOpen():
            self.connected = True
            return True
        else:
            self.connected = False
            return False

    def write(self, string):
        if self.connected:
            self.serialPort.write(string)

    def disconnect(self):
        if self.connected:
            self.serialPort.close()

    def clear(self):
        self.write(chr(LCD_CMD) + chr(LCD_CMD_CLEAR))
        self.write(chr(LCD_CMD) + chr(LCD_CMD_CLEAR))

    def setBacklight(self, brightness):
        if brightness < 0:
            brightness = 0
        elif brightness > 1:
            brightness = 1

        self.backlight = brightness
        LCD_newBrightness = int(self.backlight * (BACKLIGHT_MAX - LCD_BACKLIGHT_OFF)) + LCD_BACKLIGHT_OFF
        self.write(chr(LCD_BACLKIGHT_CMD) + chr(newBrightness))

    def setCursorPos(self, pos):
        if pos < 0:
            pos = 0
        elif pos > (LCD_MAX_CHARS - 1):
            pos = (LCD_MAX_CHARS - 1)

        line = int(pos)/int(LCD_CHARS)
        lineStart = LCD_LINESTART[line]
        linePos = pos - line * LCD_CHARS + lineStart

        self.write(chr(LCD_CMD) + chr(LCD_CMD_CURSORPOS + int(linePos)))

    def cursorMoveRight(self, times = 1):
        for dummy in range(times):
            self.write(chr(LCD_CMD) + chr(LCD_CMD_MOVRIGHT))

    def cursorMoveLeft(self, times = 1):
        for dummy in range(times):
            self.write(chr(LCD_CMD) + chr(LCD_CMD_MOVLEFT))

    def cursorScrollRight(self, times = 1):
        for dummy in range(times):
            self.write(chr(LCD_CMD) + chr(LCD_CMD_SCRLRIGHT))

    def cursorScrollLeft(self, times = 1):
        for dummy in range(times):
            self.write(chr(LCD_CMD) + chr(LCD_CMD_SCRLLEFT))

if __name__ == '__main__':
    print('Standalone operation not supported...')


import pandas as pd
import ast
import numpy as np
import matplotlib.pyplot as plt

from PIL import Image
from wordcloud import WordCloud

def get_except_keyword(filename):
    keyword_list = list()
    with open(filename, encoding='utf-8') as f:
        for keyword in f.readlines():
            keyword_list.append(keyword.strip())
    print(keyword_list)
    return keyword_list

def draw_wordcloud(kkma_result):
    # List로 되어있는 열을 Row 단위로 분리
    tokens = pd.DataFrame(kkma_result["token"].apply(lambda x: ast.literal_eval(x)).tolist())

    tokens["Date"] = kkma_result["Date"]
    tokens["Speaker"] = kkma_result["Speaker"]
    tokens["timetype"] = kkma_result["timetype"]
    tokens["time"] = kkma_result["time"]
    tokens["contents"] = kkma_result["contents"]

    tokens = tokens.set_index(["Date", "Speaker", "timetype", "time", "contents"])
    tokens = tokens.T.unstack().dropna().reset_index()

    tokens.columns = ["Date", "Person", "time_type", "time", "sntc", "index", "token"]
    print(tokens.head())

    # 빈도수 집계
    summary = tokens.groupby(["token"])["index"].count().reset_index()
    summary = summary.sort_values(["index"], ascending=[False]).reset_index(drop=True)

    # 특정 단어 필터링
    except_keyword = get_except_keyword("./raw_data/except_word.txt")
    summary = summary[summary["token"].apply(lambda x: x not in except_keyword)]
    summary = summary[summary["token"].apply(lambda x: len(x) > 1)]

    # 이미지 Mask 생성
    denne_mask = np.array(Image.open("./font/denne.png"))

    # 워드클라우드 생성
    wc = WordCloud(font_path='./font/NanumBrush.ttf', background_color='white', mask=denne_mask, width=800, height=600).generate(" ".join(summary["token"]))
    plt.imshow(wc)
    plt.axis("off")
    plt.show()

if __name__ == '__main__':
    kkma_result = pd.read_csv("./result/noun_token.csv")
    draw_wordcloud(kkma_result)


"""
Basic HTTP server for redirecting all trafic to specific URL.
Pure python no dependencies.
"""

__title__ = 'Redirect TO'
__version__ = '0.0.1'
__author__ = 'Petr Dovnar'
__license__ = 'BSD 3-Clause'

import argparse
import logging
import socket
from http.server import HTTPServer, BaseHTTPRequestHandler

logging.basicConfig(
     level=logging.INFO,
     format='[%(asctime)s] %(levelname)s - %(message)s',
     datefmt='%H:%M:%S'
 )
log = logging.getLogger(__name__)

def extract_ip():
    st = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    try:
        st.connect(('10.255.255.255', 1))
        IP = st.getsockname()[0]
    except Exception:
        IP = '127.0.0.1'
    finally:
        st.close()
    return IP

def serve():
    parser = argparse.ArgumentParser(description='Redirects requests to URL')
    parser.add_argument('url', type=str, help='Targer URL for redirects')
    args = parser.parse_args()

    if not args.url:
        log.error('URL is missing')
        exit(1)

    url = args.url

    log.info('Redirect URL: %s', url)

    class RedirectHandler(BaseHTTPRequestHandler):
        def do_GET(self):
            self.send_response(302)
            self.send_header('Location', url)
            self.end_headers()

    ip_address = extract_ip()
    port = 3005

    log.info('Listening on http://%s:%d ...', ip_address, port)

    try:
        log.info('Press Ctrl+C to stop server')
        server = HTTPServer((ip_address, port), RedirectHandler)
        server.serve_forever()
    except KeyboardInterrupt:
        log.info('Server stopped by user')

    log.info('Done')

if __name__ == '__main__':
    serve()


import pandas as pd
import plotly.express as px
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score,f1_score,confusion_matrix, plot_confusion_matrix, classification_report
import xgboost as xgb
import shap
shap.initjs()
import matplotlib.pyplot as plt
from pickle import dump
import numpy as np
import dask.dataframe as dd
from dask.distributed import Client
from scipy.stats import ttest_ind
from collections import defaultdict
from tqdm.auto import tqdm
import logging
from umap import UMAP
import hdbscan
from contextlib import nullcontext
from IPython.utils import io
from collections.abc import Iterable
from functools import wraps
from dataclasses import dataclass, field, make_dataclass, asdict
from typing import ClassVar
from functools import partial
from collections import UserDict
import pprint
from typing import Any, Callable
import seaborn as sns
from sklearn.metrics import silhouette_score

#was trying to write custom __eq__ and custom __hash__ to override eq = True, which needs the two instances being compared to be of the exact same class
def dataclass_from_dict(name,input_dict):
#     def __eq__(self, other):
#         inst_fields = self.__dataclass_fields__.keys()
#         return all([self.__dict__[attr] == other.__dict__[attr] for attr in inst_fields])
#     def __hash__(self):
#         inst_fields = self.__dataclass_fields__.keys()
#         return hash(tuple([self.__dict__[attr] for attr in inst_fields]))

    return make_dataclass(name, fields = [(k,type(v)) for k,v in input_dict.items()], frozen=True,) #eq=False, namespace={"__eq__": __eq__, "__hash__": __hash__})

#subclassing dict to extend dict functionalities
class DataClassDict(UserDict):

    def __init__(self, data, cls):
        super(DataClassDict, self).__init__()
        self.data = data
        self._cls = cls
        self._keys =  super(DataClassDict, self).keys

    def __repr__(self):
        from pprint import pformat
        return pformat(str(list(self.keys()))).strip("()")

    def query(self, *args, **kwargs):
        key_instance = self._cls(**kwargs)
        return self.data[key_instance]

    def keys(self):
        return list(self._keys())#[k for k,v in self.data.items()]

    def to_pandas(self, attr: str) -> pd.core.frame.DataFrame:
        def to_pandas_single_df(dc_val, attr):
            col_name = ("|").join([f"func={dc_val.func_name}", *[("=").join([str(k), str(v)]) for k,v in dc_val.args.items()]])
            attr_val = dc_val.return_vals[attr]
            df = pd.DataFrame(attr_val, columns = [col_name] * attr_val.shape[1] if isinstance(attr_val, (np.ndarray, pd.core.frame.DataFrame)) else col_name)
            return df
        return pd.concat([to_pandas_single_df(dc_val, attr) for dc_val in self.values()], axis=1)

#     def to_pandas(self):
#         return pd.DataFrame

def repeated_call(parameters_dict):

    def wrapper(func):
        @wraps(func)
        def _wrapped_func(*args, **kwargs):
            """A wrapper function"""

    return wrapper

def return_with_args(return_vals, filterable=False, return_args=None, pre_process=None,*args, **kwargs):
#     def _preprocess(ele: ):
#         if isinstance(ele, )

    import inspect
    def wrapper(func):
        @wraps(func)
        def _wrapped_func(*args, rename_dict=None, return_vals_filtered=[], **kwargs):
            """A wrapper function"""
            # Extend some capabilities of func
            make_dataclass_mod = partial(make_dataclass, frozen=True, eq=True)
            func_locals_dict = func(*args, **kwargs)
            ##allow user to filter and rename variables
            return_vals_mod = return_vals_filtered if filterable else return_vals
            return_vals_mod = return_vals_mod if rename_dict is None else [rename_dict[val] for val in return_vals_mod]
            return_args_mod = inspect.getargspec(func).args if return_args is None else return_args

            base_class_name = f"{func.__name__}_Result_Base"

            locals()[base_class_name]= make_dataclass_mod(base_class_name, [('args', dict), ('return_vals', dict)])
            created_class = locals()[base_class_name]

            #include the result class within the object

            wrapper_class_name = f"{func.__name__}_Result"
            locals()[wrapper_class_name] = make_dataclass_mod(wrapper_class_name,
                                                                              fields=[('result_class', object), ("fields", list),("func_name", str)], bases=(created_class,))

            created_wrapper_class = locals()[wrapper_class_name]

            args_class_name = f"{func.__name__}_Args"

            return_dict = {
                           "args": {k:v for k,v in func_locals_dict.items() if k in return_args_mod},
                           "return_vals": {k:v for k,v in func_locals_dict.items() if k in return_vals_mod}
                          }

            created_wrapper_class.__repr__ = lambda self: f"<{created_wrapper_class.__name__} object, fields: {list(created_wrapper_class.__dataclass_fields__.keys())}>"
            return_result_instance = created_class(**return_dict)
            return_wrapper_instance = created_wrapper_class(**{"result_class": created_class, "fields" : list(created_class.__dataclass_fields__.keys()), "func_name": func.__name__}, **return_dict)
            return return_wrapper_instance
        return _wrapped_func

    return wrapper

def predict_ensemble(ensemble, X):
    """
    predict_ensemble runs the X data set through
    each classifier in the ensemble list to get predicted
    probabilities.
    Those are then averaged out across all classifiers.
    """
    probs = [r.predict_proba(X)[:,1] for r in ensemble]
    return np.vstack(probs).mean(axis=0)

def adjusted_classes(y_scores, t):
    """
    This function adjusts class predictions based on the prediction threshold (t).
    Will only work for binary classification problems.
    """
    return [1 if y >= t else 0 for y in y_scores]

def print_report(m, X_valid, y_valid, t=0.5, X_train=None, y_train=None, show_output=True):
    """
    print_report prints a comprehensive classification report
    on both validation and training set (if provided).
    The metrics returned are AUC, F1, Precision, Recall and
    Confusion Matrix.
    It accepts both single classifiers and ensembles.
    Results are dependent on the probability threshold t
    applied to individual predictions.
    """
#     X_train = X_train.values
#     X_valid = X_valid.values

    if isinstance(m, list):
        probs_valid = predict_ensemble(m, X_valid)
        y_val_pred = adjusted_classes(probs_valid, t)

        if X_train is not None:
            probs_train = predict_ensemble(m, X_train)
            y_train_pred = adjusted_classes(probs_train, t)
    else:
        probs_valid = m.predict_proba(X_valid)[:,1]
        y_val_pred = adjusted_classes(probs_valid, t)

        if X_train is not None:
            probs_train = m.predict_proba(X_train)[:,1]
            y_train_pred = adjusted_classes(probs_train, t)

    res = [roc_auc_score(y_valid, probs_valid),
           f1_score(y_valid, y_val_pred),
           confusion_matrix(y_valid, y_val_pred)]
    result = f'AUC valid: {res[0]} \nF1 valid: {res[1]}'

    if X_train is not None:
        res += [roc_auc_score(y_train, probs_train),
                f1_score(y_train, y_train_pred)]
        result += f'\nAUC train: {res[3]} \nF1 train: {res[4]}'

    acc_train = m.score(X_train, y_train)
    acc_valid = m.score(X_valid, y_valid)

    if show_output:
        logging.info(f"train acc: {acc_train}")
        logging.info(f"test acc: {acc_valid} ")

        logging.info(result)
        plot_confusion_matrix(m, X_valid, y_valid,
                                     display_labels= y_valid.unique())
        logging.info(classification_report(y_valid, y_val_pred))
        plt.show()
    return {"train":{"AUC": res[3], "F1": res[4], "acc": acc_train}, "test":{"AUC": res[0], "F1": res[1], "acc": acc_valid}}

def train_output_metrics( X,y, *, test_size=0.3, name, path=None, no_seed=False, fitted=None, show_output=True, early_stopping, model=None,**kwargs):
    if no_seed:
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,
                                                        stratify=y)
    else:
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size,random_state=42, stratify=y)

    if not fitted:
        print(f"extra XGBoost kwargs: {kwargs}")
        if not model:
            model = xgb.XGBClassifier(verbosity = 0, **kwargs)#xgb.XGBClassifier(n_estimators=100,  subsample=0.8, colsample_bytree=0.5)
        eval_set = [(X_train, y_train), (X_test, y_test)]
        eval_metric = ["error", "logloss", "auc"]

        # check if multi-label classification
        if "num_class" in kwargs.keys():
            eval_metric += ["merror", "mlogloss"]
        print(X_train.shape, y_train.shape)
        print(eval_metric)

        if early_stopping:
            fitted = model.fit(X_train,y_train, early_stopping_rounds=5, eval_metric=eval_metric,  eval_set=eval_set)

        else:
            fitted = model.fit(X_train,y_train, eval_metric=["error", "logloss", "auc", "aucpr", "map"],  eval_set=eval_set)

        results = model.evals_result()
    if show_output:
        logging.info(f"train size:{1- test_size}, test_size:{test_size}")
        logging.info(f"train case/control num {y_train.value_counts()[1]}/{y_train.value_counts()[0]}")
        logging.info(f"test case/control num  {y_test.value_counts()[1]}/{y_test.value_counts()[0]}")

    metrics_dict = print_report(model, X_test, y_test, t=0.5, X_train=X_train, y_train=y_train, show_output=show_output)

    if path:
        dump(fitted, open(path, 'wb'))

    shap_values_non_zero, important_feat = get_shap_info(model, X, name)

    return {"shap_df": shap_values_non_zero, "important_feat":important_feat, "metrics":metrics_dict, "model":model, "train_index": X_train.index, "test_index": X_test.index}

def get_shap_info(model,X,name):
    explainer = shap.TreeExplainer(model)
    shap_values = explainer.shap_values(X)
    shap_values_non_zero = pd.DataFrame(shap_values, columns = X.columns).loc[:, shap_values.sum(axis=0) != 0]
    important_feat = np.abs(shap_values_non_zero).mean().sort_values(ascending=False)

    formatted_name = name.format()
    important_feat.name = f"{name}_shap_vals"
    important_feat = important_feat.to_frame()
    return shap_values_non_zero, important_feat

# def train_multi_params(*args, param_dict:dict=None, func=None, **kwargs):
#     if None in [changing_param, func]:
#         raise ValueError
#     else:
#         results = defaultdict(dict)
#         for key in tqdm(param_dict):
#             for val in tqdm(param_dict[key]):
#                 new_dict = kwargs.copy()
#                 new_dict[key] = val
#                 results[key][val] = func(*args,**new_dict)

def repeat_train(X, y, num_times, different_train_test_split, name, model=None, silent=False, **kwargs):
    result_list =[]
    print(f"Extra params: {kwargs}")
    for i in tqdm(range(num_times)):
        with io.capture_output() if silent else nullcontext() as captured :
            result_dict = train_output_metrics(X, y, name="base_model",no_seed=different_train_test_split, early_stopping=True, model=model, **kwargs)
            result_list.append(result_dict)
    auc_list = [metric_dict["metrics"]["test"]["AUC"] for metric_dict in result_list]
    auc_df = pd.DataFrame({"AUC": auc_list, "dataset":name})
    fig = px.violin(auc_df, box=True, y = "AUC", color = "dataset")
    return {"AUC":auc_df, "result_list": result_list, "fig": fig}

def plot_curve(model_evals_result):
    results = model_evals_result
    epochs = len(results['validation_0']['error'])
    x_axis = range(0, epochs)
    # plot log loss
    train_df = pd.DataFrame.from_dict(base_model.evals_result()["validation_0"], orient="columns")
    train_df["set"] = "train"
    test_df = pd.DataFrame.from_dict(base_model.evals_result()["validation_1"], orient="columns")
    test_df["set"] = "test"
    combined = pd.concat([train_df, test_df])

    for column in combined.drop(columns="set").columns:
        fig = px.line(combined, x = combined.index, y=column, color ="set")
        fig.show()
#         fig, ax = plt.subplots()
#         ax.plot(x_axis, results['validation_0'][metric], label='Train')
#         ax.plot(x_axis, results['validation_1'][metric], label='Test')
#         ax.legend()
#         plt.ylabel(f'{metric}')
#         plt.title(f'XGBoost {metric}')
#         plt.show()

@return_with_args(return_vals=["multi_param_result_dict"])
def draw_umap(data,n_neighbors=None, min_dist=None, c=None,  n_components=None, metric='euclidean', title="Untitled_exp", plot=True, plotly_3D = True, return_vals_filtered=["u","mapper"], umap_kwargs={}, **kwargs):
    @return_with_args(return_vals=["u", "mapper"], filterable = True)
    def draw_one_umap(one_n_neighbor, one_min_dist, one_n_component):
        fit = UMAP(
        n_neighbors= one_n_neighbor,
        min_dist= one_min_dist,
        n_components=one_n_component,
        metric=metric,
        random_state = 42,
        **umap_kwargs
    )

        mapper = fit.fit(data);
        u = fit.transform(data);

        fig = None
        final_title = f"{title}: {one_n_component}D representation, n_neighbor: {one_n_neighbor}, min_dist: {one_min_dist}"

        if plot:
#             if one_n_component == 1:
#                 ax = fig.add_subplot(111)
#                 ax.scatter(u[:,0], range(len(u)), c=c)
            fig = plt.figure()

            if one_n_component == 2:
                ax = fig.add_subplot(111)
                scatter = ax.scatter(u[:,0], u[:,1], c=c, label=c)
                plt.title(final_title, fontsize=18)
                legend = ax.legend(*scatter.legend_elements())
                ax.add_artist(legend)
                plt.show()
            if one_n_component == 3:

#                 ax = fig.add_subplot(111, projection='3d')
#                 scatter = ax.scatter(u[:,0], u[:,1], u[:,2], c=c, s=100)

                if plotly_3D:
                    plotly_fig_3D = px.scatter_3d(x=u[:,0], y=u[:,1], z=u[:,2],color=c, opacity = 0.3, title = final_title, **kwargs)
                    plotly_fig_3D.update_layout(legend=dict(
                                    orientation="h",
                                    yanchor="bottom",
                                    y=1.02,
                                    xanchor="right",
                                    x=1
                                ))

                    display(plotly_fig_3D)

        u = pd.DataFrame(u, columns = [f"UMAP_{one_n_component}D_{i}" for i in range(1, one_n_component+1)], index=data.index)
        return locals()

    standardize_input = lambda x, default: list(x) if isinstance(x, Iterable) else default if x is None else [x]

    n_neighbor_list = standardize_input(n_neighbors, [5,10, 20, 50, 100, 200])
    min_dist_list = standardize_input(min_dist, [0, 0.1])
    n_component_list = standardize_input(n_components, [2,3])

    multi_param_result_dict = {}

    MultiParamClass = None

    for one_min_dist in min_dist_list:
        for one_n_neighbor in n_neighbor_list:
            if one_n_neighbor >= data.shape[0]:
                continue
            for one_n_component in n_component_list:
                returned_instance = draw_one_umap(one_n_neighbor, one_min_dist, one_n_component, return_vals_filtered=return_vals_filtered)

                if MultiParamClass is None:
                    MultiParamClass = dataclass_from_dict("Params", returned_instance.args)

                dict_key = MultiParamClass(**returned_instance.args)
                multi_param_result_dict[dict_key] = returned_instance

                #multi_param_result_dict[frozenset(returned_instance.args.items())] = returned_instance
    multi_param_result_dict = DataClassDict(multi_param_result_dict, MultiParamClass)
    return  locals()#multi_param_result_dict #locals() #{"embedding": u, "mapper": mapper}

def cluster_hdbscan(clusterable_embedding, min_cluster_size, viz_embedding_list, plot=True):
    print(f"min_cluster size: {min_cluster_size}")
    clusterer= hdbscan.HDBSCAN(
        min_cluster_size=min_cluster_size,
    ).fit(clusterable_embedding)
    labels = clusterer.fit_predict(clusterable_embedding)
    print(f"found {len(np.unique(labels))} clusters")
    clustered = (labels >= 0)
    frac_clustered = np.sum(clustered)/labels.shape[0]
    print(f"fraction clustered: {np.sum(clustered)/labels.shape[0]}")
    clusterer.condensed_tree_.plot(select_clusters=True,
                               selection_palette=sns.color_palette('deep', 8))

    for embedding in viz_embedding_list:

        if embedding.shape[1] == 3:
            fig = px.scatter_3d(x= embedding[:, 0],
                y= embedding[:, 1],
                z= embedding[:, 2],
                color = labels.astype("str"))
            display(fig)
            #.show()
#             ax = fig.add_subplot(111, projection='3d')

#             ax.scatter(embedding[~clustered][:, 0],
#                 embedding[~clustered][:, 1],
#                 embedding[~clustered][:, 2],
#                 c=(0.5, 0.5, 0.5),
#                 alpha=0.5)
#             ax.scatter(embedding[clustered][:, 0],
#                     embedding[clustered][:, 1],
#                     embedding[~clustered][:, 2],
#                     c=labels[clustered],
#                     cmap='Spectral')
        else:
            fig = px.scatter(x= embedding[:, 0],
                y= embedding[:, 1],
                color = labels.astype("str"))
            display(fig)
#             fig = plt.figure()
#             ax = fig.add_subplot(111)
#             ax.scatter(embedding[~clustered][:, 0],
#                 embedding[~clustered][:, 1],
#                 c=(0.5, 0.5, 0.5),
#                 s=10,
#                 alpha=0.5)
#             ax.scatter(embedding[clustered][:, 0],
#                     embedding[clustered][:, 1],
#                     c=labels[clustered].reshape(-1,1),
#                     s=10,
#                     cmap='Spectral')

#             plt.title(f"min_cluster_size = {min_cluster_size}", fontsize=18)
#             legend = ax.legend(*scatter.legend_elements())
#             ax.add_artist(legend)
#             plt.show()

    return {"labels":labels, "clusterer": clusterer}

def get_all_cluster_settings(fitted_hdbscan, embedding_used_to_cluster, unclustered_threshold=0.2, extra_cols_dict={}):
    '''
    unclustered_threshold: fraction of unclustered points allowed, if the number of points is more than this threshold, the setting is discarded
    '''
    test_label_dict = {}
    silhouette_scores = set()
    df_list = []
    linkage_tree = fitted_hdbscan.single_linkage_tree_
    condensed_tree_df = fitted_hdbscan.condensed_tree_.to_pandas().query("child_size > 1")
    for i, lambda_val in tqdm(enumerate(linkage_tree.to_pandas().sort_values("size", ascending = False).query("size > @condensed_tree_df.child_size.min()").distance.unique() - 0.01)):
        for child_size in sorted(condensed_tree_df["child_size"].unique() - 1):
            test_labels = fitted_hdbscan.single_linkage_tree_.get_clusters(lambda_val, min_cluster_size=2)
            # remove settings where alot is unclustered
            if (test_labels == -1).sum()/len(test_labels) > unclustered_threshold:
                continue
            try:
                #print(str(len(np.unique(test_labels))))
                test_silhouette_score = silhouette_score(embedding_used_to_cluster, test_labels)
            except ValueError:
                #print("skipped")
                continue

            if test_silhouette_score in silhouette_scores:
                continue

            col_name = str(len(np.unique(test_labels)))

            j = 0
            if col_name in test_label_dict:
                while f"{col_name}_{j}" in test_label_dict:
                    j+=1
                col_name = f"{col_name}_{j}"

            silhouette_scores.add(test_silhouette_score)
            #test_labels_df = pd.DataFrame(test_labels, columns=[f"{len(np.unique(test_labels))}_clusters_label"]).assign(**)
            df_list.append({"cluster_setting_num": i,
                            "col_name" : f"{col_name}_clusters_label",
                            "silhouette_score": test_silhouette_score,
                            "num_clusters": len(np.unique(test_labels)),
                            })#test_labels_df[[f"{len(np.unique(test_labels))}_clusters_label"]])
            test_label_dict[col_name] = test_labels
            #display(test_labels_df.head())
            #test_label_dict.append({f"{len(np.unique(test_labels))}_cluster_labels":list(test_labels), "silhouette_score": silhouette_score(test_clusteval_embed, test_labels) })

    all_labels_df = pd.DataFrame.from_dict(test_label_dict)

    #sort by number of clusters
    all_labels_df = all_labels_df[sorted(all_labels_df.columns, key=lambda x: tuple(map(int,x.split('_'))))].add_suffix("_clusters_label")
    all_labels_df = all_labels_df.assign(**extra_cols_dict)
    all_labels_df.index = embedding_used_to_cluster.index
    display(all_labels_df.head())
    cluster_metrics_df = pd.DataFrame(df_list)#.sort_values(["silhouette_score","num_clusters"], ascending= [False, True])

    return {"all_labels_df": all_labels_df, "cluster_metrics_df": cluster_metrics_df}

def get_cluster_defining_features(X, clustering_label, cluster_setting_name):
    """Need to provide X as features only and NO clustering label """
    cluster_names = np.unique(clustering_label)
    final_dict = {}
    all_cluster_info_mean_peaks_per_cluster = []

    for cluster_name in tqdm(cluster_names):
        if cluster_name == -1:
            continue
        print(cluster_name)
        final_dict[cluster_name] = {}
        cluster_one_v_all_labels = (clustering_label == cluster_name).astype(int)
        #use scale_pos_weight to address imbalance between clustering class vs rest
        with io.capture_output() as captured:
            all_OnevsAll_models_result_dict = repeat_train(X, cluster_one_v_all_labels,num_times=1000, different_train_test_split=True, name=cluster_setting_name, scale_pos_weight=cluster_one_v_all_labels.value_counts()[0]/cluster_one_v_all_labels.value_counts()[1], show_output=False)

        final_dict[cluster_name]["AUC_fig"] = all_OnevsAll_models_result_dict["fig"]

        #breakpoint()
        shap_mean_peaks_per_cluster_all_models = (pd.concat([result["shap_df"].assign(model_num=i) for i, result in enumerate(all_OnevsAll_models_result_dict["result_list"]) if result["metrics"]["test"]["AUC"] > 0.9]).reset_index())
        #explainer = shap.TreeExplainer(shap_mean_peaks_per_cluster_list[0][2])

        final_dict[cluster_name]["shap_summary_plot"]= shap.summary_plot(shap_mean_peaks_per_cluster_all_models.drop(columns=["model_num"]).groupby("index").mean().values, X[shap_mean_peaks_per_cluster_all_models.drop(columns=["index", "model_num"]).columns])

        aggregate_peak_info_mean_peaks_per_cluster = X[shap_mean_peaks_per_cluster_all_models.drop(columns=["index", "model_num"]).columns].assign(is_cluster = np.where(cluster_one_v_all_labels ==1, "cases_in_cluster", "cases_not_in_cluster")).groupby("is_cluster").agg(["mean", "median"]).swaplevel(-2,-1, axis=1)

        aggregate_peak_info_mean_peaks_per_cluster = pd.concat([aggregate_peak_info_mean_peaks_per_cluster["mean"].T.add_suffix("_mean"), aggregate_peak_info_mean_peaks_per_cluster["median"].T.add_suffix("_median")], axis=1)
        aggregate_peak_info_mean_peaks_per_cluster

        cluster_info_mean_peaks_per_cluster = shap_mean_peaks_per_cluster_all_models.drop_duplicates("model_num").drop(columns=["index", "model_num"]).count().to_frame()
        cluster_info_mean_peaks_per_cluster.columns = ["num_model_used_this_peak"]
        cluster_info_mean_peaks_per_cluster = cluster_info_mean_peaks_per_cluster.merge(aggregate_peak_info_mean_peaks_per_cluster, how = "outer", left_index = True, right_index=True)
        cluster_info_mean_peaks_per_cluster["cluster"] = cluster_name

        final_dict[cluster_name]["cluster_info_df"] = cluster_info_mean_peaks_per_cluster
        all_cluster_info_mean_peaks_per_cluster.append(cluster_info_mean_peaks_per_cluster)

    final_dict["combined_cluster_info_df"] = pd.concat(all_cluster_info_mean_peaks_per_cluster)
    final_dict["cluster_setting"] = cluster_setting_name
    return final_dict