# -*- coding: utf-8 -*-
"""
	Title:
	Description:
	Author:			haithem ben abdelaziz
	Date:
	Version:
	Environment:
"""

from __future__ import absolute_import

import stbt
import time
import logging
import os
import csv
import cv2
import inspect
import subprocess
import stbts
import unicodedata
from multiprocessing.pool import ThreadPool

logging.basicConfig(format="[%(asctime)s] %(message)s", level=logging.INFO)

def write_csv_file(file_name, rows, mode="a"):
    """

    write_csv_file  append to a csv file the rows passed on params if the file is exist
    and create it if not
    file_name: is the csv file name
    rows is a list of lines (rows) and not just one line
    mode:method of write into the csv file, default mode is append
    Example:
    write_csv_file("csv_file_name.csv", [ ["a", 1], ["b", 2], ["c", 3] ])
    the csv file will contain
    a | 1
    b | 2
    c | 3

    """
    csv_file = open(file_name, mode)
    writer = csv.writer(csv_file)
    writer.writerows(rows)

def detect_many_match(image, timeout_secs=10, match_parameters=None, region=stbt.Region.ALL):
    count = 0
    _exit = False
    # import time
    while (count < len(image) and _exit == False):
        if stbt.detect_match(image[count], timeout_secs=timeout_secs, match_parameters=match_parameters,
                             region=region).next().match:
            _exit = True
        count += 1
        # time.sleep(0.1)
    return _exit

def wait_for_many_match(image, timeout_secs=5, match_parameters=None, region=stbt.Region.ALL):
    _exit = False
    if stbt.wait_until(lambda: detect_many_match(image=image,
                                                 region=region,
                                                 match_parameters=match_parameters,
                                                 timeout_secs=10),
                       timeout_secs=timeout_secs,
                       interval_secs=0):
        _exit = True
        # time.sleep(0.3)
    return _exit

def press_until_many_match(key, image, max_presses=1, interval_secs=3, region=stbt.Region.ALL,match_parameters=None):
    occurence = 0
    _exit = False
    if match_parameters is None:
        match_parameters = stbt.get_config("navigation", "match_param_template")
    while not _exit and (occurence < max_presses):
        press(key)
        wait(interval_secs)
        for i in range(0, len(image)):
            try:
                _exit = stbt.wait_for_match(image=image[i],
                                            timeout_secs=interval_secs,
                                            region=region,
                                            match_parameters=match_parameters).match
                if _exit is True:
                    break
                time.sleep(0.3)
            except:
                pass
        occurence += 1
    return _exit

def wait(interval_secs):
    time.sleep(interval_secs)

def get_all_templates_in_directory(_path):
    result_templates = []
    list_dir = os.listdir(_path)
    for element in list_dir:
        if element.endswith(".png"):
            if _path.endswith("/"):
                result_templates.append(_path + element)
            else:
                result_templates.append(_path + "/" + element)

    return result_templates

def fuzzy_match(string1, string2, threshold=None):
    import difflib
    # logging.info("(found: %s - wanted: %s)", str(string1), str(string2))
    # logging.info(str(difflib.SequenceMatcher(None, string1, string2).ratio()))
    if difflib.SequenceMatcher(None, string1, string2).ratio() >= threshold:
        # stbt.draw_text("FOUND: " + "'" + string1 + "'", duration_secs=4)
        return True
    else:
        # stbt.draw_text("NOT FOUND: " + "'" + string2 + "'", duration_secs=4)
        return False
        # return difflib.SequenceMatcher(None, string1, string2).ratio() >= threshold

def crop(frame, region, file_name):
    frame_region = stbt.Region(x=0, y=0, width=frame.shape[1], height=frame.shape[0])
    region = stbt.Region.intersect(frame_region, region)
    stbt.save_frame(frame[region.y:region.bottom, region.x:region.right], file_name + ".png")

def get_text(region, frame=None, menu_text=None, threshold=None, mode=stbt.OcrMode.PAGE_SEGMENTATION_WITHOUT_OSD,
             tesseract_user_patterns=None, text_color=None, lang="eng"):
    """
    get_text, core, read text with ocr and compared with menu_text
    region: region where to detect the reference text
    Frame: None or disable parameter ==> detect frame automatically in real time

    """
    if threshold is None:
        threshold = stbt.get_config("core", "fuzzy_threshold", type_=float)

    text = stbt.ocr(frame=frame, region=region, mode=mode, lang=lang,
                    tesseract_user_patterns=tesseract_user_patterns, text_color=text_color)
    if menu_text is None:
        return text
    else:
        if not(isinstance(menu_text, str)):
            menu_text = menu_text.encode("utf-8")
        else:
            menu_text  = unicode(menu_text, "utf-8")
        return fuzzy_match(text, menu_text, threshold)

def is_text(region, frame=None, threshold=None, mode=stbt.OcrMode.PAGE_SEGMENTATION_WITHOUT_OSD,
             tesseract_user_patterns=None, text_color=None, lang="eng", timeout=7):
    """
    is_text: get text
    :return: True when text is not empty
    """
    if threshold is None:
        threshold = stbt.get_config("core", "fuzzy_threshold", type_=float)
    start_time = time.time()
    while time.time() - start_time < timeout:
        text = get_text(frame=frame,
                        threshold=threshold,
                        region=region,
                        mode=mode,
                        text_color=text_color,
                        tesseract_user_patterns=tesseract_user_patterns,
                        lang=lang)
        if text != "":
            debug("Text is not empty")
            return text
    assert False, "Empty text"

def ocr_until(region, menu_text, threshold, timeout, interval_secs, text_color=None,
              mode=stbt.OcrMode.PAGE_SEGMENTATION_WITHOUT_OSD,
              tesseract_user_patterns=None, lang="eng", filter=False):
    """
    ocr_until, core, implements get_text until found text or timeout

    region: region where to detect the reference text
    menu_text:  reference string used to refer to the menu

    """
    if filter == True:

        if filter_template(thresh_value=None,
                           callable=lambda: stbt.wait_until(lambda: get_text(region=region,
                                                                             frame=stbt.load_image("end_frame.png"),
                                                                             menu_text=menu_text,
                                                                             threshold=threshold,
                                                                             mode=mode,
                                                                             tesseract_user_patterns=tesseract_user_patterns,
                                                                             text_color=text_color,
                                                                             lang=lang),
                                                            timeout_secs=timeout,
                                                            interval_secs=interval_secs)):
            debug("FOUND: " + "'" +str(menu_text) + "'")
            return True
        else:
            return False
    else:
        frame = None
        if stbt.wait_until(
                lambda: get_text(region, frame, menu_text, threshold, mode, tesseract_user_patterns, text_color, lang),
                timeout_secs=timeout, interval_secs=interval_secs):
            debug("FOUND: " + "'" +str(menu_text) + "'")
            return True
        else:
            debug("NOT FOUND: " + "'" + str(menu_text) + "'")
            return False

def repeat(callable, occurence=None, time_out=1000, tolerance=None, wait_=2):
    """
    repeat, core, repeat a callable

    Repeating a processing of the functions for occurrence times
    time_out: unity time of execution on the function callable
    occurence: numbers of repeat
    tolerance: time added to global time_out
    """

    if occurence is not None:
        time_out = time_out * occurence
        if tolerance is None:
            tolerance = 10 * time_out / 100

    else:
        if tolerance is None:
            tolerance = 0

    start = time.time()
    repeat_test = 0
    while time.time() - start < (time_out + tolerance):
        callable()
        wait(wait_)
        if occurence is not None:
            repeat_test += 1
            if repeat_test == occurence:
                break

def debug(msg=None, text=""):
    # Get the previous frame in the stack, otherwise it would
    # be this function!!!
    func = inspect.currentframe().f_back.f_code
    if not( isinstance(msg, str) ):
        msg=msg.encode("utf-8")
    if not( isinstance(text, str) ):
        text=text.encode("utf-8")

    #logging.info("%s: %s", msg, str(text))
    #print "%s" % inspect.stack()[1][3]
    # Dump the message + the name of this function to the log.
    message = msg + text
    logging.info("%s in %s:%i : %s " % (func.co_name, func.co_filename, func.co_firstlineno, message))
    stbt.draw_text(msg + ":" + text, duration_secs=3)

def press(press, delay_sec=2, second_press=None, hold=False):
    """
     Press key using some delay

    """
    from android import press_adb
    quad = eval(stbt.get_config("quad","quad_press"))
    adb  = eval(stbt.get_config("adb","adb_press"))
    automate = eval(stbt.get_config("bluetooth" , "automate"))
    usb_automate = stbt.get_config("bluetooth" , "usb_automate")
    ble_press = eval(stbt.get_config("ble_rcu" , "ble_press"))
    if ble_press :
        blePress(press ,delay_sec, second_press, hold)
    elif adb :
         press_adb(key=press)
    elif quad :
        quad_press(key=press)
    elif automate :
        subprocess.call(get_generic_template_path()+"/../../../trunk/tests/bleutooth/./wakeup_by_RC_multi /dev/ttyUSB"+usb_automate+" "+press+" "+str(delay_sec*10), shell=True)

    else:
        stbt.press(press, interpress_delay_secs=delay_sec)
        logging.info("Pressing %s", str(press))

_dicBlePres= { "KEY_OK" : "101001" , "KEY_POWER" : "110001" , "KEY_1" : "111001" , "KEY_2" : "100001" , "KEY_3" : "101101" , "KEY_4" : "110101",
               "KEY_5" : "111101" , "KEY_6" : "100101" , "KEY_7" : "101011" , "KEY_8" : "110011" ,  "KEY_9" : "111011" ,  "KEY_0" : "100011",
               "KEY_MUTE" : "101111"  , "KEY_UP" : "110111" , "KEY_DOWN" : "111111" , "KEY_RIGHT" : "100111" , "KEY_VOICE" : "201001",
               "KEY_LEFT" : "210001" ,  "KEY_CHANNELUP" : "211001" ,  "KEY_CHANNELDOWN" : "200001" ,  "KEY_VOLUMEDOWN" : "201101",
               "KEY_VOLUMEUP" : "210101", "KEY_BACK" : "211101" , "KEY_STOP" : "200101" , "KEY_PLAY" : "200101" , "KEY_FASTFORWARD" : "201011",
               "KEY_REWIND" : "210011", "KEY_RECORD" : "211011" , "KEY_EXIT" : "200011" , "KEY_RED" : "201111",
               "KEY_GREEN" : "210111" , "KEY_YELLOW" : "211111" , "KEY_BLUE" : "200111", "KEY_MENU" : "301001",
               "KEY_ZOOM" : "310001" , "KEY_VIDEO_NEXT" : "311001" , "KEY_VIDEO_PREV" : "300001"  }

#dictionary for mapping pin-address
#Pins reserved pin1 to pin36

 #dic_pin = { "pin1" : "101001" | "pin9" :  "101011" | "pin17" : "201001" | "pin25" : "201011" | "pin33" : "301001" | "pin41" : "301011" |
            # "pin2" : "110001" | "pin10" : "110011" | "pin18" : "210001" | "pin26" : "210011" | "pin34" : "310001" | "pin42" : "310011" |
            # "pin3" : "111001" | "pin11" : "111011" | "pin19" : "211001" | "pin27" : "211011" | "pin35" : "311001" | "pin43" : "311011" |
            # "pin4" : "100001" | "pin12" : "100011" | "pin20" : "200001" | "pin28" : "200011" | "pin36" : "300001" | "pin44" : "300011" |
            # "pin5" : "101101" | "pin13" : "101111" | "pin21" : "201101" | "pin29" : "201111" | "pin37" : "301101" | "pin45" : "301111" |
            # "pin6" : "110101" | "pin14" : "110111" | "pin22" : "210101" | "pin30" : "210111" | "pin38" : "310101" | "pin46" : "310111" |
            # "pin7" : "111101" | "pin15" : "111111" | "pin23" : "211101" | "pin31" : "211111" | "pin39" : "311101" | "pin47" : "311111" |
            # "pin8" : "100101" | "pin16" : "100111" | "pin24" : "200101" | "pin32" : "200111" | "pin40" : "300101" | "pin48" : "300111" }

def setCommand(frame_key, init_frame="000000000000000"):
    """
    setCommand: configure frame key
    :param frame_key: frame key
    :return: new frame configured
    """
    # logging.info("Frame BLE %s", frame_key)

    if frame_key[0]=="1":
        key = frame_key[1:]+ init_frame[5:]

    elif frame_key[0]=="2":
        key = init_frame[:5] + frame_key[1:]+ init_frame[10:]

    elif frame_key[0]=="3":
        key = init_frame[:10] + frame_key[1:]

    else:
        return frame_key
    return key

def blePress(press, delay_sec, second_press, hold):
    """
    blePress: unit function to press with ble rcu
    :param press: keyt press
    :param delay_sec:  delay seconds between press
    :param second_press: the second press
    :param hold: enable hold press
    :return:
    """
    uart = stbt.get_config("ble_rcu", "uart")
    device = stbt.get_config("ble_rcu", "device")

    press_frame_init = _dicBlePres.get(press)
    press_frame = setCommand(press_frame_init)

    if second_press is not None:
        second_press_frame = _dicBlePres.get(second_press)
        second_press_frame = setCommand(second_press_frame)
        press_frame = setCommand(press_frame_init,second_press_frame)

    press_frame = uart + press_frame + "1"

    os.system("stty -F "+device+" 9600 cs8 -cstopb -parenb")

    subprocess.call("echo -e '"+press_frame+"' >"+device+"", shell=True)
    if second_press is not None:
        press = press+"+"+second_press
    logging.info("Pressing BL %s", press)
    stbt.draw_text(press)

    if hold:
        wait(delay_sec)

    subprocess.call("echo -e '00000000000000000' >"+device+"" , shell=True)

def quad_press(key):
    # Take the path of the valid folders
    dir_path = get_generic_template_path()+'/../../../trunk/_sc_stbt/'
    INTERPRESS_DELAY_SECS=0.3
    #Check the STB id on config file
    stb_id=stbt.get_config("quad","stb-id")
    logging.info("Pressing %s", str(press))
    #Printing the remote commands on the fifo file
    with open(dir_path + '/ir_commands', 'w') as fifo:
        ir_commands = "{} ".format(stb_id) + key + "\n"
        print ("Sending : " + ir_commands)
        fifo.write(ir_commands)
        fifo.flush()
        fifo.close()
        time.sleep(INTERPRESS_DELAY_SECS)
        stbt.draw_text(key)
        logging.info("Pressing %s", str(key))

def combo_press(combo=None, delay_sec=2, number_press=None):
    """
    combo_press ; press combo list press
    """
    if number_press is None:
        for i in range(0, len(combo)):
            press(combo[i], delay_sec=delay_sec)
    else:
        for j in range(number_press):
            for i in range(0, len(combo)):
                press(combo[i], delay_sec=delay_sec)

def get_generic_template_path():
    path = os.environ.get('STBT_PROJECT_PATH')
    return path

def write_statut_test(statut):
    """

    :return:
    """
    import os
    config_file = os.environ.get('STBT_CONFIG_FILE_CUSTOM')

    command = "sed -i -e 's,^'state' =.*,'state' = '" + statut + "',' " + config_file

    os.system(command)

def get_statut_test(message):
    """

    :return:
    """
    state = stbt.get_config("global", "state")
    if state == "test_ko":
        assert False, 'TEST IS BLOCKED: %s' % message

def write_statut_boot_test(statut):
    """

    :return:
    """
    import os
    config_file = os.environ.get('STBT_CONFIG_FILE_CUSTOM')

    command = "sed -i -e 's,^'boot_state' =.*,'boot_state' = '" + statut + "',' " + config_file

    os.system(command)

def get_statut_boot_test(message):
    """

    :return:
    """
    state = stbt.get_config("global", "boot_state")
    if state == "test_ko":
        assert False, 'TEST IS BLOCKED: %s' % message

list_timer = []

def timer(region_current_timer,
          region_end_timer,
          timer_value=False,
          press="KEY_UP",
          wait_after_press=1):
    """

    :param timer_value:
    :return:
    """
    import re
    global list_timer
    check_current_time = None
    check_end_time = None
    check_end_time2 = None
    list_timer.__init__()
    current_time = ''
    end_time = ''
    start_time = time.time()
    while (check_current_time is None and (
            check_end_time is None or check_end_time2 is None)) and time.time() - start_time < 5:
        # Read current time & end time
        current_time = get_text(region=region_current_timer,
                                mode=stbt.OcrMode.SINGLE_WORD,
                                tesseract_user_patterns=[r'\d\d\d\d\d']).replace(" ", "")
        end_time = get_text(region=region_end_timer,
                            mode=stbt.OcrMode.SINGLE_WORD,
                            tesseract_user_patterns=[r'\d\d\d\d\d']).replace(" ", "")

        # delete all caracteres only 0-9
        current_time = re.sub(r"[^0-9:]+", '', current_time)
        end_time = re.sub(r"[^0-9:]+", '', end_time)
        # check if current_time and end_time content a  right time format

        check_current_time = re.match(r"\d?\:?\d+\d?:\d+\d+", current_time)
        check_end_time = re.match(r"\d+\d?\:+\d+\d?", end_time)
        check_end_time2 = re.match(r"\d+\:+\d+\d?\:+\d+\d?", end_time)

    if (check_current_time is not None) or (check_end_time is not None or check_end_time2 is not None):
        # debug("CURRENT TIME",current_time)
        # debug("END_TIME",end_time)
        if timer_value:
            list_timer.append(current_time)
            list_timer.append(end_time)
        return True
    else:
        import sc_stbt
        if press is None:
            return False
        else:
            sc_stbt.press(press)
            wait(wait_after_press)
            return False

def get_timer_value(region_current_timer,
                    region_end_timer,
                    timer_value=True,
                    press="KEY_UP",
                    wait_after_press=1):
    """
    get_timer_value: get timer
    :return:
    """
    if timer(region_current_timer=region_current_timer,
             region_end_timer=region_end_timer,
             timer_value=timer_value,
             press=press,
             wait_after_press=wait_after_press):
        return list_timer
    else:
        assert False, "timer not detected"

def filter_template(thresh_value=None, callable=None):
    """

    :return:
    """
    stbt.save_frame(stbt.get_frame(), 'start_frame.png')
    img = cv2.imread('start_frame.png', 0)
    # create a CLAHE object (Arguments are optional).
    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
    cl1 = clahe.apply(img)
    cv2.imwrite('clahe.png', cl1)

    # binarisation
    im_gray = cv2.imread('clahe.png', cv2.CV_LOAD_IMAGE_GRAYSCALE)
    (thresh, im_bw) = cv2.threshold(im_gray, 0, 255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
    debug("Thresh detected:", str(thresh))
    if thresh_value is not None:
        thresh_compter = thresh_value
    else:
        thresh_compter = thresh
    if callable is None:
        debug("callable none")
        im_bw_1 = cv2.threshold(im_gray, thresh_compter, 255, cv2.THRESH_BINARY)[1]
        cv2.imwrite('end_frame.png', im_bw_1)
        debug("save end_frame with filter")
        return True
    else:
        while thresh_compter < 255:
            im_bw_1 = cv2.threshold(im_gray, thresh_compter, 255, cv2.THRESH_BINARY)[1]
            # cv2.imwrite('end_frame.png',im_bw_1)
            stbt.save_frame(im_bw_1, 'end_frame.png')
            thresh_compter += 10
            debug("Thresh", str(thresh_compter))
            if callable():
                return True
    return False

def write_technical_information(callable):
    """
    write_technical_information: this function write in file.txt the technical information
    :param callable: it is a function wish return two parametres: UI and MW
    :return: True if
    """
    ui_version, mw_version = callable()
    if ui_version and mw_version is not None:
        file = open('technical_information.txt', 'a')
        printinfo = "UI:" + ui_version + '\n' "MW:" + mw_version + '\n'
        file.write(printinfo.encode('utf8'))
        file.close()
        return True
    else:
        return False

def convert_mask_to_regions(path_mask):
    """
    Function: convert a mask to list of regions
    :param path_mask:
    :return: list of regions if mask contain white spaces
             None if passed mask is black screen
    """
    mask = cv2.imread(path_mask, cv2.CV_LOAD_IMAGE_GRAYSCALE)
    regions_list = []
    contours, hierarchy = cv2.findContours(mask, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)

    for contour in contours:
        if cv2.contourArea(contour) > 0:
            [x, y, width, height] = cv2.boundingRect(contour)
            region = stbt.Region(x=x, y=y, width=width, height=height)
            regions_list.append(region)

    if regions_list:
        return regions_list
    else:
        return None

def play_voice_command(command = None, android_ = None):
    """
    play_voice_command
    :return:
    """
    if android_ is None :

       android_ = eval(stbt.get_config("voice","android"))

    if android_ :
       voice_path= get_generic_template_path() + "/../../../android/tests/templates"
    else :
       voice_path= get_generic_template_path()

    command_=command.replace(" ","_")
    voice_path=(voice_path+"/../voice_templates/"+command_+".mp3")
    empty_path=(get_generic_template_path() + "/../../../android/tests/voice_templates/empty.mp3")
    # os.system("cvlc --play-and-exit "+empty_path)   #old api take much time in execution
    # os.system("cvlc --play-and-exit "+voice_path)   #old api take much time in execution
    os.system("mpg123  "+empty_path) #new api take less time in execution
    os.system("mpg123  "+voice_path) #new api take less time in execution

def press_voice_command(device=None, android_=True, port_voice=None ,press_time=None):
    """
    press_voice_command
    :param device:
    :param android_:
    :param port_voice:
    :param press_time:
    :return:
    """

    from android import remote_ATV_press,press_adb

    if port_voice is None :
       port_voice = stbt.get_config("voice", "port_voice")
    if press_time is None :
       press_time = stbt.get_config("voice", "press_time")

    debug("waiting for key voice")
    if device is None :
        usb_automate = stbt.get_config("bluetooth", "usb_automate")
        subprocess.call(get_generic_template_path()+"/../../../android/tests/bluetooth/./wakeup_by_RC_multi /dev/ttyUSB"+usb_automate+" "+port_voice+" "+press_time, shell=True)
    else :
        remote_ATV_press("KEY_VOICE")

    if android_ :
        if wait_for_many_match(get_all_templates_in_directory(get_generic_template_path()+ "/../../../android/tests/templates/voice/" ),
                                                 region=eval(stbt.get_config("voice","logo_voice")),
                                                 timeout_secs=10):
            debug("Key voice pressed")
        else :
            assert False , "No Key Voice"
    else:
        if wait_for_many_match(get_all_templates_in_directory(get_generic_template_path()+ "/voice/" ),
                                                 region=eval(stbt.get_config("voice","logo_voice")),
                                                 timeout_secs=10):
            debug("Key voice pressed")
        else :
            assert False , "No Key Voice"

def voice_command(callable=None, command=None, device=None, port_voice=None, press_time=None, android_= None,wait_ = 5,threshold = 0.8):
    """
    Press on KEY_VOICE and command the box then check if the callable is done
    :return: True when the command is excecuted , False if not
    """
    press_voice_command(device ,android_ , port_voice , press_time)
    wait(1)
    play_voice_command(command , android_)

    if stbts.match_text(text=command,region=eval(stbt.get_config("voice","region_command")) ,threshold=threshold, timeout_secs=5):
        debug("Your command is "+ command)
    else:
        assert False, "Your request is not considered"
    wait(wait_)
    if callable is None :
        pass
    else :
        if callable():
            return True, " OK  voice command executed "
        else:
            assert False ," FAIL  voice command NOT executed "

def multi_threading(callable_1, callable_2, callable_3=None, callable_4=None):
    """
    multi_threading:  run 4 API in parallel
    :param callable_1: API 1
    :param callable_2: API 2
    :param callable_3: API 3
    :param callable_4: API 4
    :return:
    """
    if callable_3 is None and callable_4 is None:
        pool = ThreadPool(processes=2)
        pool.map(apply, [callable_1, callable_2])
        pool.terminate()

    if (callable_3 is not None or callable_4 is not None) and (callable_3 is None or callable_4 is None):
        pool = ThreadPool(processes=3)
        pool.map(apply, [callable_1, callable_2, callable_3])
        pool.terminate()

    if callable_3 is not None and callable_4 is not None:
        pool = ThreadPool(processes=4)
        pool.map(apply, [callable_1, callable_2, callable_3, callable_4])
        pool.terminate()


# import libraries
import argparse
import logging
import os
import pickle
import random
import numpy as np
import torch
import pandas as pd
from tqdm import tqdm

from transformers import (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer)
from vae_gpt2_model import VAE_config, VAE_GPT2

logger = logging.getLogger(__name__)
MODEL_CLASSES = {
    'gpt2': (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer)}

"""====================== CLASSES AND METHODS DEFINITIONS ======================"""

def set_seed(args):
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

# truncating/padding method
def truncating_padding_sentence(tokens, block_size):
    if (len(tokens) > block_size):
        original_tokens_len = block_size
        tokens = tokens[:block_size]
    else:
        original_tokens_len = len(tokens)
        tokens = tokens + ["[PAD]"]*(block_size - len(tokens))
    return tokens, original_tokens_len
# creating attention mask method
def create_attention_mask(sentence_length, seq_length, gpt2_config, mask_type = "encoder_mask"):
    # args:
    #   sentence_length is length of real text, from [SOS] to <|endoftext|>
    #   seq_length is length with [PAD] (32, 64, 128, ...)

    if mask_type == "encoder_mask":
        mask_one_head = np.zeros([seq_length, seq_length])
        mask_one_head[:, :sentence_length] = 1  # the attention of [PAD] is also the input sentence
        mask_all_heads = [mask_one_head] * gpt2_config.n_head
        mask_all_heads = np.array(mask_all_heads)
    if mask_type == "decoder_mask":
        # attention, the triangular matrix is [seq_length,seq_length+1] becasuse the original has one more "past" token
        mask_one_head = np.tril(np.ones([seq_length,seq_length+1]),1)
        mask_all_heads = [mask_one_head] * gpt2_config.n_head
        mask_all_heads = np.array(mask_all_heads)
    return mask_all_heads

def convert_tensor_inference(model, sentence_embedding, args, device, current_generated_sentences):

    # convert to tensor and put on device
    sentence_embedding_converted = torch.FloatTensor(sentence_embedding).unsqueeze(0).to(device)

    # generate sentence
    generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = sentence_embedding_converted, args = args, device = device)
    generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
    first_endoftext = generated_sample.find("<|endoftext|>")
    generated_sample = str(generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)])
    count = 1
    while ((generated_sample in current_generated_sentences) and (count<10)):
        generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = sentence_embedding_converted, args = args, device = device)
        generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
        first_endoftext = generated_sample.find("<|endoftext|>")
        generated_sample = str(generated_sample[:(first_endoftext)])
        count += 1

    current_generated_sentences.append(generated_sample)

    ## DEBUGGING
    # logger.info("len(current_generated_sentences): " + str(len(current_generated_sentences)))
    # logger.info("current_generated_sentences: " + str(current_generated_sentences))
    # logger.info("generated_sample: " + str(generated_sample))
    # print("count+: "+ str(count))

    # print generated sentence sample
    generated_sample = generated_sample[6: ]
    print(str(generated_sample))

    return current_generated_sentences

def inference_test_1(model, args, device):
    print(" === inference_test_1 ===")
    print("Task 1: generate text from input sentence")

    # reading file
    sentences = []
    with open(args.sentences_file, encoding="utf-8") as file_p:
        for sentence in file_p:
            sentences.append(sentence)

    # tokenizing, truncating/padding
    tokenized_sentences = []
    converted_tokenized_sentences = []
    attention_mask_sentences = []
    for sentence in sentences:
        sentence_tokenized = model.tokenizer.tokenize(sentence)
        # encoder_input
        encoder_input = sentence_tokenized
        encoder_input, encoder_input_len = truncating_padding_sentence(encoder_input, args.block_size)
        encoder_input = model.tokenizer.convert_tokens_to_ids(encoder_input)
        encoder_input = np.array(encoder_input)
        # encoder_attention_mask
        encoder_attention_mask = create_attention_mask(encoder_input_len, args.block_size, args.gpt2_config, "encoder_mask")

        tokenized_sentences.append(truncating_padding_sentence(sentence_tokenized, args.block_size))
        converted_tokenized_sentences.append(encoder_input)
        attention_mask_sentences.append(encoder_attention_mask)

    # get average embeddings and inferring
    for i in range(len(sentences)):
        sentence = sentences[i]
        sentence_tokenized = tokenized_sentences[i]
        converted_tokenized_sentence = converted_tokenized_sentences[i]
        attention_mask_sentence = attention_mask_sentences[i]
        sentence_tokenized = model.tokenizer.decode(model.tokenizer.convert_tokens_to_ids(sentence_tokenized), clean_up_tokenization_spaces=True)
        converted_tokenized_sentence = torch.tensor(converted_tokenized_sentence).long().unsqueeze(0).to(device) # adding batch_size dimension
        attention_mask_sentence = torch.tensor(attention_mask_sentence).long().unsqueeze(0).to(device) # adding batch_size dimension

        sentence_embedding, z = model.get_embedding(converted_tokenized_sentence, attention_mask_sentence, device)

        ## DEBUGGING
        # logger.info("converted_tokenized_sentence_1: " + str(converted_tokenized_sentence_1.shape))
        # logger.info("attention_mask_sentence_1: " + str(attention_mask_sentence_1.shape))
        # logger.info("sentence_1_embedding: " + str(len(sentence_1_embedding)))
        # logger.info("sentence_1_embedding[0]: " + str(sentence_1_embedding[0].shape))
        # logger.info("average_embedding: " + str(average_embedding[0][:5]))

        # inferring from average embedding
        generated, decoder_attentions = model.inference(sentence_embedding = sentence_embedding, args = args, device = device)
        generated_sentence =  model.tokenizer.decode(generated[0].tolist(), clean_up_tokenization_spaces=True)

        print("=====")
        print("input_sentence: " + str(sentence_tokenized))
        first_endoftext = generated_sentence.find("<|endoftext|>")
        print("generated_sentence: " + str(generated_sentence[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sentence)]))

    return

def inference_test_2(model, args, device):
    print(" === inference_test_2 ===")
    print("Task 2: generate text from each dimension")

    # ==== Compute the embeddings
    # check cached file
    sentences_directory, sentences_filename = os.path.split(args.sentences_file)
    model_directory = args.output_dir
    cache_embeddings_file = os.path.join(model_directory, 'cached_inference_' + sentences_filename)
    logger.info("cache_embeddings_file: " + cache_embeddings_file)
    if os.path.exists(cache_embeddings_file) and not args.overwrite_cache:
        logger.info("Loading features from cached file %s", cache_embeddings_file)
        with open(cache_embeddings_file, 'rb') as handle:
            # sentences_texts, sentences_embeddings = pickle.load(handle)
            sentences_ids, sentences_texts, sentences_embeddings = pickle.load(handle)

    # if no cached file
    else:
        # reading file
        print("reading input file.")
        sentences = []
        with open(args.sentences_file, encoding="utf-8") as file_p:
            for count, sentence in enumerate(file_p):
                sentence = sentence.strip("\n")
                sentences.append(sentence)

        # tokenizing, truncating/padding
        print("tokenizing and processing.")
        converted_tokenized_sentences = []
        attention_mask_sentences = []
        for sentence in sentences:
            sentence_tokenized = model.tokenizer.tokenize(sentence)
            # encoder_input
            encoder_input = sentence_tokenized
            encoder_input, encoder_input_len = truncating_padding_sentence(encoder_input, args.block_size)
            encoder_input = model.tokenizer.convert_tokens_to_ids(encoder_input)
            encoder_input = np.array(encoder_input)
            # encoder_attention_mask
            encoder_attention_mask = create_attention_mask(encoder_input_len, args.block_size, args.gpt2_config, "encoder_mask")

            converted_tokenized_sentences.append(encoder_input)
            attention_mask_sentences.append(encoder_attention_mask)

        # get embeddings
        batch_size = args.batch_size
        print("get embeddings with batch size {}.".format(str(batch_size)))
        sentences_ids = []
        sentences_embeddings = []
        sentences_texts = []
        for i in tqdm(list(range(0, len(sentences), batch_size))):
            sentences_text_batch = sentences[i:min(i+batch_size, len(sentences))]
            converted_tokenized_sentence = converted_tokenized_sentences[i:min(i+batch_size, len(sentences))]
            attention_mask_sentence = attention_mask_sentences[i:min(i+batch_size, len(sentences))]
            # convert to tensor, move to device
            converted_tokenized_sentence = torch.tensor(converted_tokenized_sentence).long().to(device)
            attention_mask_sentence = torch.tensor(attention_mask_sentence).long().to(device)
            # get embedding
            sentence_embedding, z = model.get_embedding(converted_tokenized_sentence, attention_mask_sentence, device)
            # append
            if sentences_embeddings == []:
                sentences_ids = list(range(i,min(i+batch_size, len(sentences))))
                sentences_embeddings = sentence_embedding.detach().cpu().numpy()
                sentences_texts = sentences_text_batch
            else:
                sentences_ids.extend(list(range(i,min(i+batch_size, len(sentences)))))
                sentences_embeddings = np.concatenate((sentences_embeddings, sentence_embedding.detach().cpu().numpy()), axis = 0)
                sentences_texts.extend(sentences_text_batch)
        logger.info("sentences_embeddings.shape: " + str(sentences_embeddings.shape))

        # save to file
        with open(cache_embeddings_file, 'wb') as handle:
            pickle.dump([sentences_texts, sentences_embeddings], handle)
            logger.info("dump file to: {}, number of records: {}, file size: {} MB".format(cache_embeddings_file, str(len(sentences_embeddings)), str(os.path.getsize(cache_embeddings_file)/pow(1024,2))))

        # save to file
        with open(cache_embeddings_file, 'wb') as handle:
            pickle.dump([sentences_ids, sentences_texts, sentences_embeddings], handle)
            logger.info("dump file to: {}, number of records: {}, file size: {}".format(cache_embeddings_file, str(len(sentences_embeddings)), str(os.path.getsize(cache_embeddings_file))))
            # save as csv file
            cache_embeddings_file_csv, file_extension = os.path.splitext(cache_embeddings_file)
            cache_embeddings_file_csv += '.csv'
            logger.info("also save to csv file: {}".format(cache_embeddings_file_csv))
            text_df = pd.DataFrame(sentences_texts)
            sentences_ids = [str(sentences_id) for sentences_id in sentences_ids]
            text_df.index = sentences_ids
            text_df.insert(0, 'message_id', sentences_ids)
            text_df.columns = ["message_id", "message"]
            embeddings_df = pd.DataFrame(sentences_embeddings)
            embeddings_df.index = sentences_ids
            embeddings_df.columns = ["COMPONENT_" + str(i) for i in range(sentences_embeddings.shape[1])]
            df = text_df.merge(embeddings_df, left_on = text_df.index, right_on = embeddings_df.index)
            df = df.drop(columns=['key_0'])
            df.message_id = df.message_id.astype(str)
            df.to_csv(cache_embeddings_file_csv, index = False)

    # ==== Analyzing the embeddings
    # method_1 for testing_2
    if args.method == "method_1":
        # method 1 (just use the embedding)
        # analyzing, find std and mean of each hidden dimension
        means = np.mean(sentences_embeddings, axis = 0)
        stds = np.std(sentences_embeddings, axis = 0)

        # set parameters
        std_steps = 5
        std_step_interval = 0.5
        std_random_level = 0
        print("UPDATED!")

        # generate sentence for each hidden dimension
        hidden_size = means.shape[0]
        for i in range(hidden_size):

            print("=====")
            print("HIDDEN DIMENISON " + str(i) + ":")

            # explore the positive direction
            for step in range(std_steps, -1, -1):
                std_position = (step+1)*std_step_interval

                print("samples around mean + {}*std:".format(std_position))
                print("generation avoid repeating!")
                generated_samples = []   # avoid repeated generated_sample
                for _ in range(args.generate_num):

                    # sample embedding around embedding + std_position*stds[i]
                    epsilon = np.random.uniform(-std_random_level,std_random_level)
                    embedding_sample = np.copy(means)
                    embedding_sample[i] = embedding_sample[i] + std_position*stds[i] + epsilon*stds[i]
                    embedding_sample = torch.tensor(embedding_sample, device = device).unsqueeze(0)

                    # generate sentence
                    generatied_count = 0
                    while True:
                        generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = embedding_sample, args = args, device = device)
                        generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
                        generatied_count += 1
                        first_endoftext = generated_sample.find("<|endoftext|>")
                        generated_sample_clean = generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]
                        if (generated_sample_clean not in generated_samples) or generatied_count >= 10:
                            generated_samples.append(generated_sample_clean)
                            break

                    # print generated sentence sample
                    first_endoftext = generated_sample.find("<|endoftext|>")
                    print("generated_sample: " + str(generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]))

            # explore the negative direction
            for step in range(0, std_steps + 1, 1):
                std_position = (step+1)*std_step_interval

                print("samples around mean - {}*std:".format(std_position))
                for _ in range(args.generate_num):

                    # sample embedding around embedding - std_position*stds[i]
                    epsilon = np.random.uniform(-std_random_level,std_random_level)
                    embedding_sample = np.copy(means)
                    embedding_sample[i] = embedding_sample[i] - std_position*stds[i] + epsilon*stds[i]
                    embedding_sample = torch.tensor(embedding_sample, device = device).unsqueeze(0)

                    # generate sentence
                    generatied_count = 0
                    while True:
                        generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = embedding_sample, args = args, device = device)
                        generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
                        generatied_count += 1
                        first_endoftext = generated_sample.find("<|endoftext|>")
                        generated_sample_clean = generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]
                        if (generated_sample_clean not in generated_samples) or generatied_count >= 10:
                            generated_samples.append(generated_sample_clean)
                            break

                    # print generated sentence sample
                    first_endoftext = generated_sample.find("<|endoftext|>")
                    print("generated_sample: " + str(generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]))

    # method_2 for testing_2
    if args.method == "method_2":

        # sentences_ids, sentences_texts, sentences_embeddings = pickle.load(handle)
        df = pd.DataFrame(np.array(sentences_embeddings))
        df.columns = [str(latent_dim) for latent_dim in range(len(sentences_embeddings[0]))]
        df.index = sentences_ids
        df['text'] = sentences_texts
        print(df.columns)
        print(df.index[:5])

        # compute *std score
        hidden_size = len(sentences_embeddings[0])
        means = np.mean(sentences_embeddings, axis = 0)
        stds = np.std(sentences_embeddings, axis = 0)
        for i in range(hidden_size):
            df['*std_' + str(i)] = round((df[str(i)] - means[i])/stds[i],2)

        ## get unique sentence with highest score for each hidden dimension
        print("(=)(=)(=) unique sentence highest score (=)(=)(=)")
        for i in range(hidden_size):
            print("=====")
            print("HIDDEN DIMENISON " + str(i) + ":")

            top_count = 10
            print("(*) top max score: ")
            sorted_df = df.sort_values(by=[str(i)], ascending=False)
            printed_sentences = []
            for index, row in sorted_df.iterrows():
                if row['text'] not in printed_sentences:
                    count = len(sorted_df[sorted_df['text'] == row['text']])
                    print("{} - score: {} - count: {}".format(row['text'], round(row[str(i)],2), count))
                    printed_sentences.append(row['text'])
                if len(printed_sentences)== top_count:
                    break

            print("(*) top min score: ")
            sorted_df = df.sort_values(by=[str(i)], ascending=True)
            printed_sentences = []
            for index, row in sorted_df.iterrows():
                if row['text'] not in printed_sentences:
                    count = len(sorted_df[sorted_df['text'] == row['text']])
                    print("{} - *std score: {} - count: {}".format(row['text'], round(row['*std_' + str(i)],2), count))
                    printed_sentences.append(row['text'])
                if len(printed_sentences)== top_count:
                    break

    # method_3 for testing_2
    if args.method == "method_3":

        # sentences_ids, sentences_texts, sentences_embeddings = pickle.load(handle)
        df = pd.DataFrame(np.array(sentences_embeddings))
        df.columns = [str(latent_dim) for latent_dim in range(len(sentences_embeddings[0]))]
        df.index = sentences_ids
        df['text'] = sentences_texts
        print(df.columns)
        print(df.index[:5])

        # compute *std score
        hidden_size = len(sentences_embeddings[0])
        means = np.mean(sentences_embeddings, axis = 0)
        stds = np.std(sentences_embeddings, axis = 0)
        for i in range(hidden_size):
            df['*std_' + str(i)] = round((df[str(i)] - means[i])/stds[i],2)

        ## get sentence with highest frequency for each hidden dimension
        print("(=)(=)(=) sentence highest frequency (=)(=)(=)")
        for i in range(hidden_size):
            print("=====")
            print("HIDDEN DIMENISON " + str(i) + ":")

            top_count = 8
            std_threshold = 1
            print("(*) top highest frequency above {}*std ".format(str(std_threshold)))
            df_pass_threshold = df.loc[df[str(i)] >= (means[i] + std_threshold*stds[i])]
            df_pass_threshold['text_std'] = df_pass_threshold['text'] + '    ' + df_pass_threshold[('*std_' + str(i))].astype(str)
            print(df_pass_threshold['text_std'].value_counts()[:top_count])
            print("(*) top highest frequency below -{}*std: ".format(str(std_threshold)))
            df_pass_threshold = df.loc[df[str(i)] <= (means[i] - std_threshold*stds[i])]
            df_pass_threshold['text_std'] = df_pass_threshold['text'] + '    ' + df_pass_threshold[('*std_' + str(i))].astype(str)
            print(df_pass_threshold['text_std'].value_counts()[:top_count])

            df_pass_threshold.groupby(['text', '*std_' + str(i)])

    return

"""====================== MAIN FUNCTION ======================"""

# main function
def main():
    parser = argparse.ArgumentParser()

    # dataset/save path parameters
    parser.add_argument("--sentences_file", default=None, type=str,
                        help="An optional input generate data file.")
    parser.add_argument('--overwrite_cache', action='store_true',
                        help="Overwrite the cached training and evaluation sets")
    parser.add_argument("--primals_file", default=None, type=str,
                        help="An optional input generate data file.")
    parser.add_argument("--output_dir", default=None, type=str, required=False,
                        help="The output directory where the model predictions and checkpoints will be written.")

    # model parameters
    parser.add_argument("--gpt2_model_type", default="bert", type=str,
                        help="The model architecture to be fine-tuned.")
    parser.add_argument("--gpt2_model_name_or_path", default="bert-base-cased", type=str,
                        help="The model checkpoint for weights initialization.")
    parser.add_argument("--latent_size", default=-1, type=int, required=True,
                        help="Size of latent VAE layer.")
    parser.add_argument("--do_lower_case", action='store_true',
                        help="Set this flag if you are using an uncased model.")

    # generating parameters
    parser.add_argument("--inference_test", default=0, type=int)
    parser.add_argument("--method", type=str, default=None)
    parser.add_argument("--generate_num", type=int, default=None)
    parser.add_argument("--generate_length", type=int, default=None)
    parser.add_argument("--temperature", type=float, default=1.0)
    parser.add_argument("--top_k", type=int, default=0)
    parser.add_argument("--top_p", type=float, default=0.9)
    parser.add_argument("--block_size", default=-1, type=int,
                        help="Optional input sequence length after tokenization. The training dataset will be truncated in block of this size for training. Default to the model max input length for single sentence inputs (take into account special tokens).")
    parser.add_argument("--batch_size", type=int, default=None)

    # other generating parameters
    parser.add_argument("--no_cuda", action='store_true',
                        help="Avoid using CUDA when available")
    parser.add_argument('--seed', type=int, default=42,
                        help="random seed for initialization")

    # other parameters
    parser.add_argument('--fp16', action='store_true',
                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
    parser.add_argument('--fp16_opt_level', type=str, default='O1',
                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
                             "See details at https://nvidia.github.io/apex/amp.html")
    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")

    # parsing parameters
    args = parser.parse_args()

    # =========== checking parameters and setting up  =========== #
    # setting things up
    # Setup CUDA, GPU & distributed training
    device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")    # CHECK! make sure we use all 3 GPUs
    args.n_gpu = torch.cuda.device_count()
    args.device = device
    # Setup logging
    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt = '%m/%d/%Y %H:%M:%S',
                        level = logging.INFO)
    # Set seed
    set_seed(args)

    # =========== bulilding model and inferencing  =========== #
    # Building model
    gpt2_config_class, gpt2_class, tokenizer_class = MODEL_CLASSES[args.gpt2_model_type]
    gpt2_config = gpt2_config_class.from_pretrained(args.gpt2_model_name_or_path, cache_dir = None)
    vae_config = VAE_config(latent_size = args.latent_size)
    model = VAE_GPT2(gpt2_config, vae_config, args)

    # Load from checkpoint model
    model.from_pretrained(args)
    if args.block_size <= 0:  # modify args.block_size variable
        args.block_size = model.tokenizer.max_len_single_sentence  # Our input block size will be the max possible for the model
    args.block_size = min(args.block_size, model.tokenizer.max_len_single_sentence)

    # Send model to GPU
    model.to(args.device)

    # Logging info
    logger.info("Inference parameters %s", args)

    # Testing inference
    args.gpt2_config = model.gpt2_config

    if args.inference_test == 1:
        # Inference test 1 - regenerate text
        inference_test_1(model, args, device)
    if args.inference_test == 2:
        # Inference test 2 - sample for each of latent dimension
        inference_test_2(model, args, device)
    if args.inference_test == 3:
        # Inference test 3 - interpolation generating
        inference_test_3(model, args, device)

if __name__ == "__main__":
    main()


import vk
from datetime import date
from collections import deque
import json
from time import time, sleep
from pymongo import MongoClient

connection = MongoClient("<ds>")
database = connection['grigins_base']

collection = database['AdTargeting']
id = collection.find_one()
while id is None:
    id = collection.find_one()
groupId = id["group_id"]

sleepcollection = database['sleep']
sleepID = sleepcollection.find_one()
while sleepID.get('sleepMode') is None:
    sleepID = sleepcollection.find_one()
sleepMode = sleepID["sleepMode"]

print('Starting')

lt = 0
v = 5.64
FPS = 18
now = int(date.today().isoformat().split('-')[0])

def letters(text):
    text = text.lower()
    newt = ''
    text.replace('\n\n', ' ')
    text.replace('\n', ' ')
    for i in text:
        if (ord('а') <= ord(i) <= ord('я')) or i == ' ' or (ord('a') <= ord(i) <= ord('z')):
            newt += i
    return newt

def CheckGroup(id_user):
    offset = 0
    array = []
    try:
        while True:
            resp = api().groups.get(user_id=id_user, offset=offset, v=5.64, fields="description, status", extended=1)
            tick(FPS)
            array += resp["items"]
            offset += 1000
            if offset > resp["count"]:
                break
    except:
        pass
    newarr = []
    for i in array:
        d = dict()
        d['description'] = letters(i.get('description', ''))
        d['name'] = letters(i['name'])
        newarr.append(d)
    return newarr

def get(id_group):
    arr = []
    c = 0
    for i in getMembers(id_group):
        c += 1
        print(c)
        arr += getFriends(i)
    return arr

def getMembers(id_group):
    offset = 0
    array = []

    while True:
        resp = api().groups.getMembers(group_id=id_group, offset=offset, fields="bdate", v=5.64)
        tick(FPS)
        array += resp["items"]
        offset += 1000
        if offset > resp["count"]:
            break
    return [x['id'] for x in array]

def getFriends(id_user):
    offset = 0
    array = []
    minage = 12
    maxage = 20

    fyear, lyear = now - maxage, now - minage
    try:
        while True:
            resp = api().friends.get(user_id=id_user, offset=offset, v=5.64, fields="bdate")
            tick(FPS)
            array += resp["items"]
            offset += 5000
            if offset > resp["count"]:
                break
    except:
        pass
    narr = []
    for person in array:
        if not person.get('bdate') is None:
            bdate = person['bdate'].split('.')
            if len(bdate) == 3:
                year = int(bdate[2])
                if fyear <= year <= lyear:
                    narr.append(person['id'])
            else:
                narr.append(person['id'])
        else:
            narr.append(person['id'])
    return narr

def api():
    global ses
    ans = ses.popleft()
    ses.append(ans)
    return ans

def tick(t):
    t = 1/t
    global lt
    if time() - lt >= t:
        pass
    else:
        sleep(t - (time() - lt))
    lt = time()

def usend(sid, mess, fm=False):
    if fm:
        api().messages.send(v=v, user_id=sid, message=mess, forward_messages=fm)
    else:
        api().messages.send(v=v, user_id=sid, message=mess)

session1 = vk.Session(access_token='token1')
api1 = vk.API(session1)
session2 = vk.Session(access_token='token2')
api2 = vk.API(session2)
session3 = vk.Session(access_token='token3')
api3 = vk.API(session3)
session4 = vk.Session(access_token='token4')
api4 = vk.API(session4)
session5 = vk.Session(access_token='token5')
api5 = vk.API(session5)
session6 = vk.Session(access_token='token6')
api6 = vk.API(session6)
ses = deque([api1, api2, api3, api4, api5, api6])

with open('bin.txt', 'w') as f:
    f.write('0')

print('In process')
members = {x:1 for x in getMembers(groupId)}
a = get(groupId)
print(a)
ids = [x for x in a if members.get(x) is None]
print('Got ids')
parts = []

for x in range(len(ids) // 1000 + 1):
    print(x, len(ids) // 1000 + 1, 'getting userinfo')
    parts += api().users.get(user_ids=ids[x*1000:(x+1)*1000], v=5.64, fields='activities, interests')
    tick(FPS)

print(len(parts), len(ids))
with open('info.txt', 'w') as f:
    pass

for user in parts:
    with open('info.txt', 'a') as f:
        f.write(json.dumps([user['id'], user.get('activities', '') + '\t' + user.get('interests', '')]) + '\n')

with open('info3.txt', 'w') as f:
    pass
c = 0
for id in ids:
    c += 1
    print(c, len(ids), 'infogroups')
    with open('info3.txt', 'a') as f:
        f.write(json.dumps([id] + CheckGroup(id)) + '\n')

with open('info.txt', 'r') as f:
    res = [json.loads(x) for x in f.readlines()]

with open('info3.txt', 'r') as f:
    res2 = [json.loads(x) for x in f.readlines()]

with open('final.txt', 'w') as f:
    print(len(res), len(res2))
    for x in range(len(res)):
        res[x].append(res2[x])
        f.write(json.dumps(res[x]) + '\n')

print('Done')
import just_a_good


import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from GSKpy.BasicGSK import  BasicGSK
# Load the diabetes dataset
diabetes_X, diabetes_y = datasets.load_diabetes(return_X_y=True)

# Use only one feature
#diabetes_X = diabetes_X[:, np0]
print('Train data shape:',diabetes_X.shape)
diabetes_X_train = diabetes_X[:-20]
diabetes_X_test = diabetes_X[-20:]

# Split the targets into training/testing sets
diabetes_y_train = diabetes_y[:-20]
diabetes_y_test = diabetes_y[-20:]

regr = linear_model.LinearRegression()
regr.fit(diabetes_X_train, diabetes_y_train)
diabetes_y_pred = regr.predict(diabetes_X_test)
print('MSE=',mean_squared_error(diabetes_y_test, diabetes_y_pred))
dim = len(regr.coef_)
def evaluateFun(coef,func_args=None):
    fitness_val = []
    for co in coef:
        regr.coef_ = co
        diabetes_y_pred = regr.predict(diabetes_X_train)
        #print(mean_squared_error(diabetes_y_train, diabetes_y_pred))
        fitness_val.append(mean_squared_error(diabetes_y_train, diabetes_y_pred))
    return np.array(fitness_val)

for i in range(30):
    solver = BasicGSK(evaluateFun, dim, 200, [-1000]*dim,[1000]*dim,LPSR=True,kr=0.1,max_nfes=100000)
    g, best , best_fit, loss = solver.run(optimum=0)
    regr.coef_ = best
    diabetes_y_pred = regr.predict(diabetes_X_test)
    print()
    print('MSE=',mean_squared_error(diabetes_y_test, diabetes_y_pred))


import pandas as pd
import folium
from folium.plugins import MarkerCluster

cycleincidences = pd.read_csv('/users/charlotte1/documents/cfgproject/finaldata.csv')

cyclemap = folium.Map(location = [51.5074, -0.1278],
                      tiles='CartoDB positron',
                      zoom_start = 11,
                      width = 800, height = 600)

marker_cluster = MarkerCluster().add_to(cyclemap)

for each in cycleincidences.iterrows():
     folium.Marker(
        location = [each[1]['X'],each[1]['Y']]
        ).add_to(marker_cluster)

cyclemap.save('cyclesmap4.html')


"""
    Implementation of bagged models
"""
import multiprocessing
import numpy as np
from joblib import Parallel, delayed

class BaggedModel:
    """
        Implements a bagged model.

        Used methods:
        * fit, get_rank_cell
    """

    def __init__(self, n_sample, card_sample, model_instantiator):
        self.n_sample = n_sample
        self.card_sample = card_sample
        self.parallel = True
        # private
        self._model_instantiator = model_instantiator
        self._fitted_models = list()

    def fit(self, X, y):
        """Fits several trees."""
        samples = [self._get_random_sample(X, y)
                   for _ in range(self.n_sample)]

        if self.parallel:
            pool = Parallel(n_jobs=multiprocessing.cpu_count()-10)
            self._fitted_models = pool(
                delayed(self._get_fitted_model)(X_samp, y_samp)
                for X_samp, y_samp in samples
            )
        else:
            self._fitted_models = [
                self._get_fitted_model(X_samp, y_samp)
                for X_samp, y_samp in samples
            ]

    def median_rank(self, X):
        """
        Returns the median of the rank of the cells of the X's.

        Which with that notation just means that lots of cells
        will have the same rank.
        """
        rank_values = list()

        if self.parallel:
            pool = Parallel(n_jobs=multiprocessing.cpu_count()//2)
            rank_values = pool(delayed(lambda x: x.get_rank_cell(X))(model)
                               for model in self._fitted_models)
        else:
            for model in self._fitted_models:
                rank_values.append(model.get_rank_cell(X))
        rank_array = np.array(rank_values).transpose()
        return np.mean(rank_array, axis=1)
        # return np.median(rank_array, axis=1)

    def score(self, X):
        """A score, derived from the median rank."""
        return - self.median_rank(X)

    def _get_random_sample(self, X, y):
        ind_sample = np.random.randint(0, X.shape[0], self.card_sample)
        return X[ind_sample], y[ind_sample]

    def _get_fitted_model(self, X_resampled, y_resampled):
        cur_model = self._model_instantiator()
        cur_model.fit(X_resampled, y_resampled)
        return cur_model


"""empty message

Revision ID: d91e0e2fb7a3
Revises: 9e6583e6f8a6
Create Date: 2023-10-29 04:12:30.717424

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = 'd91e0e2fb7a3'
down_revision = '9e6583e6f8a6'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('board', schema=None) as batch_op:
        batch_op.alter_column('member_name',
               existing_type=mysql.INTEGER(),
               type_=sa.String(length=30),
               nullable=False)
        batch_op.create_foreign_key(None, 'users', ['member_name'], ['member_name'], ondelete='CASCADE')

    with op.batch_alter_table('comment', schema=None) as batch_op:
        batch_op.add_column(sa.Column('member_name', sa.String(length=30), nullable=False))
        batch_op.create_foreign_key(None, 'users', ['member_name'], ['member_name'], ondelete='CASCADE')

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('comment', schema=None) as batch_op:
        batch_op.drop_constraint(None, type_='foreignkey')
        batch_op.drop_column('member_name')

    with op.batch_alter_table('board', schema=None) as batch_op:
        batch_op.drop_constraint(None, type_='foreignkey')
        batch_op.alter_column('member_name',
               existing_type=sa.String(length=30),
               type_=mysql.INTEGER(),
               nullable=True)

    # ### end Alembic commands ###


"""Alpaca real time news streaming."""
# %% codecell
import json
from pathlib import Path
import asyncio
import websockets
from io import BytesIO
import pandas as pd

try:
    from scripts.dev.multiuse.help_class import baseDir, getDate, write_to_parquet
    from scripts.dev.data_collect.alpaca.api_calls.apca_auth import ApcaAuth
except ModuleNotFoundError:
    from multiuse.help_class import baseDir, getDate, write_to_parquet
    from data_collect.alpaca.api_calls.apca_auth import ApcaAuth

# %% codecell

class ApcaNewsStream(ApcaAuth):
    """Apca real-time news stream with websockets."""

    # Url for websocket streaming
    uri = 'wss://stream.data.alpaca.markets/v1beta1/news'
    # All stock and crypto symbols
    subscribe_all = {"action": "subscribe", "news": ["*"]}
    # Unsubscribe
    unsubscribe_all = {"action": "unsubscribe", "news": ["*"]}

    def __init__(self, **kwargs):
        ApcaAuth.__init__(self, **kwargs)
        self.auth = self._create_auth_msg(self, **kwargs)
        self.fpath = self._create_fpath(self, **kwargs)
        self._start_news_streaming(self)

    @classmethod
    def _create_auth_msg(cls, self, **kwargs):
        """Create auth message."""
        auth = ({"action": "auth",
                 "key": self.headers['APCA-API-KEY-ID'],
                 "secret": self.headers['APCA-API-SECRET-KEY']})

        auth = json.dumps(auth)
        return auth

    @classmethod
    def _create_fpath(cls, self, **kwargs):
        """Construct and return fpath."""
        bpath = Path(baseDir().path, 'alpaca', 'news', 'real_time')
        dt = getDate.query('iex_eod')
        f_path = bpath.joinpath(f"_{str(dt.year)}.parquet")
        return f_path

    @classmethod
    def _start_news_streaming(cls, self):
        """instantiate news streaming function."""
        try:
            loop = asyncio.get_running_loop()
            if loop and loop.is_running():
                print('Loop is running')
                tsk = loop.create_task(self._news_streaming(self))
        except RuntimeError:
            asyncio.run(self._news_streaming(self))

    @classmethod
    async def _news_streaming(cls, self):
        """Asyncio news streaming."""
        async for ws in websockets.connect(self.uri):
            try:
                a = await ws.send(self.auth)
                print(str(await ws.recv()))
                sa = await ws.send(json.dumps(self.subscribe_all))
                print(str(await ws.recv()))

                msg = await self._process_message(self, ws)
            except websockets.ConnectionClosed:
                continue

    @classmethod
    async def _process_message(cls, self, ws):
        """Process message and write to dataframe."""
        while True:
            data = await ws.recv()
            try:
                data = json.loads(data)[0]
                if 'headline' in data.keys():
                    df = pd.json_normalize(data)
                    kwargs = {'cols_to_drop': 'id'}
                    write_to_parquet(df, self.fpath, combine=True, **kwargs)
            except AttributeError as ae:
                print(str(ae))
                print(str(data))

                try:
                    df = pd.DataFrame(json.load(BytesIO(data)))
                    kwargs = {'cols_to_drop': 'id'}
                    write_to_parquet(df, self.fpath, combine=True, **kwargs)
                except Exception as e:
                    print(f"{str(e)} {type(e)}")
                continue


from django.db import models
from django.db.models.functions import Lower
from django.contrib.auth.models import AbstractUser

class BuickVehicleManager(models.Manager):
    def get_queryset(self):
        return super(BuickVehicleManager, self).get_queryset().filter(make=1)

class ChevyVehicleManager(models.Manager):
    def get_queryset(self):
        return super(ChevyVehicleManager, self).get_queryset().filter(make=3)

YESNO_CHOICES = (
    (True, 'Yes'),
    (False, 'No'),
)

MAKE_CHOICES = (
    (1, 'Buick'),
    (2, 'Cadillac'),
    (3, 'Chevrolet'),
)

class Vehicle(models.Model):
    vehicle_model = models.ForeignKey(
        'VehicleModel',
        on_delete=models.CASCADE,
        verbose_name='Model',
        related_name='model_vehicle',
        blank=True,
        null=True
    )
    engine = models.ForeignKey(
        'Engine',
        on_delete=models.CASCADE,
        verbose_name='engine',
        related_name='engine_vehicle',
        blank=True,
        null=True
    )
    make = models.PositiveIntegerField(
        choices=MAKE_CHOICES,
        verbose_name='Vehicle Make/Brand',
        blank=True,
        null=True
    )
    vin = models.CharField(
        verbose_name='VIN',
        max_length=75,
        unique=True,
        blank=True,
        null=True,
    )
    sold = models.BooleanField(
        verbose_name='Sold?',
        choices=YESNO_CHOICES,
        default=False,
        blank=True,
        null=True,
    )
    price = models.DecimalField(max_digits=19, decimal_places=2, null=True)

    # adding the model managers
    # the default model manager
    objects = models.Manager()
    # The buick specific manager
    buick_objects = BuickVehicleManager()
    # the chevy specific manager
    chevy_objects = ChevyVehicleManager()

    def __str__(self):
        MAKE_CHOICES_DICT = dict(MAKE_CHOICES)
        return MAKE_CHOICES_DICT[self.make] + ' ' + self.vehicle_model.name

    # custom model method, add @property so that we access the function as a field of that model, rather than as a
    # function e.g my_object.full_vehicle_name rather than my_object.full_vehicle_name()
    @property
    def full_vehicle_name(self):
        return self.__str__() + ' - ' + self.engine.name

class VehicleModel(models.Model):
    name = models.CharField(
        verbose_name='Model',
        max_length=75,
        unique=True,
        blank=True,
        null=True,
    )

    class Meta:
        verbose_name = 'Vehicle Model'
        verbose_name_plural = 'Vehicle Models'

        # ordering = ['name', 'someotherfield'] ascending order
        ordering = ['-name']  # descending order

        indexes = [
            models.Index(fields=['name']),
            models.Index(
                fields=['-name'],
                name='desc_name_idx'
            ),
            # models.Index(
            #     Lower('name').desc(),
            #     name='lower_name_idx'
            # )
            models.Index(
                fields=['-name'.lower()],
                name='lower_name_idx'
            )

        ]

class Engine(models.Model):
    vehicle_model = models.ForeignKey(
        VehicleModel,
        on_delete=models.CASCADE,
        verbose_name='Model',
        related_name='model_engine',
        blank=True,
        null=True
    )
    name = models.CharField(
        verbose_name='Engine',
        max_length=75,
        blank=True,
        null=True,
    )

class Seller(AbstractUser):
    name = models.CharField(
        verbose_name='Seller Name',
        max_length=150,
        blank=True,
        null=True,
    )
    vehicle = models.ManyToManyField(
        Vehicle,
        verbose_name='Vehicles',
        related_name='vehicle_sellers',
        related_query_name='vehicle_seller',
        blank=True,
    )

# for side practice we use the engine2 in lowercase model
class engine2(models.Model):
    name = models.CharField(
        verbose_name='Engine',
        max_length=75,
        blank=True,
        null=True
    )
    vehicle_model = models.ForeignKey(
        VehicleModel,
        on_delete=models.CASCADE,
        verbose_name='Model',
        related_name='model_engine2',
        blank=True,
        null=True,
    )

    # using db_table to explicitly specify the db_table of engine2
    # by default it would be {{app_name}}_{{model_name}}
    class Meta:
        abstract = True  # allows us to use the class as a parent class
        db_table = 'chapter3_practice_engine'
        ordering = ['name']
        verbose_name = 'Practice Engine'
        verbose_name_plural = 'Practice Engines'

class engine3(engine2):
    other_name = models.CharField(
        verbose_name='Other Engine Name',
        max_length=75,
        blank=True,
        null=True,
    )


# coding: utf-8

import os
import requests
import time
import logging

from logging.handlers import QueueHandler
from multiprocessing import Process
from multiprocessing.shared_memory import SharedMemory
from queue import Empty

from legendary.models.chunk import Chunk
from legendary.models.downloading import DownloaderTaskResult, WriterTaskResult

class DLWorker(Process):
    def __init__(self, name, queue, out_queue, shm, max_retries=5,
                 logging_queue=None, dl_timeout=10):
        super().__init__(name=name)
        self.q = queue
        self.o_q = out_queue
        self.session = requests.session()
        self.session.headers.update({
            'User-Agent': 'EpicGamesLauncher/11.0.1-14907503+++Portal+Release-Live Windows/10.0.19041.1.256.64bit'
        })
        self.max_retries = max_retries
        self.shm = SharedMemory(name=shm)
        self.log_level = logging.getLogger().level
        self.logging_queue = logging_queue
        self.dl_timeout = float(dl_timeout) if dl_timeout else 10.0

    def run(self):
        # we have to fix up the logger before we can start
        _root = logging.getLogger()
        _root.handlers = []
        _root.addHandler(QueueHandler(self.logging_queue))

        logger = logging.getLogger(self.name)
        logger.setLevel(self.log_level)
        logger.debug(f'Download worker reporting for duty!')

        empty = False
        while True:
            try:
                job = self.q.get(timeout=10.0)
                empty = False
            except Empty:
                if not empty:
                    logger.debug(f'Queue Empty, waiting for more...')
                empty = True
                continue

            if job.kill:  # let worker die
                logger.debug(f'Worker received kill signal, shutting down...')
                break

            tries = 0
            dl_start = dl_end = 0
            compressed = 0
            chunk = None

            try:
                while tries < self.max_retries:
                    # print('Downloading', job.url)
                    logger.debug(f'Downloading {job.url}')
                    dl_start = time.time()

                    try:
                        r = self.session.get(job.url, timeout=self.dl_timeout)
                        r.raise_for_status()
                    except Exception as e:
                        logger.warning(f'Chunk download for {job.guid} failed: ({e!r}), retrying...')
                        continue

                    dl_end = time.time()
                    if r.status_code != 200:
                        logger.warning(f'Chunk download for {job.guid} failed: status {r.status_code}, retrying...')
                        continue
                    else:
                        compressed = len(r.content)
                        chunk = Chunk.read_buffer(r.content)
                        break
                else:
                    raise TimeoutError('Max retries reached')
            except Exception as e:
                logger.error(f'Job for {job.guid} failed with: {e!r}, fetching next one...')
                # add failed job to result queue to be requeued
                self.o_q.put(DownloaderTaskResult(success=False, chunk_guid=job.guid, shm=job.shm, url=job.url))
            except KeyboardInterrupt:
                logger.warning('Immediate exit requested, quitting...')
                break

            if not chunk:
                logger.warning(f'Chunk somehow None?')
                self.o_q.put(DownloaderTaskResult(success=False, chunk_guid=job.guid, shm=job.shm, url=job.url))
                continue

            # decompress stuff
            try:
                size = len(chunk.data)
                if size > job.shm.size:
                    logger.fatal(f'Downloaded chunk is longer than SharedMemorySegment!')

                self.shm.buf[job.shm.offset:job.shm.offset + size] = bytes(chunk.data)
                del chunk
                self.o_q.put(DownloaderTaskResult(success=True, chunk_guid=job.guid, shm=job.shm,
                                                  url=job.url, size=size, compressed_size=compressed,
                                                  time_delta=dl_end - dl_start))
            except Exception as e:
                logger.warning(f'Job for {job.guid} failed with: {e!r}, fetching next one...')
                self.o_q.put(DownloaderTaskResult(success=False, chunk_guid=job.guid, shm=job.shm, url=job.url))
                continue
            except KeyboardInterrupt:
                logger.warning('Immediate exit requested, quitting...')
                break

        self.shm.close()

class FileWorker(Process):
    def __init__(self, queue, out_queue, base_path, shm, cache_path=None, logging_queue=None):
        super().__init__(name='FileWorker')
        self.q = queue
        self.o_q = out_queue
        self.base_path = base_path
        self.cache_path = cache_path if cache_path else os.path.join(base_path, '.cache')
        self.shm = SharedMemory(name=shm)
        self.log_level = logging.getLogger().level
        self.logging_queue = logging_queue

    def run(self):
        # we have to fix up the logger before we can start
        _root = logging.getLogger()
        _root.handlers = []
        _root.addHandler(QueueHandler(self.logging_queue))

        logger = logging.getLogger(self.name)
        logger.setLevel(self.log_level)
        logger.debug(f'Download worker reporting for duty!')

        last_filename = ''
        current_file = None

        while True:
            try:
                try:
                    j = self.q.get(timeout=10.0)
                except Empty:
                    logger.warning('Writer queue empty!')
                    continue

                if j.kill:
                    if current_file:
                        current_file.close()
                    self.o_q.put(WriterTaskResult(success=True, kill=True))
                    break

                # make directories if required
                path = os.path.split(j.filename)[0]
                if not os.path.exists(os.path.join(self.base_path, path)):
                    os.makedirs(os.path.join(self.base_path, path))

                full_path = os.path.join(self.base_path, j.filename)

                if j.empty:  # just create an empty file
                    open(full_path, 'a').close()
                    self.o_q.put(WriterTaskResult(success=True, filename=j.filename))
                    continue
                elif j.open:
                    if current_file:
                        logger.warning(f'Opening new file {j.filename} without closing previous! {last_filename}')
                        current_file.close()

                    current_file = open(full_path, 'wb')
                    last_filename = j.filename

                    self.o_q.put(WriterTaskResult(success=True, filename=j.filename))
                    continue
                elif j.close:
                    if current_file:
                        current_file.close()
                        current_file = None
                    else:
                        logger.warning(f'Asking to close file that is not open: {j.filename}')

                    self.o_q.put(WriterTaskResult(success=True, filename=j.filename, closed=True))
                    continue
                elif j.rename:
                    if current_file:
                        logger.warning('Trying to rename file without closing first!')
                        current_file.close()
                        current_file = None
                    if j.delete:
                        try:
                            os.remove(full_path)
                        except OSError as e:
                            logger.error(f'Removing file failed: {e!r}')
                            self.o_q.put(WriterTaskResult(success=False, filename=j.filename))
                            continue

                    try:
                        os.rename(os.path.join(self.base_path, j.old_filename), full_path)
                    except OSError as e:
                        logger.error(f'Renaming file failed: {e!r}')
                        self.o_q.put(WriterTaskResult(success=False, filename=j.filename))
                        continue

                    self.o_q.put(WriterTaskResult(success=True, filename=j.filename))
                    continue
                elif j.delete:
                    if current_file:
                        logger.warning('Trying to delete file without closing first!')
                        current_file.close()
                        current_file = None

                    try:
                        os.remove(full_path)
                    except OSError as e:
                        if not j.silent:
                            logger.error(f'Removing file failed: {e!r}')

                    self.o_q.put(WriterTaskResult(success=True, filename=j.filename))
                    continue

                pre_write = post_write = 0

                try:
                    if j.shm:
                        pre_write = time.time()
                        shm_offset = j.shm.offset + j.chunk_offset
                        shm_end = shm_offset + j.chunk_size
                        current_file.write(self.shm.buf[shm_offset:shm_end].tobytes())
                        post_write = time.time()
                    elif j.cache_file:
                        pre_write = time.time()
                        with open(os.path.join(self.cache_path, j.cache_file), 'rb') as f:
                            if j.chunk_offset:
                                f.seek(j.chunk_offset)
                            current_file.write(f.read(j.chunk_size))
                        post_write = time.time()
                    elif j.old_file:
                        pre_write = time.time()
                        with open(os.path.join(self.base_path, j.old_file), 'rb') as f:
                            if j.chunk_offset:
                                f.seek(j.chunk_offset)
                            current_file.write(f.read(j.chunk_size))
                        post_write = time.time()
                except Exception as e:
                    logger.warning(f'Something in writing a file failed: {e!r}')
                    self.o_q.put(WriterTaskResult(success=False, filename=j.filename,
                                                  chunk_guid=j.chunk_guid,
                                                  release_memory=j.release_memory,
                                                  shm=j.shm, size=j.chunk_size,
                                                  time_delta=post_write-pre_write))
                else:
                    self.o_q.put(WriterTaskResult(success=True, filename=j.filename,
                                                  chunk_guid=j.chunk_guid,
                                                  release_memory=j.release_memory,
                                                  shm=j.shm, size=j.chunk_size,
                                                  time_delta=post_write-pre_write))
            except Exception as e:
                logger.warning(f'Job {j.filename} failed with: {e!r}, fetching next one...')
                self.o_q.put(WriterTaskResult(success=False, filename=j.filename, chunk_guid=j.chunk_guid))

                try:
                    if current_file:
                        current_file.close()
                        current_file = None
                except Exception as e:
                    logger.error(f'Closing file after error failed: {e!r}')
            except KeyboardInterrupt:
                logger.warning('Immediate exit requested, quitting...')
                if current_file:
                    current_file.close()
                return


# Import the libraries

import time
import sys
import numpy as np   # pip install numpy

import matplotlib.pyplot as plt

#%matplotlib inline #install pandas # pip install matplotlib
# if you dont use jupyter , then delete this line / or install jjupyter:   pip install jupyter
# you have to hide %matplotlib inline :: im not sure why

# Plotting functions

def Plotvec1(u, z, v):

    ax = plt.axes()
    ax.arrow(0, 0, *u, head_width=0.05, color='r', head_length=0.1)
    plt.text(*(u + 0.1), 'u')

    ax.arrow(0, 0, *v, head_width=0.05, color='b', head_length=0.1)
    plt.text(*(v + 0.1), 'v')
    ax.arrow(0, 0, *z, head_width=0.05, head_length=0.1)
    plt.text(*(z + 0.1), 'z')
    plt.ylim(-2, 2)
    plt.xlim(-2, 2)

    plt.show()

def Plotvec2(a,b):
    ax = plt.axes()
    ax.arrow(0, 0, *a, head_width=0.05, color ='r', head_length=0.1)
    plt.text(*(a + 0.1), 'a')
    ax.arrow(0, 0, *b, head_width=0.05, color ='b', head_length=0.1)
    plt.text(*(b + 0.1), 'b')
    plt.ylim(-2, 2)
    plt.xlim(-2, 2)


from collections import deque

class Solution:
    def alertNames(self, keyName: List[str], keyTime: List[str]) -> List[str]:
        accessTimes = list(zip(keyName, keyTime))
        accessTimes.sort()

        alertedPersons = set()

        keyCard = {}

        for name, time in accessTimes:
            time = self.changeMins(time)
            if not name in keyCard:
                keyCard[name] = deque()

            keyCard[name].append(time)

            while len( keyCard[name]) > 1 and time - keyCard[name][0] > 60:
                keyCard[name].popleft()

            # print(keyCard)
            if len(keyCard[name]) >= 3:
                alertedPersons.add(name)

        return sorted(list(alertedPersons))

    def changeMins(self, time):
        hour, mins = time.split(":")
        return int(hour) * 60 + int(mins)


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 24 02:56:00 2022

@author: reco1
"""
#1.kütüphaneler
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#2.veri Onisleme
#veri yukleme
veriler = pd.read_csv("maaslar.csv")
print(veriler)

#dataframe dilimleme (slice)
#egitim seviyesi x
#maas y olarak bölüyoruz
x = veriler.iloc[:,1:2]
y = veriler.iloc[:,2:]

#Numpy array dönüsümü
X = x.values
Y = y.values

#linear regression
#dogrusal model olusturma
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(X,Y)

#polynomial regression
#dogrusal olmayan (nonlinear) model olusturma
#2. dereceden polinom
#herhangi bir sayıyı polinomal olarak ifade etmeye yarıyor esasında
from sklearn.preprocessing import PolynomialFeatures
poly_reg = PolynomialFeatures(degree=2) #2.dereceden obje olustur
x_poly = poly_reg.fit_transform(X) #X= 1 den 10'a kadar olan degerler
# print(x_poly) #1 den 10'a kadar giden degerler ve onların kareleri geldi, waoww
lin_reg2 = LinearRegression()
lin_reg2.fit(x_poly,y)

#4. dereceden polinom
poly_reg3 = PolynomialFeatures(degree=4)
x_poly3 = poly_reg3.fit_transform(X)
#print(x_poly)
lin_reg3 = LinearRegression()
lin_reg3.fit(x_poly3,y)

# Gorsellestirme
# plt.scatter(X,Y,color="red")
# plt.plot(X, lin_reg2.predict(poly_reg.fit_transform(X)), color="blue")
# plt.show()

# plt.scatter(X,Y, color="red")
# plt.plot(x, lin_reg.predict(X), color="blue") # herbir x'e karsilik gelen tahminleri gorsellestirme
# plt.show()

plt.scatter(X,Y,color="red")
plt.plot(X, lin_reg3.predict(poly_reg3.fit_transform(X)), color="blue")
plt.show()

#tahminler
#linear regression tahminleri
print(lin_reg.predict([[11]])) #bir kisinin egitimseviyesi 11 ise verileceği maas
print(lin_reg.predict([[6.6]])) #bir kisinin egitimseviyesi 6 ile 7 arasında ise verileceği maas

#polinomal regression tahminleri
print(lin_reg2.predict(poly_reg.fit_transform([[6.6]])))
print(lin_reg2.predict(poly_reg.fit_transform([[11]])))


from django.db import models
from django.core.validators import MinValueValidator

from accounts.models import UserAccount
from products.models import Product

class Order(models.Model):
    user = models.ForeignKey(
        UserAccount, on_delete=models.SET_NULL, null=True)
    order_status = models.CharField(max_length=20, default="PENDING", choices=[("PENDING", "Pending"), ("PROCESSING", "Processing"), (
        "INTRANSIT", "In Transit"), ("WAITING", "Waiting Collection"), ("COMPLETED", "Completed"), ("CANCELLED", "Cancelled")])
    order_created_time = models.DateTimeField(auto_now_add=True)
    order_updated_time = models.DateTimeField(auto_now=True)

    def __str__(self):
        return self.user

class OrderItem(models.Model):
    order = models.ForeignKey(Order, on_delete=models.CASCADE, null=True)
    product = models.ForeignKey(Product, on_delete=models.SET_NULL, null=True)
    quantity = models.IntegerField(validators=[MinValueValidator(1)])
    price = models.DecimalField(
        max_digits=6, decimal_places=2, validators=[MinValueValidator(0)])

    def __str__(self):
        return self.id


from collections import namedtuple

from ...baseclient import BaseClient
from ...exceptions import FailedToExecuteException

TestTorrent = namedtuple(
    "TestTorrent",
    [
        "torrent_data",
        "torrent_files",
        "download_path",
        "torrent_file_data",
        "is_active",
    ],
)

class TestClient(BaseClient):
    identifier = "testclient"
    display_name = "TestClient"

    def __init__(self):
        self._action_queue = []
        self._test_connection = True
        self._torrents = {}

    def list(self):
        return [t.torrent_data for t in self._torrents.values()]

    def list_active(self):
        return [t.torrent_data for t in self._torrents.values() if t.is_active]

    def start(self, infohash):
        if infohash not in self._torrents:
            raise FailedToExecuteException("Torrent does not exist")
        self._action_queue.append(("start", {"infohash": infohash}))

    def stop(self, infohash):
        if infohash not in self._torrents:
            raise FailedToExecuteException("Torrent does not exist")
        self._action_queue.append(("stop", {"infohash": infohash}))

    def test_connection(self):
        return self._test_connection

    def add(
        self,
        torrent,
        destination_path,
        fast_resume=False,
        add_name_to_folder=True,
        minimum_expected_data="none",
        stopped=False,
    ):
        self._action_queue.append(
            (
                "add",
                {
                    "torrent": torrent,
                    "destination_path": destination_path,
                    "fast_resume": fast_resume,
                    "add_name_to_folder": add_name_to_folder,
                    "minimum_expected_data": minimum_expected_data,
                    "stopped": stopped,
                },
            )
        )

    def remove(self, infohash):
        if infohash not in self._torrents:
            raise FailedToExecuteException("Torrent does not exist")
        self._action_queue.append(("remove", {"infohash": infohash}))

    def retrieve_torrentfile(self, infohash):
        if infohash not in self._torrents:
            raise FailedToExecuteException("Torrent does not exist")
        return self._torrents[infohash].torrent_file_data

    def get_download_path(self, infohash):
        if infohash not in self._torrents:
            raise FailedToExecuteException("Torrent does not exist")
        return self._torrents[infohash].download_path

    def move_torrent(self, infohash, destination_path):
        self._action_queue.append(
            (
                "move_torrent",
                {
                    "infohash": infohash,
                    "destination_path": destination_path,
                },
            )
        )

    def get_files(self, infohash):
        if infohash not in self._torrents:
            raise FailedToExecuteException("Torrent does not exist")
        return self._torrents[infohash].torrent_files

    def serialize_configuration(self):
        return f"{self.identifier}://"

    def auto_configure(cls):
        raise FailedToExecuteException("Cannot autoconfigure")

    def _inject_torrent(
        self,
        torrent_data,
        torrent_files,
        download_path,
        torrent_file_data=None,
        is_active=True,
    ):
        self._torrents[torrent_data.infohash] = TestTorrent(
            torrent_data, torrent_files, download_path, torrent_file_data, is_active
        )


import itertools

def is_palindrome(pair):
    """ true if the number is a palindrome """

    num = pair[0] * pair[1]
    stringed = str(num)
    half = len(stringed) / 2
    front, back = stringed[:half], stringed[half:]

    return front == back[::-1]

def solve(n):
    candidates = list(itertools.product(range(n, 0, -1), range(n, 0, -1)))
    palindromes = filter(is_palindrome, candidates)

    largest_pair = sorted(palindromes, key=lambda p: p[0] * p[1], reverse=True)[0]

    return largest_pair[0] * largest_pair[1]

if __name__ == "__main__":
    import sys

    maximum = 999

    if len(sys.argv) > 1:
        maximum = int(sys.argv[1])

    # prevent subtle breakage
    # assert is_palindrome(9999) is True
    # assert is_palindrome(1234) is False
    # assert is_palindrome(12345) is False

    print(solve(maximum))


import torch
import os
import matplotlib.pyplot as plt
import numpy as np
from torchvision.utils import make_grid
import pandas as pd
from Project import Project

device = 'cuda' if torch.cuda.is_available() else 'cpu'
plt.interactive(False)

def show(img):
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)), interpolation='nearest')
    plt.show()

def show_dataset(dataset, n=6):
    imgs = [dataset[i][0] for i in range(n)]
    # print(imgs[0].shape)
    grid = make_grid(imgs)
    plt.imshow(grid.numpy().transpose((1, 2, 0)))
    plt.tight_layout()
    plt.show()

def show_dl(dl, n=6):
    batch = None
    for batch in dl:
        break
    imgs = batch[0][:n]
    grid = make_grid(imgs)
    plt.imshow(grid.numpy().transpose((1, 2, 0)))
    plt.tight_layout()
    plt.show()

def show_one_batch(dl, rows, cols):
    dataiter = iter(dl)
    images, labels, xl , yl = dataiter.next()
    images,labels = images.to(device), labels.to(device)
    imgs=[]
    for i in range(rows):
        for j in range(cols):
            imgs.append(images[i,5*j,:,:])

    print(len(imgs))
    grid = make_grid(imgs,nrow=rows)
    plt.imshow(grid.cpu().numpy().transpose((1, 2, 0)))
    plt.tight_layout()
    plt.savefig('trainloader_grid.png')
    plt.show()

def max_seq_length():
    seq_length =[]
    project = Project()
    data_path = project.data_dir
    fnames = os.listdir(data_path)

    all_names = []
    for f in fnames:
        all_names.append(f)

    for i in range(len(all_names)):
        folder_dir = os.path.join(data_path, all_names[i])
        action_filename = os.path.join(folder_dir,'action.csv')
        df = pd.read_csv(action_filename, header=None)
        seq_length.append(len(df))
        max_sequence_length = np.max(seq_length)
        min_sequence_length = np.min(seq_length)

    return seq_length, max_sequence_length, min_sequence_length

def max_seq_length_list(train_list):
    seq_length =[]
    project = Project()
    data_path = project.data_dir
    for i in range(len(train_list)):
        folder_dir = os.path.join(data_path, train_list[i])
        action_filename = os.path.join(folder_dir,'action.csv')
        df = pd.read_csv(action_filename, header=None)
        seq_length.append(len(df))
        max_sequence_length = np.max(seq_length)
        min_sequence_length = np.min(seq_length)
    return seq_length, max_sequence_length, min_sequence_length

def feat_eng(data):
    data['totl_xyz'] = (data['pos_x']**2 + data['pos_y']**2 + data['pos_z']**2)**0.5
    data['totl_ang_xyz'] = (data['roll']**2 + data['pitch']**2 + data['yaw']**2)**0.5

    def mean_change_of_abs_change(x):
        return np.mean(np.diff(np.abs(np.diff(x))))

    for col in data.columns:
        data[col + '_mean'] = data[col].mean()
        data[col + '_median'] = data[col].median()
        data[col + '_max'] = data[col].max()
        data[col + '_min'] = data[col].min()
        data[col + '_std'] = data[col].std()
        data[col + '_range'] = data[col + '_max'] - data[col + '_min']
        data[col + '_maxtoMin'] = data[col + '_max'] / data[col + '_min']
        data[col + '_mean_abs_change'] = data[col].diff()
#         data[col + '_mean_change_of_abs_change'] = data[col].apply(mean_change_of_abs_change)
    return data

def feature_engineering_poses(data_frame):
    data_frame[data_frame.columns[-3:]]= data_frame[data_frame.columns[-3:]]*np.pi/180
    data_frame['roll'] =  data_frame['roll'].abs()

    #Basic features
    data_frame = feat_eng(data_frame)
    data_frame = data_frame.fillna(0)
    data_frame = data_frame.values

    return data_frame

def save_checkpoint(state, is_best, filename='/output/checkpoint.pth.tar'):
    """Save checkpoint if a new best is achieved"""
    if is_best:
        print ("=> Saving a new best")
        torch.save(state, filename)  # save checkpoint
    else:
        print ("=> Validation Accuracy did not improve")


import configparser
import RedditApi as api
import json_parser as parser
import tts_service as tts
import video_service as video
import sys
import os
'''
********************************
*    Console Line arguments    *
-genjson [number of posts]     *
-novid                         *
********************************
'''
client_id = "client_id"
client_secret = "client_secret"
user_agent = "user_agent"

def init_config():
    config = configparser.ConfigParser()
    config.read('config.ini')
    client_id = config['reddit']['client_id']
    client_secret = config['reddit']['client_secret']
    user_agent = config['reddit']['user_agent']
    return client_id, client_secret, user_agent

def generate_json_file(count,subreddit = 'StoryTime'):
    client_id,client_secret,user_agent = init_config()
    reddit = api.get_reddit_instance(user_agent, client_id, client_secret)
    hot_posts = api.get_hot_posts(reddit, subreddit, count)
    parser.parse_to_json(hot_posts)

def generate_tts(id):
    mapping = tts.generate_speech('./json_data/data.json', id)
    return mapping

def generate_video(id,text,audio_mapping):
    video.generate_video(video='./video_files/Default.mp4', audio=f'./audio_files/', text=text, audio_mapping=audio_mapping)

def delete_audio_files():
    folder = './audio_files/'
    for filename in os.listdir(folder):
        file_path = os.path.join(folder, filename)
        if os.path.isfile(file_path):
            os.remove(file_path)

if __name__ == "__main__":
    id = 1
    if len(sys.argv) > 1:
        if sys.argv[1] == '-genjson':
            generate_json_file(int(sys.argv[2]))
        elif sys.argv[1] == '-novid':
            mapping = generate_tts(id)
    else:
        mapping = generate_tts(id)
        generate_video(id = id, text=parser.get_90_content_from_json(id), audio_mapping=mapping)


from tqdm import tqdm
import cv2
import torch
import numpy as np
from src.data.twitter_image_dataset import TwitterDataset
from src.models.resnet import ResNet101

def get_features(image_path, batch):
    # move the input and model to GPU for speed if available
    if torch.cuda.is_available():
        device = 'cuda'
    else:
        device = 'cpu'

    # initialize our implementation of ResNet
    model = ResNet101(pretrained=True)
    model.eval()
    model.to(device)

    # read the dataset and initialize the data loader
    ds = TwitterDataset(image_path)
    dataloader = torch.utils.data.DataLoader(ds, batch_size=batch, shuffle=True)

    features = None
    image_paths = []

    for batch in tqdm(dataloader, desc='Running the model inference'):
        images = batch['image'].to(device)
        image_paths += batch['image_path']

        with torch.no_grad():
            output = model.forward(images)

        current_features = output.cpu().numpy()
        if features is not None:
            features = np.concatenate((features, current_features))
        else:
            features = current_features

    return features, image_paths


import click
import colorlog
from pylab import savefig

from sequana.compare import RNADiffCompare

from .utils import CONTEXT_SETTINGS, common_logger

logger = colorlog.getLogger(__name__)

@click.command(context_settings=CONTEXT_SETTINGS)
@click.option(
    "--file1",
    type=click.Path(),
    default=None,
    required=True,
    help="""The first input RNA-seq table to compare""",
)
@click.option(
    "--file2",
    type=click.Path(),
    default=None,
    required=True,
    help="""The second input RNA-seq table to compare""",
)
@common_logger
def rnaseq_compare(**kwargs):
    """Compare 2 tables created by the 'sequana rnadiff' command"""
    c = RNADiffCompare(kwargs["file1"], kwargs["file2"])
    print(c.r1.summary())
    print(c.r2.summary())
    c.plot_volcano_differences()
    savefig("sequana_rnaseq_compare_volcano.png", dpi=200)