import logging
import math
import re
import threading
import uuid
from re import match
import xlwt
from xlwt import Workbook
from threading import Thread
import xlrd
import constants
import cluster
import definitions as defs
import numpy as np
from sys import maxsize

vm_features = cluster.vm_features
vm_lock = threading.Lock()
history_lock = threading.Lock()
mode = ""

class Worker:
    def __init__(self, worker, episode):
        self.sorted_request_history_per_window = {}
        self.pod_object_dict_by_type = {}
        self.pending_pods = {}
        self.required_pod_count = {}
        self.pod_counter = {}
        self.fn_request_rate = {}
        self.pods = []
        self.sorted_events = []
        self.dropped_requests = []
        self.clock = 0
        self.simulation_running = True
        self.vm_up_time_dict = {}
        self.vm_up_time_cost_prev = 0
        self.episodic_reward = 0
        self.function_latency = 0
        self.fn_failures = 0
        self.ver_cpu_action_total = 0
        self.ver_mem_action_total = 0
        self.hor_action_total = 0
        self.Episodic_failure_rate = 0
        self.Episodic_latency = 0
        self.total_vm_cost_diff = 0
        self.pod_id = 1
        self.episode_no = episode
        self.worker_id = worker
        self.episodic_loss = 0

        self.vms = cluster.init_vms()

        self.serv_env_state_init = cluster.gen_serv_env_state_init()
        self.serv_env_state_min = cluster.gen_serv_env_state_min()
        self.serv_env_state_max = cluster.gen_serv_env_state_max()

        logging.info(
            "ServEnv_Base to be initialized for worker {} episode: {}".format(worker, episode))

        self.fn_features = cluster.init_fn_features()

        self.wb = Workbook()
        self.req_wl = self.wb.add_sheet('Results')
        self.req_wl.write(0, 0, 'Fn ID')
        self.req_wl.write(0, 1, 'Fn type')
        self.req_wl.write(0, 2, 'Allocated pod')
        self.req_wl.write(0, 3, 'Arrival time')
        self.req_wl.write(0, 4, 'Start/Dropped time')
        self.req_wl.write(0, 5, 'Finish time')
        self.req_wl.write(0, 6, 'Result')

        self.pod_data_sheet = self.wb.add_sheet('Pods')
        self.pod_data_sheet.write(0, 0, 'Pod ID')
        self.pod_data_sheet.write(0, 1, 'Pod type')
        self.pod_data_sheet.write(0, 2, 'Start time')
        self.pod_data_sheet.write(0, 3, 'Finish time')

        self.results_sheet_row_counter = 1
        self.pod_sheet_row_counter = 1

        self.fn_types = []
        self.fn_iterator = -1

        self.init_workload()
        # self.init_workload_test()

    def sorter_events(self, item):
        time = float(item.received_time)
        return time

    def pod_sorter(self, item):
        util = float(item.cpu_util)
        return util

    def pod_creator(self, fn_type):
        global vm_features
        run_req_list = []
        term_req_list = []
        running_pod_list = {}
        term_pod_list = []
        vm = defs.VM(0, len(self.vms), int(vm_features['vm0_cpu_total_speed']), int(vm_features['vm0_mem']),
                     int(vm_features['vm0_cpu_used']), int(vm_features['vm0_mem_used']),
                     int(vm_features['vm0_cpu_allocated']),
                     int(vm_features['vm0_mem_allocated']), int(vm_features['vm0_price']),
                     int(vm_features['vm0_nw_bandwidth']),
                     int(vm_features['vm0_diskio_bandwidth']), running_pod_list, term_pod_list)
        self.pods.append(
            defs.POD(self.pod_id, vm, self.fn_features[str(fn_type) + "_name"], 0, 0,
                     int(self.fn_features[str(fn_type) + "_pod_cpu_req"]),
                     int(self.fn_features[str(fn_type) + "_req_MIPS"]),
                     int(self.fn_features[str(fn_type) + "_pod_ram_req"]),
                     int(self.fn_features[str(fn_type) + "_pod_cpu_util"]),
                     int(self.fn_features[str(fn_type) + "_pod_ram_util"]),
                     int(self.fn_features[str(fn_type) + "_req_per_sec"]),
                     int(self.fn_features[str(fn_type) + "_num_current_replicas"]),
                     int(self.fn_features[str(fn_type) + "_num_max_replicas"]),
                     int(self.fn_features[str(fn_type) + "_inflight_requests"]), run_req_list, term_req_list))

        if self.pod_id == 1:
            print("POD 1")

        # logging.info(
        #     "CLOCK: {}  Pod id: {} Type: {} worker: {} created. Req sent to schedule ".format(self.clock, self.pod_id,
        #                                                                                fn_type, self.worker_id))

        self.pod_id += 1

        self.sorted_events.append(defs.EVENT(self.clock + constants.pod_scheduling_time, constants.schedule_pod,
                                             self.pods[len(self.pods) - 1]))
        self.sorted_events = sorted(self.sorted_events, key=self.sorter_events)

    def pod_scaler(self):
        pod_info = {}

        for pod in self.pods:
            completed_count = 0
            if pod.type in pod_info:
                if pod in self.pod_object_dict_by_type[pod.type]:
                    # pod.cpu_util is in no of cores (not %)
                    pod_info[pod.type]['pod_cpu_util_total'] += pod.cpu_util
                    pod_info[pod.type]['pod_count'] += 1
                    pod_info[pod.type]['pod_inflight_request_count'] += pod.num_inflight_requests
                    for req in reversed(pod.completed_req_list):
                        if req.finish_time > self.clock - constants.faas_rps_window:
                            if req.status != "Dropped":
                                completed_count += 1
                    pod_info[pod.type]['pod_completed_requests'] += completed_count

            else:
                if pod.type in self.pod_object_dict_by_type:
                    if pod in self.pod_object_dict_by_type[pod.type]:
                        pod_info[pod.type] = {}
                        pod_info[pod.type]['pod_cpu_util_total'] = pod.cpu_util
                        pod_info[pod.type]['pod_count'] = 1
                        pod_info[pod.type]['pod_inflight_request_count'] = pod.num_inflight_requests
                        for req in reversed(pod.completed_req_list):
                            if req.finish_time > self.clock - constants.faas_rps_window:
                                if req.status != "Dropped":
                                    completed_count += 1
                        pod_info[pod.type]['pod_completed_requests'] = completed_count
        # logging.info(
        #     "CLOCK: {} worker: {} Now in pod scaler. now running pod count: {} ".format(self.clock, self.worker_id,
        #                                                                                pod_info['pod_count']))
        for pod_type, pod_data in pod_info.items():
            # ***********COMPARISON ALGOS - Also change action to no changes
            if mode == 'knative':
                desired_replicas = math.ceil(pod_data['pod_count'] * ((pod_data['pod_inflight_request_count'] /
                                                                       pod_data[
                                                                           'pod_count']) / constants.knative_pod_concurrency))
            elif mode == 'kube':
                desired_replicas = math.ceil(pod_data['pod_count'] * (pod_data['pod_cpu_util_total'] / int(
                    self.fn_features[str(pod_type) + "_pod_cpu_req"])) / pod_data[
                                                 'pod_count'] / constants.kube_pod_cpu_threshold)
            elif mode == 'faas':
                if self.fn_features[str(pod_type) + "_req_exec_time"] > 2:
                    desired_replicas = math.ceil(pod_data['pod_count'] * ((pod_data['pod_inflight_request_count'] /
                                                                           pod_data[
                                                                               'pod_count']) / constants.faas_capacity))
                elif self.fn_request_rate[pod_type] > 20:
                    desired_replicas = math.ceil(pod_data['pod_count'] * ((pod_data['pod_completed_requests'] /
                                                                           pod_data[
                                                                               'pod_count']) / constants.faas_rps_threshold))
                    print("pod count: {} total completed requests: {}".format(pod_data['pod_count'],
                                                                              pod_data['pod_completed_requests']))
                else:
                    desired_replicas = math.ceil(pod_data['pod_count'] * (pod_data['pod_cpu_util_total'] / int(
                        self.fn_features[str(pod_type) + "_pod_cpu_req"])) / pod_data[
                                                     'pod_count'] / constants.faas_cpu_threshold)
            # ************************************************
            else:
                logging.info(
                    "CLOCK: {}  now running pod count: {} Pod type: {} threshold: {} cpu util: {} cpu req: {}".format(
                        self.clock, pod_data['pod_count'], pod_type, self.fn_features[
                            str(pod_type) + "_scale_cpu_threshold"], pod_data['pod_cpu_util_total'], int(
                            self.fn_features[str(pod_type) + "_pod_cpu_req"])))
                desired_replicas = math.ceil(pod_data['pod_count'] * (pod_data['pod_cpu_util_total'] / int(
                    self.fn_features[str(pod_type) + "_pod_cpu_req"])) / pod_data['pod_count'] / self.fn_features[
                                                 str(pod_type) + "_scale_cpu_threshold"])
                logging.info("CLOCK: {} Desired replicas: {} now running pod count: {} Pod type: {}".format(self.clock,
                                                                                desired_replicas,
                                                                                pod_data[
                                                                                'pod_count'],
                                                                                pod_type))
            if desired_replicas > 0:
                new_pod_count = min(desired_replicas, int(constants.max_num_replicas))
            else:
                new_pod_count = 1

            self.required_pod_count[pod_type] = new_pod_count
            if pod_type in self.pending_pods:
                pending = self.pending_pods[pod_type]
            else:
                pending = 0
            # logging.info("CLOCK: {} worker: {} No of pending pods of this type: {}".format(self.clock, self.worker_id, pending))
            if new_pod_count > pod_data['pod_count'] + pending:
                new_pods_to_create = new_pod_count - pod_data['pod_count'] - pending
                logging.info(
                    "CLOCK: {} worker: {} fn: {} now running pod count: {} pending pods: {} Scaling up, new pods to create: {}".format(
                        self.clock, self.worker_id, pod_type, pod_data['pod_count'], self.pending_pods[pod_type],
                        new_pods_to_create))
                for x in range(new_pods_to_create):
                    self.sorted_events.append(
                        defs.EVENT(self.clock + constants.pod_creation_time, constants.scale_pod, pod_type))
                    self.pending_pods[pod_type] += 1

            # scaling down of pods if pod has no running requests
            if new_pod_count < pod_data['pod_count'] + pending:
                pods_to_scale_down = pending + pod_data['pod_count'] - new_pod_count
                logging.info(
                    "CLOCK: {} worker: {} Scaling down, num pods to remove: {}".format(self.clock, self.worker_id,
                                                                                pods_to_scale_down))
                pod_list_length = len(self.pod_object_dict_by_type[pod_type])
                removed_pods = 0
                pods_to_remove = []
                for i in range(pod_list_length):
                    pod = self.pod_object_dict_by_type[pod_type][i]
                    if pod.num_inflight_requests == 0:
                        # with vm_lock:
                        # logging.info(
                        #     "CLOCK: {} worker: {} Pod Id: {} to be removed, Allocated VM id: {}, vm running pod list: {}".format(self.clock, self.worker_id, pod.id,
                        #                                                                                pod.allocated_vm.id,
                        #                                                                                pod.allocated_vm.running_pod_list))
                        # print(PODS)
                        self.pods.remove(pod)
                        pods_to_remove.append(pod)
                        pod.allocated_vm.running_pod_list[pod.type].remove(pod)

                        if len(pod.allocated_vm.running_pod_list[pod.type]) == 0:
                            pod.allocated_vm.running_pod_list.pop(pod.type)

                        if len(pod.allocated_vm.running_pod_list) == 0:
                            self.vm_up_time_dict[pod.allocated_vm]['status'] = "OFF"
                            self.vm_up_time_dict[pod.allocated_vm]['total_time'] += self.clock - \
                                                                                self.vm_up_time_dict[
                                                                                pod.allocated_vm][
                                                                                'time_now']
                            logging.info(
                                "clock: {} Worker: {} vm {} up time so far: {} latest segment: {}".format(self.clock,
                                                                                self.worker_id,
                                                                                pod.allocated_vm.id,
                                                                                self.vm_up_time_dict[
                                                                                pod.allocated_vm][
                                                                                'total_time'],
                                                                                self.clock - \
                                                                                self.vm_up_time_dict[
                                                                                pod.allocated_vm][
                                                                                'time_now']))

                            self.vm_up_time_dict[pod.allocated_vm]['time_now'] = self.clock

                        pod.allocated_vm.cpu_allocated -= pod.cpu_req
                        pod.allocated_vm.mem_allocated -= pod.ram_req
                        pod.term_time = self.clock
                        removed_pods += 1

                        self.pod_data_sheet.write(self.pod_sheet_row_counter, 0, pod.id)
                        self.pod_data_sheet.write(self.pod_sheet_row_counter, 1, pod.type)
                        self.pod_data_sheet.write(self.pod_sheet_row_counter, 2, pod.start_time)
                        self.pod_data_sheet.write(self.pod_sheet_row_counter, 3, pod.term_time)
                        self.pod_sheet_row_counter += 1

                        if removed_pods == pods_to_scale_down:
                            break
                self.wb.save(
                    "results/" + str(self.worker_id) + "/" + "Results_Episode_" + str(self.episode_no) + ".xls")

                for p in pods_to_remove:
                    logging.info(
                        "Clock {} worker: {} Pod {} of type {} removed from list".format(self.clock, self.worker_id,
                                                                                p.id,
                                                                                p.type))
                    self.pod_object_dict_by_type[p.type].remove(p)

        self.sorted_events = sorted(self.sorted_events, key=self.sorter_events)

        for pod in self.pods:
            if pod.type in self.pod_object_dict_by_type:
                pod.num_current_replicas = len(self.pod_object_dict_by_type[pod.type])

    def pod_scheduler(self, pod):
        vm_allocated = False
        if self.episode_no == 4:
            print("debug at pod scheduler ")
        # with vm_lock:
        for vm in self.vms:
            # print("CPU: " + str(vm.cpu) + " CPU allocated: " + str(vm.cpu_allocated))
            # print("ram: " + str(vm.ram) + " ram allocated: " + str(vm.mem_allocated))
            if ((vm.cpu - vm.cpu_allocated) >= pod.cpu_req) & ((vm.ram - vm.mem_allocated) >= pod.ram_req):
                pod.allocated_vm = vm
                logging.info(
                    "worker: {} Pod {} type: {} is allocated to VM {}".format(self.worker_id, pod.id, pod.type, vm.id))
                vm.cpu_allocated += pod.cpu_req
                vm.mem_allocated += pod.ram_req
                if vm in self.vm_up_time_dict:
                    if self.vm_up_time_dict[vm]['status'] != "ON":
                        # logging.info(
                        #     "VM {} is already in ON status so not making changes to dict".format(vm.id))
                        # else:
                        self.vm_up_time_dict[vm]['status'] = "ON"
                        self.vm_up_time_dict[vm]['time_now'] = self.clock
                else:
                    self.vm_up_time_dict[vm] = {}
                    self.vm_up_time_dict[vm]['status'] = "ON"
                    self.vm_up_time_dict[vm]['time_now'] = self.clock
                    self.vm_up_time_dict[vm]['total_time'] = 0

                if pod.type in vm.running_pod_list:
                    vm.running_pod_list[pod.type].append(pod)
                else:
                    vm.running_pod_list[pod.type] = []
                    vm.running_pod_list[pod.type].append(pod)
                # for pod in vm.running_pod_list:
                #     # logging.info(
                #     #     "Pod {} is in running list of vm {}".format(pod.id, vm.id))
                pod.start_time = self.clock

                # print(PODS[len(PODS) - 1])
                if pod.type in self.pod_object_dict_by_type:
                    self.pod_object_dict_by_type[pod.type].append(pod)
                else:
                    self.pod_object_dict_by_type[pod.type] = []
                    self.pod_object_dict_by_type[pod.type].append(pod)
                pod.num_current_replicas = len(self.pod_object_dict_by_type[pod.type])

                # if str(pod.type) + "_pod_counter" in pod_counters:
                #     pod_counters[str(pod.type) + "_pod_counter"] += 1
                # else:
                #     pod_counters[str(pod.type) + "_pod_counter"] = 0

                if pod.type not in self.pod_counter:
                    self.pod_counter[pod.type] = 0

                if self.pending_pods[pod.type] > 0:
                    self.pending_pods[pod.type] -= 1
                vm_allocated = True
                break

    def req_scheduler(self, request, rescheduling):
        pod_allocated = False

        rescheduling_try_count = request.reschedule_tries
        # pod_counter_name = request.type + "_pod_counter"

        if rescheduling and rescheduling_try_count >= constants.max_reschedule_tries:
            # logging.info(
            #     "CLOCK: {} worker: {} Request has exceeded the rescheduling count, thus dropping".format(
            #         self.clock, self.worker_id))
            self.dropped_requests.append(request)
            request.start_time = self.clock
            request.finish_time = self.clock
            request.status = "Dropped"
            self.req_wl.write(self.results_sheet_row_counter, 0, request.id)
            self.req_wl.write(self.results_sheet_row_counter, 1, request.type)
            self.req_wl.write(self.results_sheet_row_counter, 2, "None")
            self.req_wl.write(self.results_sheet_row_counter, 3, request.arrival_time)
            self.req_wl.write(self.results_sheet_row_counter, 4, request.start_time)
            self.req_wl.write(self.results_sheet_row_counter, 5, self.clock)
            self.req_wl.write(self.results_sheet_row_counter, 6, "Dropped")

            # with history_lock:
            if request.type in self.sorted_request_history_per_window:
                self.sorted_request_history_per_window[str(request.type)].append(request)
            else:
                self.sorted_request_history_per_window[str(request.type)] = []
                self.sorted_request_history_per_window[str(request.type)].append(request)

            self.results_sheet_row_counter += 1
            self.wb.save("results/" + str(self.worker_id) + "/" + "Results_Episode_" + str(self.episode_no) + ".xls")
        else:
            if request.type in self.pod_object_dict_by_type:
                pod_list_length = len(self.pod_object_dict_by_type[request.type])
                # logging.info("Pods of this type exist, no of pods is : {}".format(pod_list_length))
                # If the global pod counter is greater than the existing pod count, start from the beginning
                if request.type in self.pod_counter:
                    if self.pod_counter[request.type] >= pod_list_length:
                        self.pod_counter[request.type] = 0

                # Allocate requests starting with the pod with highest cpu util
                # sorted_pod_list = sorted(self.pod_object_dict_by_type[request.type], key=self.pod_sorter)
                # for pod in reversed(sorted_pod_list):
                # Allocate requests to pods in a RR manner using the global pod counter for each function type
                for i in range(self.pod_counter[request.type], pod_list_length):
                    pod = self.pod_object_dict_by_type[request.type][i]
                    logging.info("Now considering pod: " + str(pod.id) + "type " + str(pod.type))
                    logging.info("inflight requests: " + str(pod.num_inflight_requests))
                    logging.info(
                        "CLOCK: {} worker: {} pod cpu req : {} pods cpu util: {} pod ram req: {} pod ram util: {}".format(
                            self.clock, self.worker_id, pod.cpu_req,
                            pod.cpu_util,
                            pod.ram_req,
                            pod.ram_util))
                    if ((pod.cpu_req - pod.cpu_util) >= float(self.fn_features[str(request.type) + "_req_MIPS"]/self.fn_features[str(request.type) + "_req_exec_time"])) & ((pod.ram_req - pod.ram_util) >= int(self.fn_features[str(request.type) + "_req_ram"])):
                        request.allocated_pod = pod
                        # logging.info("This pod allocated: pod " + str(pod.id))
                        request.start_time = self.clock
                        request.finish_time = self.clock + float(self.fn_features[str(request.type) + "_req_exec_time"])
                        self.sorted_events.append(
                            defs.EVENT(request.finish_time, constants.finish_request, request))
                        # self.logging.info("Request: {} start time: {} Finish time: {}".format(request.id, request.start_time,
                        #                                                                  request.finish_time))
                        pod_allocated = True
                        pod.num_inflight_requests += 1
                        pod.cpu_util += float(self.fn_features[str(request.type) + "_req_MIPS"]/self.fn_features[str(request.type) + "_req_exec_time"])
                        pod.ram_util += int(self.fn_features[str(request.type) + "_req_ram"])
                        pod.running_req_list.append(request)
                        # with vm_lock:
                        pod.allocated_vm.cpu_used += float(self.fn_features[str(request.type) + "_req_MIPS"]/self.fn_features[str(request.type) + "_req_exec_time"])
                        pod.allocated_vm.ram_used += int(self.fn_features[str(request.type) + "_req_ram"])
                        if request.type in self.pod_counter:
                            self.pod_counter[request.type] += 1
                        else:
                            self.pod_counter[request.type] = 0
                            self.pod_counter[request.type] += 1
                        break

                if not pod_allocated:
                    request.reschedule_tries += 1
                    # logging.info(
                    #     "CLOCK: {} worker: {} Pod is not allocated but rescheduling count not exceeded, thus scheduling again".format(
                    #         self.clock, self.worker_id))
                    self.sorted_events.append(
                        defs.EVENT(self.clock + constants.req_wait_time_to_schedule, constants.re_schedule_request,
                                   request))

            else:
                if request.type in self.pending_pods:
                    request.reschedule_tries += 1
                    # logging.info(
                    #     "CLOCK: {} worker: {} Pod type is not in dict but request type is in pending pods, thus rescheduling request".format(
                    #         self.clock, self.worker_id))
                    self.sorted_events.append(
                        defs.EVENT(self.clock + constants.req_wait_time_to_schedule, constants.re_schedule_request,
                                   request))
                else:
                    self.pending_pods[request.type] = 0
                    self.pending_pods[request.type] += 1
                    request.reschedule_tries += 1
                    # logging.info(
                    #     "CLOCK: {} worker: {} Pod type is not in dict or pending pods thus creating new pod and rescheudling request".format(
                    #         self.clock, self.worker_id))
                    self.sorted_events.append(
                        defs.EVENT(self.clock + constants.pod_creation_time, constants.create_new_pod, request))
                    self.sorted_events.append(
                        defs.EVENT(self.clock + constants.req_wait_time_to_schedule, constants.re_schedule_request,
                                   request))

        self.sorted_events = sorted(self.sorted_events, key=self.sorter_events)
        # Calling pod scaler since after scheduling new requests, pod cpu utilizations would go up and thus they may need to scale up
        self.pod_scaler()

    def req_completion(self, request):

        # logging.info("CLOCK: {} worker: {} Req completion > Request ID: {}".format(self.clock, self.worker_id, request.id))

        request.finish_time = self.clock
        request.allocated_pod.running_req_list.remove(request)
        request.allocated_pod.completed_req_list.append(request)
        # logging.info("CLOCK: {} worker: {} num_inflight_requests before completion: {}".format(self.clock, self.worker_id,
        #                                                                             request.allocated_pod.num_inflight_requests))
        request.allocated_pod.num_inflight_requests -= 1
        # logging.info("CLOCK: {}  worker: {} num_inflight_requests after completion: {}".format(self.clock, self.worker_id,
        #                                                                            request.allocated_pod.num_inflight_requests))
        request.allocated_pod.cpu_util -= float(self.fn_features[str(request.type) + "_req_MIPS"]/self.fn_features[str(request.type) + "_req_exec_time"])
        request.allocated_pod.ram_util -= int(self.fn_features[str(request.type) + "_req_ram"])
        request.allocated_pod.allocated_vm.cpu_used -= float(self.fn_features[str(request.type) + "_req_MIPS"]/self.fn_features[str(request.type) + "_req_exec_time"])
        request.allocated_pod.allocated_vm.ram_used -= int(self.fn_features[str(request.type) + "_req_ram"])
        request.status = "Ok"

        self.req_wl.write(self.results_sheet_row_counter, 0, request.id)
        self.req_wl.write(self.results_sheet_row_counter, 1, request.type)
        self.req_wl.write(self.results_sheet_row_counter, 2, str(request.allocated_pod.id))
        self.req_wl.write(self.results_sheet_row_counter, 3, request.arrival_time)
        self.req_wl.write(self.results_sheet_row_counter, 4, request.start_time)
        self.req_wl.write(self.results_sheet_row_counter, 5, request.finish_time)
        self.req_wl.write(self.results_sheet_row_counter, 6, "Ok")

        if request.type in self.sorted_request_history_per_window:
            self.sorted_request_history_per_window[str(request.type)].append(request)
        else:
            self.sorted_request_history_per_window[str(request.type)] = []
            self.sorted_request_history_per_window[str(request.type)].append(request)

        self.results_sheet_row_counter += 1

        self.wb.save("results/" + str(self.worker_id) + "/" + "Results_Episode_" + str(self.episode_no) + ".xls")

        self.pod_scaler()

        if not request.done:
            request.done = True
            vm = defs.VM(0, len(self.vms), vm_features['vm0_cpu_total_speed'], vm_features['vm0_mem'],
                         vm_features['vm0_cpu_used'], vm_features['vm0_mem_used'],
                         vm_features['vm0_cpu_allocated'],
                         vm_features['vm0_mem_allocated'], vm_features['vm0_price'],
                         vm_features['vm0_nw_bandwidth'],
                         vm_features['vm0_diskio_bandwidth'], [], [])
            pod = defs.POD(self.pod_id, vm, self.fn_features["fn0" + "_name"], 0, 0,
                           self.fn_features["fn0" + "_pod_cpu_req"],
                           self.fn_features["fn0" + "_req_MIPS"],
                           self.fn_features["fn0" + "_pod_ram_req"],
                           self.fn_features["fn0" + "_pod_cpu_util"],
                           self.fn_features["fn0" + "_pod_ram_util"],
                           self.fn_features["fn0" + "_req_per_sec"],
                           self.fn_features["fn0" + "_num_current_replicas"],
                           self.fn_features["fn0" + "_num_max_replicas"],
                           self.fn_features["fn0" + "_inflight_requests"], [], [])
            # Create a request object
            req_obj = defs.REQUEST(request.id, request.chain, request.fn2, pod, self.clock, 0, 0, request.arrival_rate,
                                   0, "initial",
                                   None, True)

            self.sorted_events.append(defs.EVENT(self.clock, constants.schedule_request, req_obj))

            self.sorted_events = sorted(self.sorted_events, key=self.sorter_events)

    def gen_serv_env_state(self, fn_type):
        env_state = np.zeros(148)

        i = 0
        x = 0
        while i < len(self.vms):
            env_state[x] = (self.vms[i].cpu_used / self.vms[i].cpu)
            env_state[x + 1] = (self.vms[i].ram_used / self.vms[i].ram)
            env_state[x + 2] = (self.vms[i].cpu_allocated / self.vms[i].cpu)
            env_state[x + 3] = self.vms[i].cpu / constants.max_vm_cpu
            env_state[x + 4] = self.vms[i].mem_allocated / self.vms[i].ram
            env_state[x + 5] = self.vms[i].ram / constants.max_vm_mem

            replicas = 0
            # print(vms[i].running_pod_list)
            # print(fn_type)

            if fn_type in self.vms[i].running_pod_list:
                replicas = len(self.vms[i].running_pod_list[fn_type])

            # num of replicas of the fn in the vm
            env_state[x + 6] = replicas / constants.max_num_replicas
            x = x + 7
            i += 1

        # the MIPS of a request for the fn
        env_state[x] = self.fn_features[str(fn_type) + "_req_MIPS"] / constants.max_pod_cpu_req
        # the current requested cpu of a pod
        env_state[x + 1] = self.fn_features[str(fn_type) + "_pod_cpu_req"] / constants.max_pod_cpu_req
        # the current requested mem of a pod
        env_state[x + 2] = self.fn_features[str(fn_type) + "_pod_ram_req"] / constants.max_total_pod_mem
        # request concurrency of a pod (affects the average pod resource % used)
        if fn_type in self.pod_object_dict_by_type[fn_type]:
            if len(self.pod_object_dict_by_type[fn_type]) != 0:
                env_state[x + 3] = round(
                    (self.fn_request_rate[fn_type] / len(
                        self.pod_object_dict_by_type[fn_type])) / constants.max_request_rate, 2)

        # logging.info("For state of worker: {} Concurrency of fn {}: {}".format(self.worker_id, fn_type, env_state[x + 2]))

        exec_time = 0
        req_count = 0
        fn_latency = 0
        failed_count = 0
        total_req_count = 0
        failure_rate = 0

        if fn_type in self.sorted_request_history_per_window:
            for req in reversed(self.sorted_request_history_per_window[fn_type]):
                if req.finish_time > self.clock - constants.state_latency_window:
                    total_req_count += 1
                    # self.logging.info(
                    #     "Req id: {}, arrival time: {}, finish time: {}".format(req.id, req.arrival_time, req.finish_time))
                    if req.status != "Dropped":
                        # self.logging.info("Req id: {}, arrival time: {}, finish time: {}".format(req.id, req.arrival_time,
                        #                                                                     req.finish_time))
                        exec_time += req.finish_time - req.arrival_time
                        req_count += 1
                    else:
                        failed_count += 1

                else:
                    break

        if req_count != 0:
            fn_latency = (exec_time / req_count) / float(self.fn_features[str(fn_type) + "_req_exec_time"])

        if total_req_count != 0:
            failure_rate = failed_count / total_req_count

        # logging.info("latency of fn {}: {}".format(fn_type, fn_latency))

        # average request latency of the fn within the considered window in the past
        env_state[x + 4] = round(fn_latency / constants.max_step_latency_perfn, 2)
        env_state[x + 5] = round(failure_rate, 2)

        total_cpu_util = 0
        total_mem_util = 0
        num_pods = 0

        if fn_type in self.pod_object_dict_by_type:
            num_pods = len(self.pod_object_dict_by_type[fn_type])

        if num_pods != 0:
            for pod in self.pod_object_dict_by_type[fn_type]:
                total_cpu_util += pod.cpu_util / pod.cpu_req
                total_mem_util += pod.ram_util / pod.ram_req

            # average cpu and mem util of fn's pods
            env_state[x + 6] = round(total_cpu_util / num_pods, 2)
            env_state[x + 7] = round(total_mem_util / num_pods, 2)

            # logging.info("Avg pod cpu util of fn {}: {}".format(fn_type, env_state[x + 5]))

        return env_state

    def init_workload(self):
        print("Worker: {} New workload created for new episode {}".format(self.worker_id,
                                                                          self.episode_no % constants.num_wls))
        wbr = xlrd.open_workbook("wl/" + str(self.worker_id) + "/" + "wl" + str(
            self.episode_no % constants.num_wls) + ".xls")
        sheet = wbr.sheet_by_index(0)
        for i in range(sheet.nrows - 1):
            # for i in range(500):
            fn_name = int(sheet.cell_value(i + 1, 0))
            arr_time = sheet.cell_value(i + 1, 1)
            arr_rate = sheet.cell_value(i + 1, 2)
            idr = sheet.cell_value(i + 1, 3)
            status = True

            if not (fn_name in self.fn_types):
                self.fn_types.append(fn_name)
            if not (fn_name in self.fn_request_rate):
                self.fn_request_rate[fn_name] = arr_rate

            running_pod_list = {}
            term_pod_list = []
            run_req_list = []
            term_req_list = []

            # creating a bogus VM and pod object so that a request object can be initialized
            vm = defs.VM(0, len(self.vms), vm_features['vm0_cpu_total_speed'], vm_features['vm0_mem'],
                         vm_features['vm0_cpu_used'], vm_features['vm0_mem_used'],
                         vm_features['vm0_cpu_allocated'],
                         vm_features['vm0_mem_allocated'], vm_features['vm0_price'],
                         vm_features['vm0_nw_bandwidth'],
                         vm_features['vm0_diskio_bandwidth'], running_pod_list, term_pod_list)
            pod = defs.POD(self.pod_id, vm, self.fn_features["fn0" + "_name"], 0, 0,
                           self.fn_features["fn0" + "_pod_cpu_req"],
                           self.fn_features["fn0" + "_req_MIPS"],
                           self.fn_features["fn0" + "_pod_ram_req"],
                           self.fn_features["fn0" + "_pod_cpu_util"],
                           self.fn_features["fn0" + "_pod_ram_util"],
                           self.fn_features["fn0" + "_req_per_sec"],
                           self.fn_features["fn0" + "_num_current_replicas"],
                           self.fn_features["fn0" + "_num_max_replicas"],
                           self.fn_features["fn0" + "_inflight_requests"], run_req_list, term_req_list)
            # Create a request object
            req_obj = defs.REQUEST(idr, 'single', "fn" + str(fn_name), pod, arr_time, 0, 0, arr_rate, 0, "initial",
                                   None, status)

            # Event list (unsorted list) consists of the event received time, event name and the object associated with the event
            self.sorted_events.append(defs.EVENT(arr_time, constants.schedule_request, req_obj))

        self.sorted_events = sorted(self.sorted_events, key=self.sorter_events)

    def init_workload_test(self):
        print("Worker: {} New workload created for new episode {}".format(self.worker_id,
                                                                          self.episode_no % constants.num_wls))
        wbr = xlrd.open_workbook("wl/" + str(self.worker_id) + "/" + "wl" + str(
            self.episode_no % constants.num_wls) + ".xls")
        sheet = wbr.sheet_by_index(0)
        for i in range(sheet.nrows - 1):
            fn_type = str(sheet.cell_value(i + 1, 0))
            # for i in range(500):
            if fn_type == 'single':
                fn1_name = int(sheet.cell_value(i + 1, 1))
                fn2_name = None
                status = True
            else:
                fn_list = sheet.cell_value(i + 1, 1)
                fn_list = re.split("; |, |\]|\[|,", fn_list)
                fn1_name, fn2_name = fn_list[1], fn_list[2]
                status = False
            arr_time = sheet.cell_value(i + 1, 2)
            arr_rate = sheet.cell_value(i + 1, 3)
            idr = sheet.cell_value(i + 1, 4)

            if not (fn1_name in self.fn_types):
                self.fn_types.append(fn1_name)
            # if not (fn_name in self.fn_request_rate):
            #     self.fn_request_rate[fn_name] = arr_rate

            running_pod_list = {}
            term_pod_list = []
            run_req_list = []
            term_req_list = []

            # creating a bogus VM and pod object so that a request object can be initialized
            vm = defs.VM(0, len(self.vms), vm_features['vm0_cpu_total_speed'], vm_features['vm0_mem'],
                         vm_features['vm0_cpu_used'], vm_features['vm0_mem_used'],
                         vm_features['vm0_cpu_allocated'],
                         vm_features['vm0_mem_allocated'], vm_features['vm0_price'],
                         vm_features['vm0_nw_bandwidth'],
                         vm_features['vm0_diskio_bandwidth'], running_pod_list, term_pod_list)
            pod = defs.POD(self.pod_id, vm, self.fn_features["fn0" + "_name"], 0, 0,
                           self.fn_features["fn0" + "_pod_cpu_req"],
                           self.fn_features["fn0" + "_req_MIPS"],
                           self.fn_features["fn0" + "_pod_ram_req"],
                           self.fn_features["fn0" + "_pod_cpu_util"],
                           self.fn_features["fn0" + "_pod_ram_util"],
                           self.fn_features["fn0" + "_req_per_sec"],
                           self.fn_features["fn0" + "_num_current_replicas"],
                           self.fn_features["fn0" + "_num_max_replicas"],
                           self.fn_features["fn0" + "_inflight_requests"], run_req_list, term_req_list)
            # Create a request object
            req_obj = defs.REQUEST(idr, fn_type, "fn" + str(fn1_name), pod, arr_time, 0, 0, arr_rate, 0, "initial",
                                   "fn" + str(fn2_name), status)

            # Event list (unsorted list) consists of the event received time, event name and the object associated with the event
            self.sorted_events.append(defs.EVENT(arr_time, constants.schedule_request, req_obj))

        self.sorted_events = sorted(self.sorted_events, key=self.sorter_events)

    print("Request event creation done")


# -*- coding: utf-8 -*-

from datagrid.grids import DataGrid, Column, NonDatabaseColumn
from django.utils.translation import ugettext_lazy as _
from django.conf import settings

class ShiftGrid(DataGrid):
    when = Column()
    span = Column()
    exam_period = Column()
    enabled = Column()

def pl_image(path):
    if path:
        return '' % (settings.MEDIA_URL, path)
    return ''

def pl_desc(good):
    if good.description:
        return '%s
%s' % (good.title, good.description)
    return '%s' % good.title

def pl_price(good):
    return '%s %s' % good.current_costcur()

class PriceListGrid(DataGrid):
    img = Column(_("image"),
        data_func=pl_image,
    )
    desc = NonDatabaseColumn(_("desciption"),
        data_func=pl_desc,
    )
    price = NonDatabaseColumn(_("price"),
        data_func=pl_price,
    )


#!/usr/bin/env python
import subprocess
import json
import sys, os, shutil
from types import SimpleNamespace

# CLI Arguments
if len(sys.argv) != 3:
    raise ValueError('Usage: ./delete-vpcs.py [vpc-id] [region] \n \
        ie. ./delete-vpcs.py vpc-000000 us-west-2')

# Set Arguments
vpc_id = sys.argv[1]
region = sys.argv[2]

# Get All IGWS
igws = json.loads(subprocess.check_output("aws --region=" + region + " --filters=Name=attachment.vpc-id,Values=" + vpc_id + " ec2 describe-internet-gateways", shell=True), \
    object_hook=lambda d: SimpleNamespace(**d))

# Loop Through IGWS and delete / detatch
for ig in igws.InternetGateways:
    ig = ig.InternetGatewayId
    subprocess.check_output("aws --region=" + region + " ec2 detach-internet-gateway --internet-gateway-id=" + str(ig) + " --vpc-id=" + vpc_id, shell=True)
    subprocess.check_output("aws --region=" + region +  " ec2 delete-internet-gateway --internet-gateway-id=" + str(ig), shell=True)

# Get All Subnets
subs = json.loads(subprocess.check_output("aws --region=" + region + " ec2 describe-subnets", shell=True), \
    object_hook=lambda d: SimpleNamespace(**d))

# Loop Through Subnets and delete
for sub in subs.Subnets:

    # Check if part of same VPC
    if (sub.VpcId == vpc_id):
        sub = sub.SubnetId
        subprocess.check_output("aws --region=" + region + " ec2 delete-subnet --subnet-id=" + str(sub), shell=True)

try:
    # Delete vpc
    subprocess.check_output("aws --region=" + region + " ec2 delete-vpc --vpc-id=" + vpc_id, shell=True)
except subprocess.CalledProcessError as e:
    pass


import sys
from os import path
from tempfile import TemporaryDirectory

import netCDF4
import pytest

from cloudnetpy.exceptions import ValidTimeStampError
from cloudnetpy.instruments import basta2nc
from tests.unit.all_products_fun import Check
from tests.unit.radar_fun import RadarFun

SCRIPT_PATH = path.dirname(path.realpath(__file__))
filename = f"{SCRIPT_PATH}/data/basta/basta_1a_cldradLz1R025m_v03_20210827_000000.nc"

class TestBASTA(Check):
    site_meta = {
        "name": "Palaiseau",
        "latitude": 50,
        "longitude": 104.5,
        "altitude": 50,
    }
    date = "2021-08-27"
    temp_dir = TemporaryDirectory()
    temp_path = temp_dir.name + "/test.nc"
    uuid = basta2nc(filename, temp_path, site_meta)

    def test_variable_names(self):
        keys = {
            "Zh",
            "v",
            "time",
            "range",
            "radar_frequency",
            "height",
            "nyquist_velocity",
            "latitude",
            "longitude",
            "altitude",
            "zenith_angle",
            "radar_pitch",
            "radar_yaw",
            "radar_roll",
        }
        assert set(self.nc.variables.keys()) == keys

    def test_variables(self):
        assert self.nc.variables["radar_frequency"][:].data == 95.0  # Hard coded

    def test_common_radar(self):
        radar_fun = RadarFun(self.nc, self.site_meta, self.date, self.uuid)
        for name, method in RadarFun.__dict__.items():
            if "test_" in name:
                getattr(radar_fun, name)()

    def test_geolocation_from_source_file(self, tmp_path):
        test_path = tmp_path / "geo.nc"
        meta_without_geolocation = {"name": "Kumpula"}
        basta2nc(filename, test_path, meta_without_geolocation)
        with netCDF4.Dataset(test_path) as nc:
            for key in ("latitude", "longitude", "altitude"):
                assert key in nc.variables
                assert nc.variables[key][:] > 0

    def test_global_attributes(self):
        assert self.nc.source == "BASTA"
        assert self.nc.title == f"BASTA cloud radar from Palaiseau"

    def test_wrong_date_validation(self, tmp_path):
        with pytest.raises(ValidTimeStampError):
            basta2nc(
                filename,
                tmp_path / "invalid.nc",
                self.site_meta,
                date="2021-01-04",
            )

    def test_uuid_from_user(self, tmp_path):
        test_path = tmp_path / "uuid.nc"
        uuid_from_user = "kissa"
        uuid = basta2nc(filename, test_path, self.site_meta, uuid=uuid_from_user)
        with netCDF4.Dataset(test_path) as nc:
            assert nc.file_uuid == uuid_from_user
            assert uuid == uuid_from_user


from http import HTTPStatus

from django.contrib.auth import get_user_model
from django.test import Client, TestCase
from django.urls import reverse
from posts.models import Group, Post
from posts.utils import print_func_info

User = get_user_model()

class PostCreateFormTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        cls.user = User.objects.create_user(username='TestPotest')
        cls.group = Group.objects.create(
            title='Тестовая группа',
            slug='test_slug',
            description='Тестовое описание',
        )
        cls.post = Post.objects.create(
            author=cls.user,
            text='Тестовый пост' * 15,
            group=cls.group
        )

    def setUp(self):
        self.guest_client = Client()
        self.authorized_client = Client()
        self.authorized_client.force_login(PostCreateFormTests.user)

    @print_func_info
    def test_create_postt_correct_form(self):
        """Валидная форма создает запись в Post."""
        posts_count = Post.objects.count()
        form_data = {'text': 'Старый текст'}
        response = self.authorized_client.post(
            reverse('posts:post_create'), data=form_data, follow=True
        )
        self.assertRedirects(
            response,
            reverse('posts:profile', kwargs={'username': self.user.username})
        )
        self.assertEqual(Post.objects.count(), posts_count + 1)
        self.assertTrue(Post.objects.filter(text=form_data['text']).exists())
        self.assertEqual(response.status_code, HTTPStatus.OK)

    @print_func_info
    def test_post_edit_correct_form(self):
        """Валидная форма изменяет запись в Post."""
        posts_count = Post.objects.count()
        form_data = {'text': 'Новый текст', 'group': self.group.id}
        response = self.authorized_client.post(
            reverse('posts:post_edit', args=({self.post.id})),
            data=form_data,
            follow=True,
        )
        self.assertRedirects(
            response,
            reverse('posts:post_detail', kwargs={'post_id': self.post.id})
        )
        self.assertEqual(Post.objects.count(), posts_count)
        self.assertTrue(Post.objects.filter(text=form_data['text']).exists())
        self.assertEqual(response.status_code, HTTPStatus.OK)


import cv2
import sys
import os
from os.path import join
from glob import glob
import numpy as np
from c3dmodel import *
import torch
import argparse

BATCH_SIZE = 32

parser = argparse.ArgumentParser()
parser.add_argument('--root', help="videos data root")
parser.add_argument('--out', help="dir for output")
parser.add_argument('-f', help="number of frames per clip", default=16)
parser.add_argument('-l', help="specify output layer", default=FC6_LAYER)
args = parser.parse_args()

def extract_frames(video, frame_dir):
    if os.path.isdir(frame_dir):
        print("[Warning] frame_dir={} does exist. Will overwrite".format(frame_dir))
    else:
        os.makedirs(frame_dir)

    cap = cv2.VideoCapture(video)
    if not cap.isOpened():
        print("[Error] video={} can not be opened.".format(video))
        sys.exit(-6)

    ret, frame = cap.read()
    frame_num = 0
    while ret:
        frame_file = os.path.join(
                frame_dir,
                '{0:06d}.jpg'.format(frame_num)
                )
        cv2.imwrite(frame_file, frame)
        ret, frame = cap.read()
        frame_num += 1

    return frame_num - 1

def delete_frames(frame_dir):
    for filename in os.listdir(frame_dir):
        file_path = os.path.join(frame_dir, filename)
        try:
            os.remove(file_path)
        except Exception as e:
            print('Failed to delete file %s. Reason: %s' % (file_path, e))
    try:
        os.rmdir(frame_dir)
    except Exception as e:
        print('Failed to delete dir %s. Reason: %s' % (frame_dir, e))

def get_clips(frame_dir, num_frames_per_clip):
    overlap = int(num_frames_per_clip / 2)
    clips = sorted(glob(join(frame_dir, '*.jpg')))
    clips = np.array([cv2.resize(cv2.cvtColor(cv2.imread(frame), cv2.COLOR_BGR2RGB), (200, 112)) for frame in clips])
    clips = clips[:, :, 44:44 + 112, :]
    clips = np.array([clips[i : i + num_frames_per_clip] for i in range(0, len(clips), overlap) if len(clips[i : i + num_frames_per_clip]) == num_frames_per_clip])
    clips = clips.transpose(0, 4, 1, 2, 3)
    clips = np.float32(clips)
    return torch.from_numpy(clips)

def extractC3D1(video_file, out_layer_type=FC6_LAYER, num_frames_per_clip=16):
    video_id, _ = os.path.splitext(os.path.basename(video_file))

    tmp_dir = './tmp'
    print("Frames extraction started")
    frame_dir = os.path.join(tmp_dir, video_id)
    frames = extract_frames(video_file, frame_dir)
    print("Frames extraction finished, extracted {} frames".format(frames))

    print("Clips extraction started")
    clips = get_clips(frame_dir, num_frames_per_clip)
    if torch.cuda.is_available():
        clips = clips.cuda()
    print("Clips extraction finished")

    model = C3D()
    model.load_state_dict(torch.load('c3d.pickle'))
    if torch.cuda.is_available():
        model.cuda()
    model.eval()

    print("Features extraction started")
    features = []
    for i in range(0, len(clips), BATCH_SIZE):
        print("Batch:", str(i + 1))
        feats = model(torch.from_numpy(np.expand_dims(clips[i:BATCH_SIZE], axis=0)), out_layer_type)
        batch_features = feats.data.cpu()
        features.append(batch_features)

    features = torch.cat(features, 0)
    features = features.numpy()
    print("Features extraction finished, extracted {} features".format(features.shape[0]))

    np.save(os.path.join(args.out, video_id), features)

    print("Deleting frames")
    delete_frames(frame_dir)

def extractC3D(videos_path, out_layer_type=FC6_LAYER, num_frames_per_clip=16):
    videos = [join(videos_path, f) for f in os.listdir(videos_path) if not f.startswith('.')]
    for video in videos:
        print("############## PROCESSING VIDEO {} ##############".format(video))
        video_path = os.path.join(videos_path, video)
        print("Video path:", video_path)
        extractC3D1(video_path, out_layer_type, num_frames_per_clip)

if __name__ == '__main__':
    extractC3D(args.root, args.l, args.f)


from collections import namedtuple

from umlfri2.application.events.diagram import ElementHiddenEvent, ConnectionHiddenEvent
from ..base import Command

HiddenVisualDescription = namedtuple('HiddenVisualDescription', ('z_order', 'visual'))

class HideElementsCommand(Command):
    def __init__(self, diagram, elements):
        self.__diagram_name = diagram.get_display_name()
        self.__diagram = diagram
        self.__elements = elements
        self.__element_visual_descriptions = []
        self.__connection_visual_descriptions = []

    @property
    def description(self):
        return "Hiding selected elements from diagram {0}".format(self.__diagram_name)

    def _do(self, ruler):
        elements = set()
        connections = set()
        for element in self.__elements:
            elements.add(element)
            for connection in element.connections:
                connections.add(connection)

        for visual in elements:
            z_order = self.__diagram.get_z_order(visual)
            self.__element_visual_descriptions.append(HiddenVisualDescription(z_order, visual))

        for visual in connections:
            z_order = self.__diagram.get_z_order(visual)
            self.__connection_visual_descriptions.append(HiddenVisualDescription(z_order, visual))

        self.__element_visual_descriptions.sort(reverse=True) # sort by z_order (desc)
        self.__connection_visual_descriptions.sort(reverse=True) # sort by z_order (desc)

        self._redo(ruler)

    def _redo(self, ruler):
        for z_order, visual in self.__connection_visual_descriptions:
            self.__diagram.remove(visual)

        for z_order, visual in self.__element_visual_descriptions:
            self.__diagram.remove(visual)

    def _undo(self, ruler):
        for z_order, visual in reversed(self.__element_visual_descriptions):
            self.__diagram.add(visual, z_order=z_order)

        for z_order, visual in reversed(self.__connection_visual_descriptions):
            self.__diagram.add(visual, z_order=z_order)

    def get_updates(self):
        for z_order, visual in self.__element_visual_descriptions:
            yield ElementHiddenEvent(visual)

        for z_order, visual in self.__connection_visual_descriptions:
            yield ConnectionHiddenEvent(visual)


#!/usr/bin/python3

import json
import sys
import os.path

if len(sys.argv) < 3:
    print("usage: " + sys.argv[0] + " <file to download> [<region> [<region>]]")
    exit(1)

filename = os.path.basename(sys.argv[1])
regions = []

for region in sys.argv[2:]:
    regions.append(region)

with open(filename, 'r') as fd:
    ranges = json.load(fd)

    for range in ranges["prefixes"]:
        if len(regions) == 0:
            print(range["ip_prefix"])
        else:
            for region in regions:
                if range["region"] == region:
                    print(range["ip_prefix"])
fd.close()


import numpy as np

import torch
import torch.nn.functional as F

def mrr(gt_item, pred_items):
	if gt_item in pred_items:
		index = pred_items.index(gt_item)
		return np.reciprocal(float(index+1))
	else:
		return 0

def hit(gt_item, pred_items):
	if gt_item in pred_items:
		return 1
	return 0

def ndcg(gt_item, pred_items):
	if gt_item in pred_items:
		index = pred_items.index(gt_item)
		return np.reciprocal(np.log2(index+2))
	return 0

def metrics(model, data_test, dataloader_test, top_k, is_output, epoch):
	Mrr, Hr, Ndcg = [], [], []

	######################## COMPUTE ALL ITEMS ONCE ####################
	all_items_embed = []
	all_item_idxs = []
	for vis, text, asin in data_test.get_all_test():
		vis = torch.cuda.FloatTensor(vis).view(1, -1)
		text = torch.cuda.FloatTensor(text).view(1, -1)
		item_embed = model(None, None, vis, text, None, None, True)

		all_item_idxs.append(asin)
		all_items_embed.append(item_embed.view(-1).data.cpu().numpy())

	item_size = len(all_item_idxs)
	all_items_embed = torch.cuda.FloatTensor(np.array(all_items_embed))

	all_items_map = {i: item for i, item in enumerate(all_item_idxs)}
	user_bought = data_test.user_bought

	####################### FOR EVERY USER-QUERY PAIR ####################
	for idx, batch_data in enumerate(dataloader_test):
		user = batch_data['userID'].cuda()
		query = batch_data['query'].cuda()
		reviewerID = batch_data['reviewerID'][0]
		item = batch_data['item'][0]
		query_text = batch_data['query_text']

		item_predict = model(user, query, None, None, None, None, False)

		scores = F.pairwise_distance(
					item_predict.repeat(item_size, 1), all_items_embed)

		_, ranking_list = scores.sort(dim=-1, descending=True)
		ranking_list = [all_items_map[i] for i in ranking_list.tolist()]
		top_idx = []
		u_bought = user_bought[reviewerID]
		while len(top_idx) < top_k:
			candidate_item = ranking_list.pop()
			if candidate_item not in u_bought or candidate_item == item:
				top_idx.append(candidate_item)

		Mrr.append(mrr(item, top_idx))
		Hr.append(hit(item, top_idx))
		Ndcg.append(ndcg(item, top_idx))

	return np.mean(Mrr), np.mean(Hr), np.mean(Ndcg)


"""
============================
LASA Handwriting with ProMPs
============================

The LASA Handwriting dataset learned with ProMPs. The dataset consists of
2D handwriting motions. The first and third column of the plot represent
demonstrations and the second and fourth column show the imitated ProMPs
with 1-sigma interval.
"""
print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from movement_primitives.data import load_lasa
from movement_primitives.promp import ProMP

def draw(T, X, idx, axes, shape_name):
    h = int(idx / width)
    w = int(idx % width) * 2
    axes[h, w].set_title(shape_name)
    axes[h, w].plot(X[:, :, 0].T, X[:, :, 1].T)

    promp = ProMP(n_weights_per_dim=30, n_dims=X.shape[2])
    promp.imitate(T, X)
    mean = promp.mean_trajectory(T[0])
    std = np.sqrt(promp.var_trajectory(T[0]))
    axes[h, w + 1].plot(mean[:, 0], mean[:, 1], c="r")
    axes[h, w + 1].plot(mean[:, 0] - std[:, 0], mean[:, 1] - std[:, 1], c="g")
    axes[h, w + 1].plot(mean[:, 0] + std[:, 0], mean[:, 1] + std[:, 1], c="g")

    axes[h, w + 1].set_xlim(axes[h, w].get_xlim())
    axes[h, w + 1].set_ylim(axes[h, w].get_ylim())
    axes[h, w].get_yaxis().set_visible(False)
    axes[h, w].get_xaxis().set_visible(False)
    axes[h, w + 1].get_yaxis().set_visible(False)
    axes[h, w + 1].get_xaxis().set_visible(False)

width = 2
height = 5

fig, axes = plt.subplots(int(height), int(width * 2))

for i in range(width * height):
    T, X, Xd, Xdd, dt, shape_name = load_lasa(i)
    draw(T, X, i, axes, shape_name)
plt.tight_layout()
plt.show()


import logging
import time
from typing import Any, List

from transformers import pipeline

import cltl.emotion_extraction.emotion_mappings as mappings
from cltl.emotion_extraction.api import EmotionExtractor, EmotionType, Emotion

logger = logging.getLogger(__name__)

_MODEL_NAME = "bhadresh-savani/bert-base-go-emotion"
_THRESHOLD = 0.5

# GO Emotions is a finetuned BERT transformer with the GO Emotion data
#https://github.com/google-research/google-research/tree/master/goemotions
#https://github.com/google-research/google-research/blob/master/goemotions/goemotions_model_card.pdf

class GoEmotionDetector(EmotionExtractor):
    def __init__(self, model: str = _MODEL_NAME):
        self.emotion_pipeline = pipeline('sentiment-analysis',  model=model, return_all_scores=True)

    def extract_audio_emotions(self, audio_signal: Any) -> List[Emotion]:
        raise NotImplementedError()

    def extract_text_emotions(self, utterance: str) -> List[Emotion]:
        if not utterance:
            return []

        logger.debug(f"sending utterance to server...")
        start = time.time()

        emotions = []

        response = self.emotion_pipeline(utterance)

        emotion_labels = mappings.sort_predictions(response[0])
        emotions.extend(self._filter_by_threshold(EmotionType.GO, emotion_labels))

        ekman_labels = mappings.get_total_mapped_scores(mappings.go_ekman_map, emotion_labels)
        emotions.extend(self._filter_by_threshold(EmotionType.EKMAN, ekman_labels))

        sentiment_labels = mappings.get_total_mapped_scores(mappings.go_sentiment_map, emotion_labels)
        emotions.extend(self._filter_by_threshold(EmotionType.SENTIMENT, sentiment_labels))

        self._log_results(emotions, response, start)

        return emotions

    def _filter_by_threshold(self, emotion_type, results):
        return [Emotion(type=emotion_type, value=result['label'], confidence=result['score'])
                for result in results
                if result['score'] > 0 and result['score'] / results[0]['score'] > _THRESHOLD]

    def _log_results(self, emotions, response, start):
        logger.info("got %s from server in %s sec", response, time.time() - start)
        logger.info("All Go emotion detected: %s", [emotion.value for emotion in emotions
                                                    if emotion.type == EmotionType.GO])
        logger.info("Highest scoring Go emotion: %s", next(emotion.value for emotion in emotions
                                                           if emotion.type == EmotionType.GO))
        logger.info("Highest scoring Ekman emotion: %s", next(emotion.value for emotion in emotions
                                                              if emotion.type == EmotionType.EKMAN))
        logger.info("Highest scoring Sentiment: %s", next(emotion.value for emotion in emotions
                                                          if emotion.type == EmotionType.SENTIMENT))

if __name__ == "__main__":
    utterance = "I love cats."
    analyzer = GoEmotionDetector()
    emotions = analyzer.extract_text_emotions(utterance)
    emotion_json ={}
    for emotion in emotions:
        emotion_json.update({'type': emotion.type, 'value':emotion.value, 'confident': emotion.confidence})
        print(emotion_json)