import os
from subprocess import Popen

from escape.api.rest_API import RESTAPIManager
from escape.nffg_lib.nffg import NFFG, NFFGToolBox
from escape.orchest.ros_API import InstantiationFinishedEvent
from escape.service import LAYER_NAME, log as log  # Service layer logger
from escape.service.element_mgmt import ClickManager
from escape.service.sas_orchestration import ServiceOrchestrator
from escape.util.api import AbstractAPI, RequestStatus, \
  RequestScheduler
from escape.util.config import CONFIG
from escape.util.conversion import NFFGConverter
from escape.util.domain import BaseResultEvent
from escape.util.mapping import PreMapEvent, PostMapEvent, ProcessorError
from escape.util.misc import schedule_delayed_as_coop_task, \
  schedule_as_coop_task, VERBOSE, quit_with_ok, \
  get_global_parameter, quit_with_error
from escape.util.stat import stats
from pox.lib.revent.revent import Event

SCHEDULED_SERVICE_REQUEST_DELAY = CONFIG.get_sas_request_delay()

class InstantiateNFFGEvent(Event):
  """
  Event for passing NFFG (mapped SG) to Orchestration layer.
  """

  def __init__ (self, nffg, resource_nffg):
    """
    Init.

    :param nffg: NF-FG need to be initiated
    :type nffg: :class:`NFFG`
    :return: None
    """
    super(InstantiateNFFGEvent, self).__init__()
    self.nffg = nffg
    self.resource_nffg = resource_nffg
    stats.add_measurement_end_entry(type=stats.TYPE_SERVICE, info=LAYER_NAME)

class GetVirtResInfoEvent(Event):
  """
  Event for requesting virtual resource info from Orchestration layer.
  """

  def __init__ (self, sid):
    """
    Init.

    :param sid: Service layer ID
    :type sid: int
    :return: None
    """
    super(GetVirtResInfoEvent, self).__init__()
    # service layer ID
    self.sid = sid

class ServiceLayerAPI(AbstractAPI):
  """
  Entry point for Service Adaptation Sublayer.

  Maintain the contact with other UNIFY layers.

  Implement the U - Sl reference point.
  """
  # Defined specific name for core object as pox.core.<_core_name>
  _core_name = LAYER_NAME
  """Defined specific name for core object """
  # Layer id constant
  LAYER_ID = "ESCAPE-" + LAYER_NAME
  """Layer id constant"""
  # Events raised by this class
  _eventMixin_events = {InstantiateNFFGEvent, GetVirtResInfoEvent, PreMapEvent,
                        PostMapEvent}
  """Events raised by this class"""
  # Dependencies
  dependencies = ('orchestration', 'REST-API')
  """Layer dependencies"""

  def __init__ (self, standalone=False, **kwargs):
    """
    .. seealso::
      :func:`AbstractAPI.__init__() <escape.util.api.AbstractAPI.__init__>`
    """
    log.info("Starting Service Layer...")
    # Mandatory super() call
    self.last_sg = NFFG(id=0, name='empty')
    # Set element manager
    self.__sid = None
    self.elementManager = None
    self.service_orchestrator = None
    """:type ServiceOrchestrator"""
    self.gui_proc = None
    self.api_mgr = RESTAPIManager(unique_bb_id=False,
                                  unique_nf_id=CONFIG.ensure_unique_vnf_id(),
                                  logger=log)
    super(ServiceLayerAPI, self).__init__(standalone, **kwargs)

  def initialize (self):
    """
    .. seealso::
      :func:`AbstractAPI.initialize() <escape.util.api.AbstractAPI.initialize>`
    """
    log.debug("Initializing Service Layer...")
    self.__sid = LAYER_NAME
    log.debug("Setup ID for Service Layer: %s" % self.__sid)
    # Set element manager
    self.elementManager = ClickManager()
    # Init central object of Service layer
    self.service_orchestrator = ServiceOrchestrator(self)
    # Read input from file if it's given and initiate SG
    if self._sg_file:
      try:
        stats.init_request_measurement(request_id=self._sg_file)
        service_request = self._read_data_from_file(self._sg_file)
        log.info("Graph representation is loaded successfully!")
        if service_request.startswith('{'):
          log.debug("Detected format: JSON - Parsing from NFFG format...")
          nffg = NFFG.parse(raw_data=service_request)
        elif service_request.startswith('<'):
          log.debug("Detected format: XML - Parsing from Virtualizer format...")
          converter = NFFGConverter(domain="INTERNAL", logger=log,
                                    unique_bb_id=False,
                                    unique_nf_id=CONFIG.ensure_unique_vnf_id())
          nffg = converter.parse_from_Virtualizer(vdata=service_request)
        else:
          log.warning("Detected unexpected format...")
          return
        if nffg.mode is not None:
          log.info('Detected mapping mode in NFFG: %s' % nffg.mode)
        else:
          nffg.mode = NFFG.MODE_ADD
          log.info("No mapping mode has been detected in NFFG! "
                   "Set default mode: %s" % nffg.mode)
        log.info("Schedule service request delayed by %d seconds..."
                 % SCHEDULED_SERVICE_REQUEST_DELAY)
        stats.set_request_id(request_id=nffg.id)
        self.api_sg_delayed(id=nffg.id, data=nffg)
      except (ValueError, IOError, TypeError) as e:
        log.error(
          "Can't load service request from file because of: " + str(e))
        quit_with_error(msg=str(e), logger=log)
    else:
      # Init REST-API if no input file is given
      self._initiate_rest_api()
    # Init GUI
    if self._gui:
      self._initiate_gui()
    log.info("Service Layer has been initialized!")

  def shutdown (self, event):
    """
    .. seealso::
      :func:`AbstractAPI.shutdown() <escape.util.api.AbstractAPI.shutdown>`

    :param event: event object
    """
    log.info("Service Layer is going down...")
    if self.gui_proc:
      log.debug("Shut down GUI process - PID: %s" % self.gui_proc.pid)
      self.gui_proc.terminate()

  def _initiate_rest_api (self):
    """
    Initialize and set up REST API in a different thread.

    :return: None
    """
    rest_api = self.get_dependent_component('REST-API')
    rest_api.register_component(component=self)
    return

  def _initiate_gui (self):
    """
    Initiate and set up GUI.

    :return: None
    """
    # TODO - set up and initiate MiniEdit here???
    devnull = open(os.devnull, 'r+')
    gui_path = os.path.abspath(os.getcwd() + "/gui/gui.py")
    self.gui_proc = Popen(gui_path, stdin=devnull, stdout=devnull,
                          stderr=devnull, close_fds=True)
    log.info("GUI has been initiated!")

  def _handle_SGMappingFinishedEvent (self, event):
    """
    Handle SGMappingFinishedEvent and proceed with  :class:`NFFG
    <escape.util.nffg.NFFG>` instantiation.

    :param event: event object
    :type event: :any:`SGMappingFinishedEvent`
    :return: None
    """
    self._proceed_to_instantiate_NFFG(event.nffg)

  ##############################################################################
  # UNIFY U - Sl API functions starts here
  ##############################################################################

  # noinspection PyUnusedLocal
  @schedule_as_coop_task
  def rest_api_sg (self, id, data, *args, **kwargs):
    """
    Initiate service graph in a cooperative micro-task.

    :return: None
    """
    self.__proceed_sg_request(id=id, data=data, **kwargs)

  # noinspection PyUnusedLocal
  @schedule_delayed_as_coop_task(delay=SCHEDULED_SERVICE_REQUEST_DELAY)
  def api_sg_delayed (self, id, data, *args, **kwargs):
    """
    Initiate service graph in a cooperative micro-task.

    :return: None
    """
    return self.__proceed_sg_request(id=id, data=data)

  def __proceed_sg_request (self, id, data, params=None):
    """
    Initiate a Service Graph (UNIFY U-Sl API).

    :return: None
    """
    log.info("Invoke preprocessing on %s with SG: %s "
             % (self.__class__.__name__, id))
    stats.add_measurement_start_entry(type=stats.TYPE_SERVICE, info=LAYER_NAME)
    if CONFIG.get_rest_api_config(self._core_name)['unify_interface']:
      log.debug("Virtualizer format enabled! Start conversion step...")
      if CONFIG.get_rest_api_config(self._core_name)['diff']:
        log.debug("Diff format enabled! Start patching step...")
        if self.api_mgr.last_response is None:
          log.info("Missing cached Virtualizer! Acquiring topology now...")
          self.rest_api_topology()
        stats.add_measurement_start_entry(type=stats.TYPE_PROCESSING,
                                          info="RECREATE-FULL-REQUEST")
        log.info("Patching cached topology with received diff...")
        full_req = self.api_mgr.last_response.yang_copy()
        full_req.patch(source=data)
        stats.add_measurement_end_entry(type=stats.TYPE_PROCESSING,
                                        info="RECREATE-FULL-REQUEST")
      else:
        full_req = data
      log.info("Converting full request data...")
      stats.add_measurement_start_entry(type=stats.TYPE_CONVERSION,
                                        info="VIRTUALIZER-->NFFG")
      service_nffg = self.api_mgr.converter.parse_from_Virtualizer(
        vdata=full_req)
      stats.add_measurement_end_entry(type=stats.TYPE_CONVERSION,
                                      info="VIRTUALIZER-->NFFG")
    else:
      service_nffg = data
    log.debug("Set NFFG id: %s" % id)
    if service_nffg.service_id is None:
      service_nffg.service_id = service_nffg.id
    service_nffg.id = id
    service_nffg.add_metadata(name="params", value=params)
    # Check if mapping mode is set globally in CONFIG
    mapper_params = CONFIG.get_mapping_config(layer=LAYER_NAME)
    if 'mode' in mapper_params and mapper_params['mode'] is not None:
      mapping_mode = mapper_params['mode']
      log.info("Detected mapping mode from configuration: %s" % mapping_mode)
    elif service_nffg.mode is not None:
      mapping_mode = service_nffg.mode
      log.info("Detected mapping mode from NFFG: %s" % mapping_mode)
    else:
      mapping_mode = None
      log.info("No mapping mode was detected!")
    self.__sg_preprocessing(nffg=service_nffg)
    # Store request if it is received on REST-API
    log.getChild('API').debug("Store received NFFG request info...")
    msg_id = self.api_mgr.request_cache.cache_request_by_nffg(
      nffg=service_nffg)
    if msg_id is not None:
      self.api_mgr.request_cache.set_in_progress(id=msg_id)
      log.getChild('API').debug("Request is stored with id: %s" % msg_id)
    else:
      log.getChild('API').debug("No request info detected.")
    try:
      if CONFIG.get_mapping_enabled(layer=LAYER_NAME):
        # Initiate service request mapping
        mapped_nffg = self.service_orchestrator.initiate_service_graph(
          service_nffg)
      else:
        log.warning("Mapping is disabled! Skip instantiation step...")
        mapped_nffg = service_nffg
        mapped_nffg.status = NFFG.MAP_STATUS_SKIPPED
        log.debug("Mark NFFG status: %s!" % mapped_nffg.status)
      # Rewrite REMAP mode for backward compatibility
      if mapped_nffg is not None and mapping_mode == NFFG.MODE_REMAP:
        mapped_nffg.mode = mapping_mode
        log.debug("Rewrite mapping mode: %s into mapped NFFG..." %
                  mapped_nffg.mode)
      else:
        log.debug(
          "Skip mapping mode rewriting! Mode remained: %s" % mapping_mode)
      log.getChild('API').debug("Invoked request_service on %s is finished" %
                                self.__class__.__name__)
      # If mapping is not threaded and finished with OK
      if mapped_nffg is not None and not \
         self.service_orchestrator.mapper.threaded:
        self._proceed_to_instantiate_NFFG(mapped_nffg)
        self.last_sg = mapped_nffg
      else:
        log.warning("Something went wrong in service request initiation: "
                    "mapped service data is missing!")
        self.__handle_mapping_result(nffg_id=service_nffg.id, fail=True)
        stats.add_measurement_end_entry(type=stats.TYPE_SERVICE,
                                        info=LAYER_NAME + "-FAILED")
        self._handle_InstantiationFinishedEvent(
          event=InstantiationFinishedEvent(
            id=service_nffg.id,
            result=InstantiationFinishedEvent.MAPPING_ERROR))
    except ProcessorError as e:
      self.__handle_mapping_result(nffg_id=service_nffg.id, fail=True)
      stats.add_measurement_end_entry(type=stats.TYPE_SERVICE,
                                      info=LAYER_NAME + "-DENIED")
      self._handle_InstantiationFinishedEvent(
        event=InstantiationFinishedEvent(
          id=service_nffg.id,
          result=InstantiationFinishedEvent.REFUSED_BY_VERIFICATION,
          error=e))

  @staticmethod
  def __sg_preprocessing (nffg):
    """
    Preprocess given :class:`NFFG` based on request mode.

    :param nffg: received service request
    :type nffg: :class:`NFFG`
    :return: modified request
    :rtype: :class:`NFFG`
    """
    if nffg.mode == NFFG.MODE_DEL:
      log.debug("Explicitly mark NF nodes in DELETE request...")
      for nf in nffg.nfs:
        nf.operation = NFFG.OP_DELETE
        log.debug("%s --> %s" % (nf.id, nf.operation))
    return nffg

  def __handle_mapping_result (self, nffg_id, fail):
    """
    Perform necessary task for callback and cache functionality based on mapping
    result.

    :param nffg_id: request ID
    :type nffg_id: str or int
    :param fail: mapping result
    :type fail: bool
    :return: None
    """
    log.getChild('API').debug("Cache request status...")
    req_status = self.api_mgr.request_cache.get_request_by_nffg_id(nffg_id)
    if req_status is None:
      log.getChild('API').debug("Request status is missing for NFFG: %s! "
                                "Skip result processing..." % nffg_id)
      return
    log.getChild('API').debug("Process mapping result...")
    message_id = req_status.message_id
    if message_id is not None:
      if fail:
        self.api_mgr.request_cache.set_error_result(id=message_id)
      else:
        self.api_mgr.request_cache.set_success_result(id=message_id)
      ret = self.api_mgr.invoke_callback(message_id=message_id)
      if ret is None:
        log.getChild('API').debug("No callback was defined!")
      else:
        log.getChild('API').debug(
          "Callback: %s has invoked with return value: %s" % (
            req_status.get_callback(), ret))
    RequestScheduler().set_orchestration_finished(id=nffg_id)

  def __get_sas_resource_view (self):
    """
    Return with the resource view of SAS layer.

    :return: resource view
    :rtype: :any:`AbstractVirtualizer`
    """
    return self.service_orchestrator.virtResManager.virtual_view

  def rest_api_topology (self):
    """
    Return with the topology description.

    :return: topology description requested from the layer's Virtualizer
    :rtype: :class:`NFFG`
    """
    log.getChild('[U-Sl]').debug("Requesting Virtualizer for REST-API...")
    # Get or if not available then request the layer's Virtualizer
    sas_virt = self.__get_sas_resource_view()
    if sas_virt is not None:
      if sas_virt.revision is None:
        log.debug("Not initialized yet!")
      else:
        # Check if the resource is changed
        if self.api_mgr.topology_revision == sas_virt.revision:
          # If resource has not been changed return False
          # This causes to response with the cached topology
          log.debug("Global resource has not changed (revision: %s)! "
                    % sas_virt.revision)
          log.debug("Send topology from cache...")
          if self.api_mgr.last_response is None:
            log.error("Cached topology is missing!")
            return
          else:
            return self.api_mgr.last_response
        else:
          log.debug("Response cache is outdated (new revision: %s)!"
                    % sas_virt.revision)
        log.getChild('[U-Sl]').debug("Generate topo description...")
      # return with the virtual view as an NFFG
      res = sas_virt.get_resource_info()
      self.api_mgr.topology_revision = sas_virt.revision
      log.debug("Updated revision number: %s"
                % self.api_mgr.topology_revision)
      if CONFIG.get_rest_api_config(self._core_name)['unify_interface']:
        log.info("Convert internal NFFG to Virtualizer...")
        res = self.api_mgr.converter.dump_to_Virtualizer(nffg=res)
      log.debug("Cache acquired topology...")
      self.api_mgr.last_response = res
      return res
    else:
      log.getChild('[U-Sl]').error(
        "Virtualizer(id=%s) assigned to REST-API is not found!" %
        self._core_name)

  def api_sas_status (self, message_id):
    """
    Return the state of a request given by ``message_id``.

    Function is not invoked in coop-microtask, only write-type operations
    must not be used.

    :param message_id: request id
    :type message_id: str or int
    :return: state
    :rtype: str
    """
    status = self.api_mgr.request_cache.get_domain_status(id=message_id)
    if status == RequestStatus.SUCCESS:
      return 200, None
    elif status == RequestStatus.UNKNOWN:
      return 404, None
    elif status == RequestStatus.ERROR:
      return 500, status
    else:
      # PROCESSING or INITIATED
      return 202, None

  def _proceed_to_instantiate_NFFG (self, mapped_nffg):
    """
    Send NFFG to Resource Orchestration Sublayer in an implementation-specific
    way.

    General function which is used from microtask and Python thread also.

    This function contains the last steps before the mapped NFFG will be sent
    to the next layer.

    :param mapped_nffg: mapped Service Graph
    :type mapped_nffg: :class:`NFFG`
    :return: None
    """
    # Rebind requirement link fragments for lower layer mapping
    mapped_nffg = NFFGToolBox.rebind_e2e_req_links(nffg=mapped_nffg, log=log)
    # Log verbose mapping result in unified way (threaded/non-threaded)
    log.log(VERBOSE,
            "Mapping result of Service Layer:\n%s" % mapped_nffg.dump())
    # Sending mapped SG / NF-FG to Orchestration layer as an Event
    # Exceptions in event handlers are caught by default in a non-blocking way
    sas_res = self.__get_sas_resource_view().get_resource_info()
    self.raiseEventNoErrors(InstantiateNFFGEvent, mapped_nffg, sas_res)
    log.getChild('API').info(
      "Generated NF-FG: %s has been sent to Orchestration..." % mapped_nffg)

  ##############################################################################
  # UNIFY Sl - Or API functions starts here
  ##############################################################################

  # noinspection PyUnusedLocal
  def _handle_MissingVirtualViewEvent (self, event):
    """
    Request virtual resource info from Orchestration layer (UNIFY Sl - Or API).

    Invoked when a :class:`MissingVirtualViewEvent` raised.

    Service layer is identified with the sid value automatically.

    :param event: event object
    :type event: :any:`MissingVirtualViewEvent`
    :return: None
    """
    log.getChild('API').debug(
      "Send <Virtual View> request(with layer ID: %s) to Orchestration "
      "layer..." % self.__sid)
    self.raiseEventNoErrors(GetVirtResInfoEvent, self.__sid)

  def _handle_VirtResInfoEvent (self, event):
    """
    Save requested virtual resource info as an :class:`AbstractVirtualizer
    <escape.orchest.virtualization_mgmt.AbstractVirtualizer>`.

    :param event: event object
    :type event: :any:`VirtResInfoEvent`
    :return: None
    """
    log.getChild('API').debug("Received <Virtual View>: %s from %s layer" % (
      event.virtualizer, str(event.source._core_name).title()))
    self.service_orchestrator.virtResManager.virtual_view = event.virtualizer

  def _handle_InstantiationFinishedEvent (self, event):
    """
    Receive the result of the instantiated NFFG and save it.

    :param event: event object
    :type event: :any:`InstantiationFinishedEvent`
    :return: None
    """
    if not BaseResultEvent.is_error(event.result):
      log.getChild('API').info(
        "Service request(id=%s) has been finished successfully with result: %s!"
        % (event.id, event.result))
    else:
      log.getChild('API').error(
        "Service request(id=%s) has been finished with error result: %s!" %
        (event.id, event.result))
    if not event.is_pending(event.result):
      self.__handle_mapping_result(nffg_id=event.id,
                                   fail=event.is_error(event.result))
      # Quit ESCAPE if test mode is active
      if get_global_parameter(name="QUIT_AFTER_PROCESS"):
        stats.finish_request_measurement()
        quit_with_ok("Detected QUIT mode! Exiting ESCAPE...")


from django_registration.backends.activation import views as registration_views

from django.test import SimpleTestCase
from django.urls import reverse, resolve
from django.contrib.auth import views as auth_views

from auth_base import views

class AuthURLsTests(SimpleTestCase):
    '''
    Tests checking auth URLs
    '''

    def test_login_view_resolves(self):
        '''
        auth_base:login
        '''
        url = reverse('auth_base:login')
        self.assertEqual(resolve(url).func.view_class, auth_views.LoginView)

    def test_logout_view_resolves(self):
        '''
        auth_base:logout
        '''
        url = reverse('auth_base:logout')
        self.assertEqual(resolve(url).func.view_class, auth_views.LogoutView)

class RegistrationURLsTests(SimpleTestCase):
    '''
    Tests checking registration URLs
    '''

    def test_activate_view_resolves(self):
        '''
        auth_base:activate
        '''
        url = reverse(
            'auth_base:activate',
            args=[
                'IoelYjQi:1kckPR:JHTTCehMd752yaNJyMJY4oChloxBCkd7hxKepVbjtR4'
            ])
        self.assertEqual(
            resolve(url).func.view_class, registration_views.ActivationView
        )

    def test_register_view_resolves(self):
        '''
        auth_base:register
        '''
        url = reverse('auth_base:register')
        self.assertEqual(resolve(url).func.view_class, views.RegisterView)


import pickle

import matplotlib.pyplot as plt
import numpy as np

from common_data import scenarios, hours, units, unit_to_generation_type
plt.rcParams.update({'font.size': 18})

# 0: coal
# 1: gas
# 2: ccgt
# 3: oil
# 4: biomass
# 5: oil shale
# 6: nuclear
# 7: hydro
# 8: wind
# 9: pv
# different chp types (e=extraction, b=back pressure):
# 10: coal-e
# 11: gas-b
# 12: gas-e
# 13: oil-b
# 14: oil-e
# 15: biomass-e
# 16: waste-e
# 17: peat-e
fuel_types = [
    "coal",
    "gas",
    "oil",
    "biomass",
    "oil shale",
    "nuclear",
    "hydro",
    "wind",
    "solar",
]

order = [6, 8, 7, 3, 5, 0, 1, 4, 2]

generation_type_to_fuel_type = {
    0: 0,
    1: 1,
    2: 1,
    3: 2,
    4: 3,
    5: 4,
    6: 5,
    7: 6,
    8: 7,
    9: 8,
    10: 0,
    11: 1,
    12: 1,
    13: 2,
    14: 2,
    15: 3,
    16: 3,
    17: 3,
}

titles = ["no emission constraint", "with emission constraint"]
# light gray, red, black, green, dark gray, yellow, dark blue, light blue, orange
colors = [
    "#a6a6a6",
    "#ff0000",
    "#000000",
    "#00ff08",
    "#636363",
    "#fffb00",
    "#00058a",
    "#99dbff",
    "#ffa200",
]

def mysort(X):
    return [X[o] for o in order]

fig, ax = plt.subplots(2, 2, figsize=(14, 12))

with open("stoch.pickle", "rb") as file_:
    data = pickle.load(file_)

idx = 0

models = list(data.keys())
if "stoch" in models[0]:
    sorted_models = [models[1], models[0]]
else:
    sorted_models = models

for model in sorted_models:
    operation_decisions = data[model]
    g = operation_decisions["g"]
    f = operation_decisions["f"]

    for idx2, (offset, year) in enumerate([(0, 1), (len(hours) - 24, 10)]):
        g_by_fuel = [0.0] * len(fuel_types)

        for o in scenarios:
            for u in units:
                generation_type = unit_to_generation_type[u]
                fuel = generation_type_to_fuel_type[generation_type]

                for t in range(24):
                    g_by_fuel[fuel] += g[o, offset + t, u]

        currrent_labels = [
            l + " %.1f" % (g_by_fuel[i] / sum(g_by_fuel) * 100.0) + "%"
            #l + " %.1f" % (g_by_fuel[i] / 1000.0) + "GW"
            for i, l in enumerate(fuel_types)
        ]

        model_name = "SARO"
        if "stoch" in model:
            model_name = "SP"

        current_title = model_name + ", t = %d" % year

        my_pie, texts = ax[idx2, idx].pie(
            mysort(g_by_fuel),
            labels=mysort(currrent_labels),
            colors=mysort(colors),
            radius=0.7,
            # autopct="%.2f",
            #pctdistance=0.9,
            labeldistance=1.1,
            explode=mysort([0.15, 0.15, 0.15, 0.1, 0.15, 0.1, 0.1, 0.1, 0.1]),
        )

        #import pdb; pdb.set_trace()

        for txt in texts:
            t = txt.get_text()

            if "oil shale" in t:
                pos = txt.get_position()
                pos2 = (pos[0], pos[1]-0.025)
                txt.set_position(pos2)

            if "oil" in t and "oil shale" not in t:
                pos = txt.get_position()
                pos2 = (pos[0], pos[1]+0.05)
                txt.set_position(pos2)

            if "gas" in t or "coal" in t:
                pos = txt.get_position()
                pos2 = (pos[0], pos[1]-0.05)
                txt.set_position(pos2)

            if "wind" in t:
                pos = txt.get_position()
                pos2 = (pos[0], pos[1]-0.05)
                txt.set_position(pos2)

            if "biomass" in t:
                pos = txt.get_position()
                pos2 = (pos[0], pos[1]-0.1)
                txt.set_position(pos2)

        ax[idx2, idx].set_title(current_title)

    idx = idx + 1

plt.savefig("generation_mixes_stoch.png", bbox_inches="tight")

# fuel_types = [
#     "coal",       0
#     "gas",        1
#     "oil",        2
#     "biomass",    3
#     "oil shale",  4
#     "nuclear",    5
#     "hydro",      6
#     "wind",       7
#     "solar",      8
# ]


import os
import datetime
import json
import logging
import logging.handlers
import inspect
from jsonHelper import MyEncoder
import xdm
import traceback

LOGLINECACHESIZE = 20
# "c" is for console "p" is for i forgot, but it is used for the file logger
# the "c" strings have terminal control strings that change the color ^_^
lvlNames = {    logging.ERROR:      {'c': u'   \x1b[31;1mERROR\x1b[0m', 'p': 'ERROR'},
                logging.WARNING:    {'c': u' \x1b[35;1mWARNING\x1b[0m', 'p': 'WARNING'},
                logging.INFO:       {'c': u'    \x1b[32;1mINFO\x1b[0m', 'p': 'INFO'},
                logging.DEBUG:      {'c': u'   \x1b[36;1mDEBUG\x1b[0m', 'p': 'DEBUG'},
                logging.CRITICAL:   {'c': u'\x1b[43;1m\x1b[31;1mCRITICAL\x1b[49\x1b[0m', 'p': 'CRITICAL'}
                }

logging.captureWarnings(True)
cLogger = logging.getLogger('XDM.Console')
fLogger = logging.getLogger('XDM.File')
cLogger.setLevel(logging.INFO)
fLogger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
# this is now done in XDM.py to use the data dir
# fh = logging.handlers.RotatingFileHandler('xdm.log', maxBytes=10 * 1024 * 1024, backupCount=5)
# create console handler with a higher log level
ch = logging.StreamHandler()

# add the handlers to logger

cLogger.addHandler(ch)
# fLogger.addHandler(fh)
""" at some point i want the cherrypy stuff logged
cpLogger = logging.getLogger('cherrypy')
cph = logging.StreamHandler()
formatter = logging.Formatter('%(levelname)s| %(asctime)s: %(message)s ')
cph.setFormatter(formatter)
cpLogger.addHandler(cph)
"""

# http://stackoverflow.com/questions/2203424/python-how-to-retrieve-class-information-from-a-frame-object
def get_class_from_frame(fr):
    args, _, _, value_dict = inspect.getargvalues(fr)
    # we check the first parameter for the frame function is
    # named 'self'
    if len(args) and args[0] == 'self':
        # in that case, 'self' will be referenced in value_dict
        instance = value_dict.get('self', None)
        if instance:
            # return its class
            return getattr(instance, '__class__', None)
    # return None otherwise
    return None

class StructuredMessage(object):
    def __init__(self, lvl, message, calframe, **kwargs):
        self.lvl = lvl
        self.message = message
        self.calframe = calframe
        self.kwargs = kwargs
        self.time = datetime.datetime.now()

    def console(self):
        return u'%s| %s: %s' % (lvlNames[self.lvl]['c'],
                                self.time,
                                self.message)

    def __str__(self):
        def _json(time, lvl, message, calframe, kwargs={}):
            return json.dumps({'time': time,
                               'lvl': lvlNames[lvl]['p'],
                               'msg': message,
                               'caller': {'file': calframe[2][1], 'line': calframe[2][2], 'fn': calframe[2][3]},
                               'data': kwargs},
                               cls=MyEncoder)

        try:
            return _json(self.time, self.lvl, self.message, self.calframe, self.kwargs)
        except TypeError:
            return _json(self.time, self.lvl, self.message, self.calframe)

class LogWrapper():

    _logLineCache = []

    def _log(self, lvl, msg, censor=None, **kwargs):
        if xdm.common.STARTOPTIONS is None or (not xdm.common.STARTOPTIONS.dev):
            if type(censor) == tuple:
                for s in censor:
                    msg = msg.replace(s, '##censored##')
            elif type(censor) == dict:
                for value, name in censor.items():
                    msg = msg.replace(value, '##%s##' % name)
            elif type(censor) == str:
                msg = msg.replace(censor, '##censored##')

        if (xdm.common.SYSTEM is None or (xdm.common.SYSTEM.c.censor_xdm_dir)) and xdm.APP_PATH:
            msg = msg.replace(xdm.APP_PATH, '##xdm_path##')

        curframe = inspect.currentframe()
        calframe = inspect.getouterframes(curframe, 0)
        sm = StructuredMessage(lvl, msg, calframe, **kwargs)
        try:
            cLogger.log(lvl, sm.console())
            _line = u'%s' % sm
            fLogger.log(lvl, _line)
        except (UnicodeDecodeError, UnicodeEncodeError):
            return
        self._logLineCache.append(_line)
        if len(self._logLineCache) > LOGLINECACHESIZE:
            self._logLineCache = self._logLineCache[1:]
        if lvl in (logging.WARNING, logging.ERROR):
            callerClass = get_class_from_frame(calframe[2][0])
            # was the error/warning send by a notifier ?
            if callerClass and callerClass.__bases__ and callerClass.__bases__[0] is not None and 'Notifier' == callerClass.__bases__[0].__name__:
                sm = StructuredMessage(logging.ERROR, 'Error while sending an error message with a notifier %s' % callerClass, calframe, **kwargs)
                cLogger.log(lvl, sm.console())
                fLogger.log(lvl, sm)
            else:
                self.debug('sending %s with notifiers' % lvlNames[lvl]['p'])
                for n in xdm.common.PM.N:
                    if (n.c.on_warning and lvl == logging.WARNING) or (n.c.on_error and lvl == logging.ERROR):
                        n.sendMessage('%s: %s' % (lvlNames[lvl]['p'], msg))

    def error(self, msg, censor=None, **kwargs):
        tb = traceback.format_exc()
        msg = '%s\nTraceback:\n%s' % (msg, tb)
        self._log(logging.ERROR, msg, censor=censor, **kwargs)
        return msg

    def info(self, msg, censor=None, **kwargs):
        self._log(logging.INFO, msg, censor=censor, **kwargs)
        return msg

    def warning(self, msg, censor=None, **kwargs):
        self._log(logging.WARNING, msg, censor=censor, **kwargs)
        return msg

    def debug(self, msg, censor=None, **kwargs):
        self._log(logging.DEBUG, msg, censor=censor, **kwargs)
        return msg

    def critical(self, msg, censor=None, **kwargs):
        self._log(logging.CRITICAL, msg, censor=censor, **kwargs)
        return msg

    def __call__(self, msg, censor=None, **kwargs):
        self._log(logging.DEBUG, msg, censor=censor, **kwargs)
        return msg

    def getEntries(self, entries=10):
        if entries > len(self._logLineCache):
            f = open(xdm.LOGPATH, 'r')
            try:
                logLinesStr = tail(f, entries)
            finally:
                f.close()
        else:
            logLinesStr = self._logLineCache
        return [{
            'data': json.loads(l),
            'raw': json.dumps(json.loads(l), indent=4),
            'id': 'AA%s' % hash(l)} for l in logLinesStr]

log = LogWrapper()

# http://stackoverflow.com/a/13790289/729059
def tail(f, lines=1, _buffer=4098):
    """Tail a file and get X lines from the end"""
    # place holder for the lines found
    lines_found = []

    # block counter will be multiplied by buffer
    # to get the block size from the end
    block_counter = -1

    lines = int(lines) # make sure we get a int !!
    # loop until we find X lines
    while len(lines_found) < lines:
        try:
            f.seek(block_counter * _buffer, os.SEEK_END)
        except IOError: # either file is too small, or too many lines requested
            f.seek(0)
            lines_found = f.readlines()
            break

        lines_found = f.readlines()

        # we found enough lines, get out
        if len(lines_found) > lines:
            break

        # decrement the block counter to get the
        # next X bytes
        block_counter -= 1

    return lines_found[-lines:]

__all__ = ['log']


import matplotlib.pyplot as plt
from shapely.geometry import MultiPoint, mapping, asShape
import json, math
from itertools import combinations
import pandas as pd
from area import area
import numpy as np

def readPopulation(state):
    data_ = dict()

    df = pd.read_csv('Tower Data/county_population.csv')
    statePops = df[df['STATE'] == int(state)]

    for ind in statePops.index:
        countyID = int(statePops['COUNTY'][ind])
        population = int(statePops['POPESTIMATE2019'][ind])

        if countyID < 10:
            countyID = "00"+str(countyID)
        elif countyID < 100:
            countyID = "0"+str(countyID)
        else:
            countyID = str(countyID)

        if countyID == "000":
            continue

        data_[countyID] = population
    return data_

def readPopDensity(state):

    data = dict()
    data_ = dict()
    with open("./Tower Data/county-boundaries.json","r") as f:
        data = json.load(f)

    df = pd.read_csv('./Tower Data/county_population.csv')
    statePops = df[df['STATE'] == int(state)]

    for ind in statePops.index:
        countyID = int(statePops['COUNTY'][ind])
        population = int(statePops['POPESTIMATE2019'][ind])

        if countyID < 10:
            countyID = "00"+str(countyID)
        elif countyID < 100:
            countyID = "0"+str(countyID)
        else:
            countyID = str(countyID)

        if countyID == "000":
            continue

        area = 0
        for county in data["features"]:
            if countyID == county["properties"]["COUNTY"] and state == county["properties"]["STATE"]:
                area = county['properties']['CENSUSAREA']

        data_[countyID] = population / area
    return data_

def areaFor2Points(p1, p2):
    earthRadious = 3959
    dlat = np.deg2rad(p2[1] - p1[1])
    dlong = np.deg2rad(p2[0] - p1[0])
    a = (np.sin(dlat / 2)) ** 2 + np.cos(np.deg2rad(p1[1])) * np.cos(np.deg2rad(p2[1])) * ((np.sin(dlong / 2)) ** 2)
    b = 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
    b *= earthRadious
    b /= 2

    p = b*math.tan(math.radians(22.5))

    return 2*(p*b)

def getCoverage(basestations,CENSUSAREA):

    if(len(basestations) == 0):
        cArea = 0
    elif (len(basestations) == 1):
        cArea = 70
    elif (len(basestations) == 2):
        cArea = areaFor2Points(basestations[0]["location"],basestations[1]["location"])
    else:

        convex_hull = MultiPoint([bs["location"] for bs in basestations]).convex_hull
        cArea = area(mapping(convex_hull))/2590000
        cArea = min(cArea*1.25,CENSUSAREA)

    return cArea/CENSUSAREA

def readCoverageDiff(state,isp1,isp2):
    data1 = dict()
    data2 = dict()

    with open("./Tower Data/States/{}/{}.json".format(isp1,state), "r") as f:
        data = json.load(f)
        for k,v in data.items():
            data1[k] = getCoverage(v["basestations"],v["properties"]["CENSUSAREA"])

    with open("./Tower Data/States/{}/{}.json".format(isp2,state), "r") as f:
        data = json.load(f)
        for k,v in data.items():
            data2[k] = getCoverage(v["basestations"],v["properties"]["CENSUSAREA"])

    return {isp1:data1, isp2:data2}

def benefit_popdensity(state):

    carriers = ['VERIZON', 'T_MOBILE', 'SPRINT', 'AT_T']
    carrier_combinations = combinations(carriers,2)

    pop_density = readPopDensity(state)

    for pair in carrier_combinations:

        benefits = dict()
        with open('./Results/Experiments/County Gain/{}_{}.json'.format(pair[0],pair[1])) as f:
            benefits = json.load(f)

        d1 = {'x':[], 'y':[]}
        d2 = {'x':[], 'y':[]}

        for county in benefits[pair[0]]:
            d1['x'].append(pop_density[county["county"]])
            d1['y'].append(county["benefit"])

        for county in benefits[pair[1]]:
            d2['x'].append(pop_density[county["county"]])
            d2['y'].append(county["benefit"])

        plt.figure(figsize=(20,5))
        plt.xscale("log")
        plt.plot(d1['x'],d1['y'],'.',label=pair[0])
        plt.plot(d2['x'],d2['y'],'x',label=pair[1])
        plt.legend()

        plt.show()
        # break

def removeOutliers(klist, avglatlist):
    # klist = ['1', '2', '3', '4', '5', '6', '7', '8', '4000']
    # avglatlist = ['1', '2', '3', '4', '5', '6', '7', '8', '9']

    klist_np = np.array(klist).astype(np.float)
    avglatlist_np = np.array(avglatlist).astype(np.float)

    klist_filtered = klist_np[(abs(klist_np - np.mean(klist_np))) < (np.std(klist_np))]
    avglatlist_filtered = avglatlist_np[(abs(klist_np - np.mean(klist_np))) < (np.std(klist_np))]

    return klist_filtered,avglatlist_filtered

def benefit_coveragediff(state):

    carriers = ['VERIZON', 'T_MOBILE', 'SPRINT', 'AT_T']
    carrier_combinations = combinations(carriers,2)

    for pair in carrier_combinations:

        coverage_diff = readCoverageDiff(state,pair[0],pair[1])
        benefits = dict()
        with open('./Results/Experiments/County Gain/{}_{}.json'.format(pair[0],pair[1])) as f:
            benefits = json.load(f)

        d1 = {'x':[], 'y':[]}
        d2 = {'x':[], 'y':[]}

        for county in benefits[pair[0]]:
            coverage = abs(coverage_diff[pair[0]][county["county"]] - coverage_diff[pair[1]][county["county"]])
            d1['x'].append(coverage)
            d1['y'].append(county["benefit"])

        for county in benefits[pair[1]]:
            coverage = abs(coverage_diff[pair[0]][county["county"]] - coverage_diff[pair[1]][county["county"]])
            d2['x'].append(coverage)
            d2['y'].append(county["benefit"])

        d1['x'], d1['y'] = removeOutliers(d1['x'], d1['y'])
        d2['x'], d2['y'] = removeOutliers(d2['x'], d2['y'])

        plt.figure(figsize=(20,5))
        plt.xscale("log")
        plt.plot(d1['x'],d1['y'],'.',label=pair[0])
        plt.plot(d2['x'],d2['y'],'x',label=pair[1])
        plt.legend()

        plt.show()

def combinedGraph(state):

    carriers = ['VERIZON', 'T_MOBILE', 'SPRINT', 'AT_T']
    carrier_combinations = combinations(carriers,2)

    for pair in carrier_combinations:

        coverage_diff = readCoverageDiff(state,pair[0],pair[1])
        pop_density = readPopDensity(state)
        benefits = dict()
        with open('./Results/Experiments/County Gain/{}_{}.json'.format(pair[0],pair[1])) as f:
            benefits = json.load(f)

        d1 = {'x':[], 'y':[]}
        d2 = {'x':[], 'y':[]}

        for county in benefits[pair[0]]:
            coverage = abs(coverage_diff[pair[0]][county["county"]] - coverage_diff[pair[1]][county["county"]])
            d1['x'].append(abs(coverage - pop_density[county["county"]]))
            d1['y'].append(county["benefit"])

        for county in benefits[pair[1]]:
            coverage = abs(coverage_diff[pair[0]][county["county"]] - coverage_diff[pair[1]][county["county"]])
            d2['x'].append(abs(coverage - pop_density[county["county"]]))
            d2['y'].append(county["benefit"])

        d1['x'], d1['y'] = removeOutliers(d1['x'], d1['y'])
        d2['x'], d2['y'] = removeOutliers(d2['x'], d2['y'])

        plt.figure(figsize=(20,5))
        plt.xscale("log")
        plt.plot(d1['x'],d1['y'],'.',label=pair[0])
        plt.plot(d2['x'],d2['y'],'x',label=pair[1])
        plt.legend()

        plt.show()

def getBsCount(state):
    import os
    bsCount = {'VERIZON':{}, 'AT_T':{}, 'T_MOBILE':{}, 'SPRINT':{}}

    for root, dirs, files in os.walk("./Tower Data/States/"):

        for d in dirs:

            with open("./Tower Data/States/{}/{}.json".format(d,state),"r") as f:
                data = json.load(f)
                for k,v in data.items():

                    bsCount[d][k] = len(v["basestations"])
    return bsCount

def benefit_bsLoad(state):

    carriers = ['VERIZON', 'T_MOBILE', 'SPRINT', 'AT_T']
    carrier_combinations = combinations(carriers,2)

    bsCount = getBsCount(state)
    # population = readPopulation(state)
    population = readPopDensity(state)

    for pair in carrier_combinations:

        benefits = dict()
        with open('./Results/Experiments/County Gain/{}_{}.json'.format(pair[0],pair[1])) as f:
            benefits = json.load(f)

        d1 = {'x':[], 'y':[]}
        d2 = {'x':[], 'y':[]}

        for county in benefits[pair[0]]:

            d1['x'].append(population[county["county"]] / max(1,bsCount[pair[0]][county["county"]]))
            # d1['x'].append(bsCount[pair[0]][county["county"]] / population[county["county"]])
            d1['y'].append(county["benefit"])

        for county in benefits[pair[1]]:

            d2['x'].append(population[county["county"]] / max(1,bsCount[pair[1]][county["county"]]))
            # d2['x'].append(bsCount[pair[1]][county["county"]] / population[county["county"]])
            d2['y'].append(county["benefit"])

        d1['x'], d1['y'] = removeOutliers(d1['x'], d1['y'])
        d2['x'], d2['y'] = removeOutliers(d2['x'], d2['y'])

        plt.figure(figsize=(20,5))
        plt.xscale("log")
        plt.plot(d1['x'],d1['y'],'.',label=pair[0])
        plt.plot(d2['x'],d2['y'],'x',label=pair[1])
        plt.legend()

        plt.show()

def logTransform(axes):

    for j, axis in enumerate(axes):

        m = min(axis)
        # if m<1:
        t = 1-m
        for i, value in enumerate(axis):
            axes[j][i] += t

    return axes

# def overlapGraph():

def plotCombinedGraph(state):

    carriers = ['VERIZON', 'T_MOBILE', 'SPRINT', 'AT_T']
    labels = ['Verizon', 'T-Mobile', 'Sprint', 'At&t']
    carrier_combinations = combinations(carriers,2)
    labels = list(combinations(labels,2))

    from plot import scatterPlot
    # scatterPlot(xs=[df["AT_T Coverage"]], ys=[df["AT_T BS Count"]], cs=[df["AT_T CSAT Gain"]], zs=[df["AT_T BS Load"]], labels=["At&t"])
    for i, pair in enumerate(carrier_combinations):
        df = pd.read_csv("./Results/Experiments/County Gain/CSVs/{}_{}.csv".format(pair[0], pair[1]))

        bsCount = [df["{} BS Count".format(pair[0])], df["{} BS Count".format(pair[1])]]
        bsLoad = [df["{} BS Load".format(pair[0])], df["{} BS Load".format(pair[1])]]
        bsLoad_ = [df["{} BS Load".format(pair[0])]+(1-min(df["{} BS Load".format(pair[0])])), df["{} BS Load".format(pair[1])]+(1-min(df["{} BS Load".format(pair[1])]))]

        # bsLoad = [df["BS Load"], df["BS Load"]]
        # bsLoad_ = [df["BS Load"]+(1-min(df["BS Load"])), df["BS Load"]+(1-min(df["BS Load"]))]

        bsLoadD = [df["{} BS Load (Density)".format(pair[0])], df["{} BS Load".format(pair[1])]]
        CSATgain = [df["{} CSAT Gain".format(pair[0])], df["{} CSAT Gain".format(pair[1])]]
        CSATRgain_ = [df["{} CSAT RGain".format(pair[0])]+(1-min(df["{} CSAT RGain".format(pair[0])])), df["{} CSAT RGain".format(pair[1])]+(1-min(df["{} CSAT RGain".format(pair[1])]))]
        CSATRgain = [df["{} CSAT RGain".format(pair[0])], df["{} CSAT RGain".format(pair[1])]]

        coverageGain = [df["{} Coverage Area Gain".format(pair[0])], df["{} Coverage Area Gain".format(pair[1])]]
        popDensity = [df["Pop Density"], df["Pop Density"]]
        pop = [df["Population"], df["Population"]]
        covg = [df["{} Coverage".format(pair[0])], df["{} Coverage".format(pair[1])]]
        covgDiff = [df["Coverage Diff"], df["Coverage Diff"]]

        y = [[abs(y1 - y2) for (y1, y2) in zip(df["{} CSAT RGain".format(pair[0])], df["{} CSAT RGain".format(pair[1])])]]
        y_ = [[i+(1-min(y[0])) for i in y[0]]]

        x = [covgDiff[0]]

        # zeroLine = 1-min(df["{} CSAT RGain".format(pair[0])])
        # xlabel = r"Coverage Area Gain ($km^2$)"
        ylabel = r"Relative CSAT Gain (%)"
        zlabel = "CSAT Gain"
        # cs = [df["{} CSAT Gain".format(pair[0])], df["{} CSAT Gain".format(pair[1])]]

        # xlabel = r"Coverage Area Score Difference"
        # scatterPlot(
        #     xs=covgDiff,
        #     ys=CSATRgain,
        #     title="",
        #     labels=labels[i],
        #     xlabel=xlabel,
        #     ylabel=ylabel,
        #     zlabel=zlabel,
        #     fname="./Results/Figs/Experiments/County Gain/covgDiff_{}_{}.pdf".format(pair[0],pair[1]),
        #     show=False)

        xlabel = r"Coverage Area Gain ($km^2$)"
        scatterPlot(
            xs=coverageGain,
            ys=CSATRgain, title="",
            labels=labels[i],
            xlabel=xlabel,
            ylabel=ylabel,
            zlabel=zlabel,
            fname="./Results/Figs/Experiments/County Gain/covgGain_{}_{}.pdf".format(pair[0],pair[1]),
            show=False)

        # overlapGraph(covgDiff, CSATRgain_)

# benefit_coveragediff("48")
# benefit_popdensity("48")
# combinedGraph("48")
# benefit_bsLoad("48")
plotCombinedGraph("48")


import csv
import random
import pickle

from itertools import combinations

import numpy as np

from matplotlib import pyplot as plt
from svmutil import (
    svm_parameter,
    svm_predict,
    svm_problem,
    svm_train,
)

from common import plot_confusion, miss_rate, accuracy

DATA = "data/mnist/"

def read_data(fn):
    y, x = [], []
    with open(DATA + fn + ".csv") as f:
        for row in csv.reader(f, delimiter=','):
            y.append(int(row[-1]))
            x.append([int(n) for n in row[:-1]])
    return y, x

def svm_convert_data(fn):
    """
    Convert our data set into a format that libsvm can read.
    """

    with open(DATA + fn + "-svm", "w") as out:
        for y, x in zip(*read_data(fn)):
            out.write("%d " % y)
            out.write(" ".join(
                ["%d:%d" % (i, x) for i, x in enumerate(x) if x])
            )
            out.write("\n")

def normalize(data):
    """Scale down features to range 0-1, for faster convergence."""
    X = np.asarray(data)

    mn = np.min(X, axis=0)
    mx = np.max(X, axis=0)

    with np.errstate(divide='ignore', invalid='ignore'):
        X = (X - mn) / (mx - mn)
        X = np.nan_to_num(X, copy=False)

    return X

# This is the part a of the assignment
def pegasos(X, y, lmbd=1):
    """
    Pegasos: Primal Estimated sub-GrAdient SOlver for SVM.

    A batch SGD algorithm to find parameters w, b in SVM.

    X: Design matrix (training data's features)
    y: Labels (should be +1 / -1)

    Differences from the Pegasos paper:

    lambda is fixed to 1
    k is 1/C
    """

    m, n = X.shape

    # Batch size
    r = 100
    C = 1.0

    # Initial guess of W
    # What if this gets changed?
    W = np.zeros(n)
    b = 0

    iters = 0
    converged = False

    while not converged:

        iters += 1

        # Because this is stochastic descent
        # we decrease eta as we go ahead
        eta = 1 / iters

        # Find indices of elements in this batch
        batch = np.array(random.sample(range(m), r))
        Xb, yb = X[batch], y[batch]

        # Find examples in this batch for which T < 1
        # Are these points the "support vectors" ?
        T = yb * ((W @ Xb.T) + b)
        Tl1 = np.where(T < 1)

        Wp = W

        W = (1 - eta) * W + eta * C * np.sum(yb[Tl1] * Xb[Tl1].T, axis=1)
        b = b + eta * C * np.sum(yb[Tl1])

        # Convergence criteria
        if all(abs(Wp - W) <= 10**-3):
            converged = True

        # Prevent us from going in infinite loops
        if iters >= 10000:
            print("Abrupt Stop")
            break

    return W, b

def pegasos_train(train_x, train_y):
    """Build 1vs1 SVM classifiers using Pegasos."""

    classifiers = {}
    for classes in combinations(set(train_y), 2):

        # Let's call the first class positive
        # and the other negative
        pos, neg = classes

        # print("Training classifier between classes %d & %d" % (pos, neg))

        # Find examples of these classes
        Xpos = train_x[np.where(train_y == pos)]
        Xneg = train_x[np.where(train_y == neg)]
        X = np.concatenate((Xpos, Xneg))

        # The data contains classes from 1 - 10
        # but SVMs deal with +1 / -1
        y = np.array([1] * len(Xpos) + [-1] * len(Xneg))

        # Fit a classifier and store the parameters
        classifiers[classes] = pegasos(X, y)

    return classifiers

def pegasos_predict(data, classifiers):
    """Make predictions on data using 1v1 classifiers."""

    # Make predictions on test set
    predictions = []

    # Iterate over training set
    for x in data:

        # Prediction for this particular example
        p = []

        # Pass each example to all classifiers
        for classes, params in classifiers.items():
            pos, neg = classes
            W, b = params

            if W.T @ x + b > 0:
                p.append(pos)
            # In case of ties, predict digit with bigger value
            elif W.T @ x + b == 0:
                p.append(max(pos, neg))
            else:
                p.append(neg)

        # Find the class with the most count
        predictions.append(max(p, key=p.count))

    return predictions

def part_b():
    print("\n--- Part B ---\n")

    print("Reading Data")
    train_y, train_x = read_data("train")
    test_y, test_x = read_data("test")

    print("Normalizing")
    # train_x = np.asarray(train_x)
    # test_x = np.asarray(test_x)
    train_x = normalize(train_x)
    test_x = normalize(test_x)
    train_y = np.asarray(train_y)
    test_y = np.asarray(test_y)

    # Build 10_C_2 classifiers
    print("Building Classifiers")
    classifiers = pegasos_train(train_x, train_y)

    with open("models/svm-model-1", "wb") as m:
        pickle.dump(classifiers, m)

    print("Finding Accuracy")
    predictions_test = pegasos_predict(test_x, classifiers)
    acc = accuracy(test_y.tolist(), predictions_test)
    print("Testing Accuracy: %.2f" % (acc * 100))

    predictions_train = pegasos_predict(train_x, classifiers)
    acc = accuracy(train_y.tolist(), predictions_train)
    print("Training Accuracy: %.2f" % (acc * 100))

def part_c():
    print("\n--- Part C ---\n")

    print("Reading Data")
    train_y, train_x = read_data("train")
    test_y, test_x = read_data("test")

    print("Normalizing")
    train_x = normalize(train_x).tolist()
    test_x = normalize(test_x).tolist()

    problem = svm_problem(train_y, train_x)
    params = svm_parameter("-q -s 0 -c 1")

    # Timing calculations
    print("Training SVM (linear kernel)")
    params.parse_options("-t 0")
    model = svm_train(problem, params)

    _, p_acc, _ = svm_predict(test_y, test_x, model)
    print("Accuracy: ", p_acc)

    print("Training SVM (gaussian kernel)")
    params.parse_options("-t 2 -g 0.05")
    model = svm_train(problem, params)

    _, p_acc, _ = svm_predict(test_y, test_x, model)
    print("Accuracy: ", p_acc)

def part_d():
    print("\n--- Part D ---\n")

    print("Reading Data")
    train_y, train_x = read_data("train")
    test_y, test_x = read_data("test")

    print("Normalizing")
    train_x = normalize(train_x)
    test_x = normalize(test_x)

    problem = svm_problem(train_y, train_x)
    params = "-q -s 0 -t 2 -g 0.05"

    results = []
    for c in [10 ** -5, 10 ** -3, 1, 5, 10]:

        c = " -c %f " % c
        print("10-fold CV using" + c)
        cv_acc = svm_train(problem, params + c + "-v 10")

        print("On test data using" + c)
        model = svm_train(problem, params + c)
        _, test_acc, _ = svm_predict(test_y, test_x, model)
        print("C, Accuracy: ", c, cv_acc, test_acc)

        results.append((c, cv_acc, test_acc[0]))

def part_d_2():
    cvals = [0.00001, 0.001, 1, 5, 10]

    # These values were found by running the libsvm CLI tools
    c_acc = [71.59, 71.59, 97.355, 97.455, 97.455]
    t_acc = [72.11, 72.11, 97.23, 97.29, 97.29]

    c_line, = plt.plot(cvals, c_acc, label="Avg. Acc. after 10 fold cross-validation.",
                       linestyle='-', color='r', marker='x')

    t_line, = plt.plot(cvals, t_acc, label="Acc. on Test Set",
                       linestyle='-', color='b', marker='o')

    plt.legend(handles=[c_line, t_line])

    plt.xscale('log')
    plt.xlabel("C")
    plt.ylabel("Accuracy")
    plt.title("Effect of varying value of C on accuracy of Gaussian kernel SVM")
    plt.savefig("part_d_2.png")
    plt.close()

def part_e():

    def imshow(arr, name):
        """Display image from array of values."""
        arr = np.asarray(arr)
        plt.imshow(arr.reshape((28, 28)), cmap=plt.get_cmap('gray'))
        plt.axis('off')
        plt.savefig("miss/%s.png" % name, bbox_inches='tight', pad_inches=0)
        plt.close()

    actual, train_x = read_data("test")

    # These labels were generated by running the
    predicted = np.loadtxt(
        "models/gaussian-test-labels-5", dtype="int").tolist()
    plot_confusion(actual, predicted, range(10), "Gaussian SVM (C=5)")

    predicted = np.loadtxt(
        "models/gaussian-test-labels-10", dtype="int").tolist()
    plot_confusion(actual, predicted, range(10), "Gaussian SVM (C=10)")

    # Find the class that is the hardest to classify using misclassification rate
    mr = miss_rate(actual, predicted)
    print(mr)

    # Examples that actually belong to class 8 but we predict them to be in class 9
    # Found while making the confusion matrix in make_confusion
    a9_p8 = [151, 241, 448, 1107, 2406, 6081, 6091, 6112, 6157, 6168, 617]
    a7_p2 = [810, 1226, 1283, 1754, 1941, 2016, 2325, 2607, 3767,
             4690, 4837, 5887, 7432, 8316, 9009, 9015, 9019, 9024, 9036, 9045]

    # Let us plot these to see what they are
    for idx in a9_p8:
        imshow(train_x[idx], "a9_p8_ex_%d" % idx)

    for idx in a7_p2:
        imshow(train_x[idx], "a7_p2_ex_%d" % idx)

if __name__ == '__main__':

    # In part a we just have to implement pegasos

    part_b()

    # Convert data to a format that libsvm can recognize
    # svm_convert_data("test")
    # svm_convert_data("train")

    # I wrote these parts first and only later found out that we could
    # directly use the C programs provided with libsvm: svm-train etc.
    # Those programs are highly optimized and have very low memory footprint.

    # Look in libsvm.sh on how to call those functions

    # part_c()
    # part_d()

    # part_d_2()

    # part_e()


import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
from ...contextagg import LocalAttenModule, PSPModule
from torch.nn import BatchNorm2d, BatchNorm1d
from .gcpa_gald import FAM
from ...encoders import res2net50_v1b_26w_4s, hardnet
from .gcpanet import ResNet

class GCPAPSP2Net(nn.Module):
    def __init__(self):
        super(GCPAPSP2Net, self).__init__()

        self.hardnet = hardnet(arch=68)

        self.fam45 = FAM(640, 256, 1024)
        self.fam34 = FAM(320, 256, 256)
        self.fam23 = FAM(128, 256, 256)

        self.linear5 = nn.Conv2d(1024, 1, kernel_size=3, stride=1, padding=1)
        self.linear4 = nn.Conv2d(256, 1, kernel_size=3, stride=1, padding=1)
        self.linear3 = nn.Conv2d(256, 1, kernel_size=3, stride=1, padding=1)
        self.linear2 = nn.Conv2d(256, 1, kernel_size=3, stride=1, padding=1)

        inplanes = 1024
        interplanes = 256
        self.conva = nn.Sequential(
            nn.Conv2d(inplanes, interplanes, 3, padding=1, bias=False),
            BatchNorm2d(interplanes),
            nn.ReLU(interplanes),
        )
        self.long_relation = PSPModule(inplanes, interplanes)
        self.local_attention_4 = LocalAttenModule(interplanes)
        self.local_attention_3 = LocalAttenModule(interplanes)
        self.local_attention_2 = LocalAttenModule(interplanes)

    def forward(self, x):
        hardnetout = self.hardnet(x)
        out2 = hardnetout[0]  # [24, 128, 88, 88]
        out3 = hardnetout[1]  # [24, 320, 44, 44]
        out4 = hardnetout[2]  # [24, 640, 22, 22]
        out5 = hardnetout[3]  # [24, 1024, 11, 11]

        out5_c = self.long_relation(out5)  # bs, 256, 11, 11

        # GCF
        out4_c = self.local_attention_4(out5_c)  # bs, 256, 11, 11
        out3_c = self.local_attention_3(out5_c)  # bs, 256, 11, 11
        out2_c = self.local_attention_2(out5_c)  # bs, 256, 11, 11

        # out
        out4 = self.fam45(out4, out4_c, out5)
        out3 = self.fam34(out3, out3_c ,out4)
        out2 = self.fam23(out2, out2_c, out3)
        # we use bilinear interpolation instead of transpose convolution
        out5 = F.interpolate(self.linear5(out5), size=x.size()[2:], mode="bilinear")
        out4 = F.interpolate(self.linear4(out4), size=x.size()[2:], mode="bilinear")
        out3 = F.interpolate(self.linear3(out3), size=x.size()[2:], mode="bilinear")
        out2 = F.interpolate(self.linear2(out2), size=x.size()[2:], mode="bilinear")
        return out5, out4, out3, out2