# Store information for site, weather
from configobj import ConfigObj
import datetime, logging, ipdb
import ephem, time, math, os, sys, json, urllib2
class site:

    def __init__(self,site_name, night, configfile=''):

        self.name = site_name

        #set appropriate parameter based on aqawan_num
        #create configuration file object
        configObj = ConfigObj(configfile)

        try:
            siteconfig = configObj[self.name]
        except:
            print('ERROR accessing ', self.name, ".",
                self.name, " was not found in the configuration file", configfile)
            return

        self.latitude = siteconfig['Setup']['LATITUDE']
        self.longitude = siteconfig['Setup']['LONGITUDE']
        self.elevation = float(siteconfig['Setup']['ELEVATION'])
        self.enclosures = siteconfig['Setup']['ENCLOSURES']

        self.currentStatusFile = 'current_' + site_name + '.log'
        self.observing = True
        self.weather = -1
        self.startNightTime = -1
        self.night = night

        # touch a file in the current directory to enable cloud override
        self.cloudOverride = os.path.isfile('cloudOverride.txt')
        self.sunOverride = os.path.isfile('sunOverride.txt')

        logger_name = siteconfig['Setup']['LOGNAME']
        log_file = 'logs/' + night + '/' + siteconfig['Setup']['LOGFILE']

        self.obs = ephem.Observer()
        self.obs.lat = ephem.degrees(str(self.latitude)) # N
        self.obs.lon = ephem.degrees(str(self.longitude)) # E
        self.obs.elevation = self.elevation # meters

    def sunrise(self, horizon=0):

        self.obs.horizon = str(horizon)
        sunrise = self.obs.next_rising(ephem.Sun(), start=self.startNightTime, use_center=True).datetime()
        return sunrise

    def sunset(self, horizon=0):

        self.obs.horizon = str(horizon)
        sunset = self.obs.next_setting(ephem.Sun(), start=self.startNightTime, use_center=True).datetime()
        return sunset


import os, requests, json, dateparser, time, yaml, boto3, slack, random
from typing import List
from pathlib import Path
from pprint import pprint
from traceback import format_exc

from missing_data import get_missing_responses, get_missing_users

BATCH_SIZE = 500

wootric_session = requests.Session()
ACCESS_TOKEN = ''
CLIENT_ID = os.getenv('WOOTRIC_CLIENT_ID')
CLIENT_SECRET = os.getenv('WOOTRIC_CLIENT_SECRET')
BASE_URL = 'https://api.wootric.com'

stitch_session = requests.Session()
STITCH_CLIENT_ID = os.getenv('STITCH_CLIENT_ID')
STITCH_TOKEN = os.getenv('STITCH_TOKEN')
STITCH_BASE_URL = 'https://api.stitchdata.com'
stitch_session.headers = {'Authorization': f'Bearer {STITCH_TOKEN}', 'Content-Type': 'application/json'}

BUCKET = os.getenv('AWS_BUCKET')

os.environ['AWS_ACCESS_KEY_ID'] = os.getenv('AWS_ACCESS_KEY_ID_', os.getenv('AWS_ACCESS_KEY_ID')) # lambda doesn't allow this reserved var
os.environ['AWS_SECRET_ACCESS_KEY'] = os.getenv('AWS_SECRET_ACCESS_KEY_', os.getenv('AWS_SECRET_ACCESS_KEY')) # lambda doesn't allow this reserved var
os.environ['AWS_SESSION_TOKEN'] = '' # lambda provides this reserved var during execution, need to set blank

STATE_KEY = 'wootric.state.json'

s3 = boto3.resource("s3").Bucket(BUCKET)

slack_client = slack.WebhookClient(url=os.getenv('SLACK_WH_TOKEN'))

# init state
state = dict(
  end_users=1420070400,
  responses=1420070400,
  declines=1420070400,
)

def get_access_token():
  global ACCESS_TOKEN
  url = f'{BASE_URL}/oauth/token'
  payload = dict(
    grant_type='client_credentials',
    client_id=CLIENT_ID,
    client_secret=CLIENT_SECRET,
  )
  resp = wootric_session.post(url, payload)
  data : dict = resp.json()
  ACCESS_TOKEN = data.get('access_token')
  if not ACCESS_TOKEN:
    raise Exception('did not find access_token')
  wootric_session.headers = dict(Authorization=f'Bearer {ACCESS_TOKEN}')

def wootric_response(user_id: str, response_id: str):
  url = f'{BASE_URL}/v1/end_users/{user_id}/responses/{response_id}'
  print(url)
  resp = wootric_session.get(url)
  return resp.json()

def wootric_user(user_id: str):
  url = f'{BASE_URL}/v1/end_users/{user_id}'
  print(url)
  resp = wootric_session.get(url)
  return resp.json()

def wootric_request(object_name: str, date_key: str, **params):
  url = f'{BASE_URL}/v1/{object_name}'
  req = requests.models.PreparedRequest()
  date_val = state[object_name]

  # put random limit because seems some get missed. an attempt to randomize sort anchors
  limit = random.randint(5,29)

  params[f"{date_key.replace('_at', '')}[gte]"] = date_val - 1
  page = 0

  all = []
  while True:
    page += 1
    if page > limit: break
    params['page'] = page
    req.prepare_url(url, params)
    print(req.url)
    try:
      resp = wootric_session.get(req.url)
      if resp is None:
        raise Exception(f'Response is for: {req.url}')
      elif not resp.ok:
        raise Exception(f'\n\nHTTP Status Code {resp.status_code} for {req.url}: \n{resp.text}')
    except Exception:
      raise Exception(f'Error for {req.url}.\n\n{format_exc()}')
    data = resp.json()
    if len(data) == 0:
      break
    all += data

  return all

def send_batch(object_name: str, schema: dict, keys: List[str], records: List[dict]):

  is_datetime = lambda k: schema['properties'].get(k, {}).get('format') == 'date-time'

  messages = []
  for record in records:
    rec = dict(
      action='upsert',
      sequence=int(time.time_ns() / 1000),
      data={},
    )
    for k, v in record.items():
      k = k.replace('.', '_')
      v = (dateparser.parse(str(v))).isoformat() if v and is_datetime(k) else v
      rec['data'][k] = v
    messages.append(rec)

  payload = dict(
    table_name=object_name,
    key_names=keys,
    schema=schema,
    messages=messages,
  )

  # with open('payload.json', 'w') as file:
  #   json.dump(payload, file)

  url = f'{STITCH_BASE_URL}/v2/import/batch'
  resp = stitch_session.post(url, json.dumps(payload))

  data : dict = resp.json()
  print(data)

  status = data.get('status')
  if status != 'OK':
    pprint(dict(status_code=resp.status_code))
    resp.raise_for_status()
  else:
    print(f'pushed {len(records)} records to "{object_name}"')

def load_state():
  global state
  state = json.loads(s3.Object(key=STATE_KEY).get()["Body"].read().decode('utf-8'))

  # re-run for past 3 days, an attempt to fill in any holes
  for k in state:
    state[k] = state.get(k, 1420070400) - 3*24*60*60

def save_state():
  global state
  s3.Object(key=STATE_KEY).put(Body=json.dumps(state))
  print(json.dumps(state))

def run(event, context):
  global state

  try:
    # load wootric access token
    get_access_token()

    # load state
    load_state()

  except Exception as E:
    slack_client.send(text=f"Error occurred for Wootric-Stitch Integration:\n{format_exc()}")
    raise E

  config_file = Path('config.yaml')
  with config_file.open() as file:
    object_configs = yaml.load(file)

  errors = []
  for object_name, object_config in object_configs.items():
    records : List[dict] = []

    try:
      print(f'Loading {object_name}')
      while True:
        date_key = object_config['date_key']
        params = object_config['params']
        schema = object_config['schema']
        keys = object_config['keys']

        data : List[dict] = wootric_request(object_name, date_key, **params)

        if len(data) == 0:
          if len(records) == 0:
            break
        else:
          records += data

        send_batch(object_name, schema, keys, records)

        record = records[-1]
        if date_key not in record:
          raise Exception(f'no datekey: {date_key}')

        records = []

        date_val = dateparser.parse(record[date_key])
        ts_val = int(date_val.timestamp())
        if date_val and ts_val > state[object_name]:
          state[object_name] = ts_val
          save_state()
        else:
          break

    except Exception as E:
      errors.append(format_exc())
    finally:
      save_state()

  # Missing users START
  # seems users are missing even with using gte. Gets the IDs from database
  try:
    users = []
    for row in get_missing_users():
      users += [wootric_user(row.end_user_id)]

    response_config = object_configs.get('end_users')
    send_batch('end_users', response_config['schema'], response_config['keys'], users)
  except Exception as E:
      errors.append(format_exc())
  # Missing users END

  # Missing responses START
  # seems some responses are missing even with using gte. Gets the IDs from database
  try:
    responses = []
    for row in get_missing_responses():
      responses += [wootric_response(row.user_id, row.last_response__id)]

    response_config = object_configs.get('responses')
    send_batch('responses', response_config['schema'], response_config['keys'], responses)
  except Exception as E:
      errors.append(format_exc())
  # Missing responses END

  if len(errors) > 0:
    e = '\n\n'.join(errors)
    slack_client.send(text=f'Error occurred for Wootric-Stitch Integration:\n{e}')
    raise Exception(e)

# run(None, None)


import datetime

from dagster import (
    PresetDefinition,
    daily_schedule,
    hourly_schedule,
    monthly_schedule,
    weekly_schedule,
)

# start_hourly_schedule

@hourly_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_time=datetime.time(hour=0, minute=25),
    execution_timezone="US/Central",
)
def my_hourly_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m-%d %H")}}}}

# end_hourly_schedule

# start_daily_schedule

@daily_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_time=datetime.time(hour=9, minute=0),
    execution_timezone="US/Central",
)
def my_daily_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m-%d")}}}}

# end_daily_schedule

# start_weekly_schedule

@weekly_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_day_of_week=1,  # Monday
    execution_timezone="US/Central",
)
def my_weekly_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m-%d")}}}}

# end_weekly_schedule

# start_monthly_schedule

@monthly_schedule(
    pipeline_name="my_pipeline",
    start_date=datetime.datetime(2020, 1, 1),
    execution_timezone="US/Central",
    execution_day_of_month=15,
    execution_time=datetime.time(hour=9, minute=0),
)
def my_monthly_schedule(date):
    return {"solids": {"process_data_for_date": {"config": {"date": date.strftime("%Y-%m")}}}}

# end_monthly_schedule

preset = PresetDefinition("test_preset", mode="basic", run_config={})

# start_preset

@daily_schedule(
    start_date=datetime.datetime(2020, 1, 1),
    pipeline_name="my_pipeline",
    solid_selection=preset.solid_selection,
    mode=preset.mode,
    tags_fn_for_date=lambda _: preset.tags,
)
def my_preset_schedule(_date):
    return preset.run_config

# end_preset

# start_modified_preset

@daily_schedule(
    start_date=datetime.datetime(2020, 1, 1),
    pipeline_name="my_pipeline",
    solid_selection=preset.solid_selection,
    mode=preset.mode,
    tags_fn_for_date=lambda _: preset.tags,
)
def my_modified_preset_schedule(date):
    modified_run_config = preset.run_config.copy()
    modified_run_config["date"] = date.strftime("%Y-%m-%d")
    return modified_run_config

# end_modified_preset


from pathlib import Path
from medpy.io import save, load
import dicom2nifti
import os
import click
import pandas as pd
import dask.dataframe as dd
import numpy as np
from typing import Dict, List, Optional, Tuple

from luna.common.utils import cli_runner
from luna.common.custom_logger import init_logger
from luna.common.utils import generate_uuid

logger = init_logger()

_params_ = [('raw_data_path', str), ('mapping_csv_path', str), ('output_dir', str),
            ('scan_table_path', str), ('npartitions', int), ('subset', bool)]

def find_z_nbound(dim: Tuple) -> Tuple[int, int]:
    """
    Return z, number of series bound

    Args:
        dim (Tuple): shape of the 4D bound series.
    """
    # axial (x, z, nbound, y)
    if dim[0] == dim[3]:
        return dim[1], dim[2]
    # sagittal (x, y, nbound, z)
    if dim[0] == dim[1]:
        return dim[3], dim[2]

def subset_bound(src_path: str, output_path: str, index: int):
    """
    Subset bound series and return the output file path.

    Args:
        src_path (str): path to the bound series
        output_path (str): path to save the subset output
        index (int): index to pull out from the bound 4D scan
    """
    try:
        file_path = src_path.split(':')[-1]
        data, header = load(file_path)
        subset = data[:, :, index, :]
        # re-arrange the array
        subset = np.swapaxes(np.swapaxes(subset, 1, 2), 0, 1)
        save(subset, output_path, header)
        return output_path

    except Exception as err:
        logger.error(output_path, err)
        return None

def subset_series(dim: str, src_path: str) -> str:
    """
    Subset scans if the scan is a bound series (with 4 dimensions).

    Args:
        dim (str): shape of the scan
        src_path (str): path to the scan

    Returns:
        str: path to the subset output
    """
    dim = eval(dim)
    posix_path = Path(src_path)

    if len(dim) == 4:
        # axial (x, z, n_bound_series, y)
        # sag (x, y, n_bound_series, z)
        z, nbound = find_z_nbound(dim)

        # if 3 series are bound, assume there are three post contrasts bound
        if nbound == 3:
            index = 0
        # if more than 3 series are bound, assume the first one is pre
        elif nbound > 3:
            index = 1

        output_path = os.path.join(str(posix_path.parent), f"subset_{index}_" + str(posix_path.name))
        if not os.path.exists(output_path):
            subset_bound(src_path, output_path, index)
            logger.info("Saved ", output_path)

    return output_path

def dicom_to_nifti(row: pd.DataFrame, raw_data_path: str, output_dir: str, subset=False)\
        -> pd.DataFrame:
    """
    Convert dicoms to nii.gz format

    Args:
        row (pd.DataFrame): dataframe row with path, accession_number columns
        raw_data_path (str): path to dicom files
        output_dir (str): path to save scans
        subset (Optional, boolean): True to extract first post contrast in bound series

    Returns:
        pd.DataFrame: row populated with record_uuid, dim, path, subset_path
    """
    accession_number = row.accession_number
    series_number = row.series_number

    dicom_dir = f"{raw_data_path}/{accession_number}/SCANS/{series_number}/DICOM"
    scan_dir = f"{output_dir}/{accession_number}"

    logger.info(scan_dir)
    os.makedirs(scan_dir, exist_ok=True)

    logger.info(f"Processing : {dicom_dir}")
    try:
        scan_file = None
        for fn in os.listdir(scan_dir):
            if fn.endswith("nii.gz") and fn.startswith(str(series_number)):
                scan_file = fn
                break
        if not scan_file:
            dicom2nifti.convert_directory(dicom_dir, scan_dir, reorient=False)

        print(os.listdir(scan_dir))
        for fn in os.listdir(scan_dir):
            if fn.endswith("nii.gz") and fn.startswith(str(series_number)):
                scan_file = fn
                break

        scan_file_path = os.path.join(scan_dir, scan_file)
        data, header = load(scan_file_path)

        row["record_uuid"] = generate_uuid(scan_file_path, ["SCAN"])
        row["path"] = scan_file_path
        row["dim"] = str(data.shape)

        if len(data.shape) == 4 and subset:
            row["subset_path"] = subset_series(row.dim, row.path)

    except Exception as err:
        logger.error(f"Failed {accession_number}: {err}")

    return row

@click.command()
@click.option('-r', '--raw_data_path', help="path to raw data. e.g /data/radiology/project_name/dicoms",
              type=click.Path())
@click.option('-c', '--mapping_csv_path', help="csv with AccessionNumber, SeriesNumber columns",
              type=click.Path())
@click.option('-o', '--output_dir', help="path to scan output folder")
@click.option('-t', '--scan_table_path', help="path to save scan table")
@click.option('-s', '--subset', help="extract first post contrast in bound series", is_flag=True)
@click.option('-n', '--npartitions', help="npartitions for parallelization", default=20, show_default=True)
@click.option('-m', '--method_param_path', help='path to a metadata json/yaml file with method parameters to reproduce results',
              type=click.Path())
def cli(**cli_kwargs):
    """
    Convert dicoms to nii.gz format and optionally subset bound series.

    dicom2nifti is used to convert bound series into 4D volumes.
    For more simple dicom to scan function, see luna.radiology.cli.dicom_to_itk.
    """
    cli_runner(cli_kwargs, _params_, generate_scan_table)

def generate_scan_table(raw_data_path: str, mapping_csv_path: str, output_dir: str,
                        scan_table_path: str, subset=False, npartitions=20):
    """
    Convert dicoms to nii.gz format and optionally subset bound series.
    Save scan table and method params.

    Args:
        dicom_parquet_path (str): path to dicom parquet table
        mapping_csv_path (str): csv with AccessionNumber, SeriesNumber columns
        output_dir (str): path to scan output folder
        scan_table_path (str): path to save scan table
        subset (Optional, boolean): True to extract first post contrast in bound series
        npartitions (Optional, int): number of partitions for parallelization. Default is 20
    """
    input_params = locals()

    # load accession/series mapping csv
    scan_map = pd.read_csv(mapping_csv_path)
    scan_map = scan_map[['AccessionNumber', 'SeriesNumber']]
    df = scan_map \
        .rename({'AccessionNumber': 'accession_number', 'SeriesNumber': 'series_number'}, axis=1) \
        .astype(str)

    # convert to nii.gz
    df["create_ts"] = pd.Timestamp.now()
    df["dim"] = ""
    df["path"] = ""
    df["record_uuid"] = ""
    df["subset_path"] = ""

    ddf = dd.from_pandas(df, npartitions=npartitions)
    df = ddf.apply(lambda x: dicom_to_nifti(x, raw_data_path, output_dir, subset=subset), axis=1,
                   meta=ddf).compute()
    logger.info(df)

    # save table as parquet
    df = df.replace("", np.nan)
    df = df.dropna(subset=["record_uuid"])
    df.to_parquet(scan_table_path)

    return {
        'table_path': scan_table_path,
        'n_records': len(df)
    }

if __name__ == "__main__":
    cli()


import torch as T
from .module_fn import fit_loop
from ..callbacks.metrics import (MetricsAccumulator,
                                 MetricsCalculator,
                                 MetricsExecutor)
from ..callbacks.loggers import (ProgressLogger,
                                 LoggerExecutor)
from ..callbacks.schedulers import (SchedulersExecutor)
from ..callbacks import Executor
from .type_utils import (_is_list, _is_tuple, _is_dict, _check_metric, _is_str,
                         _is_optimizer_type, _is_optimizer_object, _get_optimizer,
                         _parse_optim_args)

class Module(T.nn.Module):
    def __init__(self, *args, **kwargs):
        super(Module, self).__init__(*args, **kwargs)

        self.metrics = 0
        self.metric_cb = 0

        self.logger_cb = 0
        self.sheduler_cb = 0

        self.loss_fn = 0
        self.optimizer = 0
        self.device = 'cpu'

    def init_weights(self, *args, **kwargs):
        pass

    def set_criterion(self, criterion):
        self.loss_fn = criterion

    def set_metrics(self, metrics):
        self.metrics = dict()

        if metrics is None:
            return

        if _is_list(metrics) or _is_tuple(metrics):
            for m in metrics:
                self.metrics[m.__class__.__name__] = _check_metric(m)
        elif _is_dict(metrics):
            for n, m in metrics.items():
                self.metrics[n] = _check_metric(m)

        self.metric_cb = MetricsExecutor([
            MetricsCalculator(self.metrics),
            MetricsAccumulator(self.metrics)
        ])

    def set_optimizer(self, optimizer, **kwargs):
        opt_type = None
        if _is_optimizer_object(optimizer):
            self.optimizer = optimizer
            return
        elif _is_optimizer_type(optimizer):
            opt_type = optimizer
        else:
            opt_type = _get_optimizer(optimizer)

        args = _parse_optim_args(opt_type, **kwargs)

        if 'params' in args:
            self.optimizer = opt_type(**args)
        else:
            args.update(params=self.parameters())
            self.optimizer = opt_type(**args)

    def set_schedulers(self, schedulers):
        self.scheduler_cb = SchedulersExecutor(schedulers, self)

    def compile(self, criterion, optimizer='adam', metrics=None, schedulers=None, **kwargs):
        if metrics is not None:
            self.set_metrics(metrics)

        self.set_optimizer(optimizer, **kwargs)
        self.set_criterion(criterion)

        if schedulers is not None:
            self.set_schedulers(schedulers)

        self.logger_cb = LoggerExecutor([
            ProgressLogger()
        ])

        self.callbacks = Executor(self.metric_cb.callbacks +
                                  self.logger_cb.callbacks +
                                  self.scheduler_cb.callbacks)

        self.device = T.device("cuda:0" if T.cuda.is_available() else "cpu")

    def fit(self, dataset, nb_epochs):
        # print(self)
        fit_loop(model=self,
                 train_loader=dataset.train,
                 val_loader=dataset.val,
                 nb_epochs=nb_epochs,
                 device=self.device,
                 metrics=self.metrics,
                 callbacks=self.callbacks)

    def evaluate(self, val_loader):
        pass

    def save_weights(self, filename):
        T.save(self.state_dict(), filename)

    def save_model(self, filename):
        T.save(self, filename)


#!/usr/bin/env python

import os
import sys
import math
import spice
from collections import OrderedDict as OD
import unittest

class TestSMAP(unittest.TestCase):

  def setUp(self):
    mydir = os.path.dirname(__file__)
    self.kernels = [ os.path.join( mydir, i.strip() ) for i in """
### Lines in this string that start with '#' are comments
### The other lines in this string are paths to kernels,
###   relative to the parent directory of this Python script, __file__
###
kernels/naif0010.tls
kernels/smap_v00.tf
kernels/pck00009.tpc
kernels/spk_drm239_WithBurn-full.bsp
kernels/smap_test.bsp
###
### kernels/SMAP_ref_150529_180529.bsp
###
### That commented SMAP SPK kernel, or one like it, should be under this URL:
###
###   http://naif.jpl.nasa.gov/pub/naif/SMAP/kernels/spk/
###
###
""".strip().split('\n') if i[0]!='#' ]

    ### Load default kernels (string variable "kernels" above")
    for kernel in self.kernels: spice.furnsh( kernel )

  def tearDown(self):
    ### Unload any kernels
    for kernel in self.kernels: spice.unload( kernel )

  def test_smap(self):

    target = 'SMAP'

    et0 = spice.utc2et( '2016-06-01T12:00:00' )

    dpr = spice.dpr()

    a,b,c = [spice.gdpool('BODY399_RADII',i,1)[1] for i in range(3)]

    ods = []

    for deltatime in [0.1 * i for i in range(1080)]:
      et = et0 + deltatime
      stSmap,lsSmap = spice.spkezr( target, et, 'IAU_EARTH', 'NONE', 'EARTH' )
      posn, veloc = stSmap[:3], stSmap[3:]
      stSun,lsSun = spice.spkezr( 'SUN', et0, 'IAU_EARTH', 'LT', 'EARTH' )
      mtx = spice.pxform( 'SMAP_REFLECTOR', 'IAU_EARTH', et)
      boreEbf = spice.mxv( mtx, [1.0,0,0] )
      point = spice.surfpt( posn, boreEbf, a, b, c)
      rsurfpt,lon,lat = spice.reclat( point )
      utc = spice.et2utc( et, 'ISOC', 3 )

      ods += [ OD(deltatime=deltatime,posn=posn,veloc=veloc,boreEbf=boreEbf
                 ,utc=utc,point=point,rsurfpt=rsurfpt
                 ,rsmap=spice.vnorm(posn),lat=lat,lon=lon
                 ,raynge=spice.vnorm(spice.vsub(point,posn))
                 ,sunsep=spice.vsep( spice.ucrss(posn,veloc), stSun[:3] )
                 )
             ]

    try:
      ### Moved matplotlib import to here so test runs to here at least
      from matplotlib import pyplot as plt
      plt.figure(1)
      keys = 'lat lon raynge'.split()
      secs = [od['deltatime'] for od in ods]
      for idx in range(len(keys)):
        scal = 1.0 if keys[idx] in 'rsurfpt rsmap raynge sunsep rp ecc t0 mu a P eccmean amean Pmean'.split() else dpr
        ordinate = [od[keys[idx]]*scal for od in ods]
        plt.subplot( 221+idx )
        plt.plot( secs, ordinate )
        plt.plot( secs, ordinate, '.')
        plt.title( keys[idx] )
        plt.ylabel( '%s%s' % (keys[idx],'' if scal==1.0 else ', deg') )
        if idx>1: plt.xlabel( 'T-T0, s' )

      abscissa = [od['lon']*dpr for od in ods]
      ordinate = [od['lat']*dpr for od in ods]
      plt.subplot( 221+idx+1 )
      plt.title( 'lon vs. lat' )
      plt.plot( abscissa, ordinate )
      plt.xlabel( 'lon, deg' )
      plt.ylabel( 'lat, deg' )
      plt.show()

    except:
      print( "Bypassed, or failed, matplotlib tests" )

if __name__=="__main__":
  unittest.main()


import cv2
import os
def format_image(img_path, size, nb_channels):
    """
    Load img with opencv and reshape
    """
    img = cv2.imread(img_path)

    img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)

    if nb_channels == 1:
        img = np.expand_dims(img, -1)
    img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA)
    img = np.expand_dims(img, 0).transpose(0, 3, 1, 2)

    return img
def build_HDF5(jpeg_dir, nb_channels, size=224):
    """
    Gather the data in a single HDF5 file.
    """

    # Put train data in HDF5
    file_name = os.path.basename(jpeg_dir.rstrip("/"))
    hdf5_file = os.path.join(data_dir, "%s_data.h5" % file_name)
    with h5py.File(hdf5_file, "w") as hfw:
        for dset_type in ["train", "test", "val"]:

            list_img = [img for img in Path(jpeg_dir).glob('%s/img_*.jpg' % dset_type)]
            list_img = [str(img) for img in list_img]
           # list_img.extend(list(Path(jpeg_dir).glob('%s/*.png' % dset_type)))
            list_img = list(map(str, list_img))
            list_img = np.array(list_img)

            list_img1 = [img for img in Path(jpeg_dir).glob('%s/dep_*.jpg' % dset_type)]
            list_img1 = [str(img) for img in list_img1]
           # list_img.extend(list(Path(jpeg_dir).glob('%s/*.png' % dset_type)))
            list_img1 = list(map(str, list_img1))
            list_img1 = np.array(list_img1)

            data_full = hfw.create_dataset("%s_data_full" % dset_type,
                                           (0, nb_channels, size, size),
                                           maxshape=(None, 3, size, size),
                                           dtype=np.uint8)

            data_sketch = hfw.create_dataset("%s_data_sketch" % dset_type,
                                             (0, nb_channels, size, size),
                                             maxshape=(None, 3, size, size),
                                             dtype=np.uint8)

            num_files = len(list_img)
            chunk_size = 100
            num_chunks = num_files / chunk_size
            arr_chunks = np.array_split(np.arange(num_files), num_chunks)

            for chunk_idx in tqdm(arr_chunks):

                list_img_path = list_img[chunk_idx].tolist()
                output = parmap.map(format_image, list_img_path, size, nb_channels, pm_parallel=False)

                arr_img_full = np.concatenate([o for o in output], axis=0)

                # Resize HDF5 dataset
                data_full.resize(data_full.shape[0] + arr_img_full.shape[0], axis=0)
                data_full[-arr_img_full.shape[0]:] = arr_img_full.astype(np.uint8)

            num_files = len(list_img1)
            chunk_size = 100
            num_chunks = num_files / chunk_size
            arr_chunks = np.array_split(np.arange(num_files), num_chunks)

            for chunk_idx in tqdm(arr_chunks):
                list_img_path = list_img1[chunk_idx].tolist()
                output = parmap.map(format_image, list_img_path, size, nb_channels, pm_parallel=False)

                arr_img_dep = np.concatenate([o for o in output], axis=0)

                # Resize HDF5 dataset
                data_dep.resize(data_dep.shape[0] + arr_img_dep.shape[0], axis=0)
                data_dep[-arr_img_dep.shape[0]:] = arr_img_dep.astype(np.uint8)


from django.urls import path

from . import views

app_name="panels"

urlpatterns = [
    path('<slug:slug>/<article_type>/', views.PanelArticleView.as_view(), name='article_detail'),
    path('<slug:slug>/', views.PanelDetailView.as_view(), name='panel_detail'),
]


from styx_msgs.msg import TrafficLight
import tensorflow as tf
import numpy as np
import datetime

from PIL import Image

class TLClassifierObjDetect(object):
    """
    This class uses the trained model from the TensorFlow Object Detection API.
    https://github.com/tensorflow/models/tree/master/research/object_detection

    For the Capstone Project, a Single Shot Detector with lightweight MobileNet
    Backbone was trained on the Bosch Traffic Light Dataset, LISA Traffic Light Dataset and
    Udacity Simulator and Site images.

    The Inference Code was adapted from:
    https://github.com/tensorflow/models/blob/master/research/object_detection/inference/detection_inference.py

    """
    def __init__(self, path_to_tensorflow_graph, confidence_thresh):

        # Threshold for detections
        self.detection_threshold = confidence_thresh

        # Create the TensorFlow session in which the graph is loaded
        self.session = tf.Session()

        # Create Tensors for results
        self.num_detections_tensor = None
        self.detected_boxes_tensor = None
        self.detected_scores_tensor = None
        self.detected_labels_tensor = None
        self.image_tensor = None

        # Load the trained and frozen model graph with respective weights
        with self.session.graph.as_default():
            od_graph_def = tf.GraphDef()
            with tf.gfile.GFile(path_to_tensorflow_graph, 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

                g = tf.get_default_graph()

                # Remember all the tensors we will need for inference
                # Most important: input image tensor:
                self.image_tensor = g.get_tensor_by_name('image_tensor:0')

                self.num_detections_tensor = tf.squeeze(g.get_tensor_by_name('num_detections:0'), 0)
                self.num_detections_tensor = tf.cast(self.num_detections_tensor, tf.int32)

                self.detected_boxes_tensor = tf.squeeze(g.get_tensor_by_name('detection_boxes:0'), 0)
                self.detected_boxes_tensor = self.detected_boxes_tensor[:self.num_detections_tensor]

                self.detected_scores_tensor = tf.squeeze(g.get_tensor_by_name('detection_scores:0'), 0)
                self.detected_scores_tensor = self.detected_scores_tensor[:self.num_detections_tensor]

                self.detected_labels_tensor = tf.squeeze(g.get_tensor_by_name('detection_classes:0'), 0)
                self.detected_labels_tensor = tf.cast(self.detected_labels_tensor, tf.int64)
                self.detected_labels_tensor = self.detected_labels_tensor[:self.num_detections_tensor]

    def get_classification(self, image):
        """
        Determines the color of the traffic light in the image by
        using the TensorFlow graph.

        We run the operations that will give us the boxes, scores and labels.
        Then we filter out the most probable scores (> threshold) and use the
        biggest box, since this will be the nearest traffic light.

        The graph will give us the following IDs :
        4: NA
        3: green
        2: yellow
        1: red

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """

        traffic_light_id = 4  # 4 equals to unknown

        id_mapping = {4: TrafficLight.UNKNOWN,
                      3: TrafficLight.GREEN,
                      2: TrafficLight.YELLOW,
                      1: TrafficLight.RED}

        results = []
        with self.session.graph.as_default():
            boxes, scores, labels = self.session.run([self.detected_boxes_tensor,
                                                      self.detected_scores_tensor,
                                                      self.detected_labels_tensor],
                                                     feed_dict={self.image_tensor: image})
            # Filter for probability (score) and classification
            for i, score in enumerate(scores):
                if score > self.detection_threshold and labels[i] != traffic_light_id:
                    results.append({'box': boxes[i],
                                    'score': score,
                                    'id': labels[i]})

        if len(results) > 0:
            # print('Nums: '+str(len(results))+' '+str(results[0]['score'])+ ' ' + str(results[0]['id']))

            # The boxes are encoded as xmin, xmax, ymin, ymax with normalized coordinates [0..1].
            # So lets find just the biggest box and take the traffic light state from it.
            # max_sized_result = max(results, key=lambda bb: (bb['box'][1] - bb['box'][0]) * (bb['box'][3] - bb['box'][2]))
            # traffic_light_id = max_sized_result['id']

            # Better take the best score than the biggest box !
            max_score_result = max(results, key=lambda bb: bb['score'])
            traffic_light_id = max_score_result['id']

        return id_mapping[traffic_light_id]

class TestTLClassifier(object):

    def __init__(self):
        self.detector = TLClassifier()

    def test_classification(self):
        # Load image
        image_path_green = ('light_classification/test_images/green.jpg', TrafficLight.GREEN)
        image_path_yellow = ('light_classification/test_images/yellow.jpg', TrafficLight.YELLOW)
        image_path_red = ('light_classification/test_images/red.jpg', TrafficLight.RED)
        image_path_na = ('light_classification/test_images/NA.jpg', TrafficLight.UNKNOWN)

        for image_path in [image_path_green, image_path_yellow, image_path_red, image_path_na]:
            image = np.asarray(Image.open(image_path[0]))
            image = np.expand_dims(image, 0)
            gt_result = image_path[1]
            pred_result = self.detector.get_classification(image)
            print(image_path[0])
            print('Prediction success: ' + str(gt_result == pred_result))

            if gt_result != pred_result:
                raise Exception('Prediction error.')

    def measure_time(self):
        # Load image
        image_path = 'light_classification/test_images/green.jpg'
        image = np.asarray(Image.open(image_path))
        image = np.expand_dims(image, 0)

        repeats = 25

        t0 = datetime.datetime.now()
        for i in range(repeats):
            _ = self.detector.get_classification(image)

        delta = datetime.datetime.now() - t0
        print('Time per image in ms: ' + str(delta.seconds * 100.0 / float(repeats)))

if __name__ == '__main__':
    tester = TestTLClassifier()
    tester.measure_time()
    tester.test_classification()


from django.test import TestCase

from model_mommy import mommy

from manager.models import Show, Episode, Segment

class ManagerModelsTest(TestCase):

    def test_show_refresh(self):
        valid_rss_url = "http://feeds.gimletmedia.com/hearstartup"
        show = mommy.make(Show, rss_url = valid_rss_url)
        show.refresh()
        episodes = show.episode_set.all()
        self.assertGreater(len(episodes), 0)


from openerp import models
from openerp.report import report_sxw
import datetime
from datetime import date, datetime, timedelta
import dateutil.parser
import time
import calendar
from ..utility.utils import hour_float_to_time as hftt
from time import gmtime, strftime
import pytz

class employee_overtime_work_inspection_report(report_sxw.rml_parse):
    _name = 'report.sisb_hr.employee_overtime_work_inspection'
    def __init__(self, cr, uid, name, context=None):
        if context is None:
            context = {}
        super(employee_overtime_work_inspection_report, self).__init__(cr, uid, name, context = context)
        self.localcontext.update({
            'get_employee_ot': self._get_employee_ot,
            'get_daily_overtime': self._get_daily_overtime,
            'get_section_code': self._get_section_code,
        })

    def _get_section_code(self, company):
        section_code = ''
        if company.school_ids:
            for rec in company.school_ids:
                section_code = '00' + str(rec.sequence) + ' :Section : SISB-' + rec.code
                break
        else:
            section_code = 'Section: SISB'
        return section_code

    def _get_daily_overtime(self, employee_id, date_from , date_to, attendance_id):
        data = []
        print('attendance_id = ', attendance_id)
        emp_obj = self.pool.get('hr.employee')
        emp_id = emp_obj.search(self.cr, self.uid, [('id', '=', employee_id)])
        employee = emp_obj.browse(self.cr, self.uid, emp_id)
        all_day = emp_obj.delta_days(self.cr, self.uid, date_from, date_to)
        for att_id in attendance_id:
            print('att_id = ', att_id)
            self.cr.execute("SELECT name + INTERVAL '8 HOURS', sign_out + INTERVAL '8 HOURS' \
                            FROM hr_attendance \
                            WHERE id = %s \
                            ",(att_id,))
            ot_att = self.cr.fetchall()
            att_obj = self.pool.get('hr.attendance').browse(self.cr, self.uid, att_id)
            sign_in_obj = datetime.strptime(ot_att[0][0], '%Y-%m-%d %H:%M:%S')
            sign_out_obj = datetime.strptime(ot_att[0][1], '%Y-%m-%d %H:%M:%S')
            sign_in = sign_in_obj.strftime('%d/%b/%Y %H:%M')
            sign_out = sign_out_obj.strftime('%d/%b/%Y %H:%M')
            date_sign_in = datetime.strptime(ot_att[0][0], '%Y-%m-%d %H:%M:%S')
            day_name = date_sign_in.strftime('%d/%b/%Y') + ' [' + date_sign_in.strftime('%a') + ']'
            att_obj = self.pool.get('hr.attendance').browse(self.cr, self.uid, att_id)
            schedule = att_obj.shift_id
            tz  = pytz.timezone(schedule.default_timezone)
            tz_time = datetime.now(tz)
            year = date_sign_in.year
            month = date_sign_in.month
            day = date_sign_in.day
            # year = int(datetime.strftime(tz_time, '%Y'))
            # month = int(datetime.strftime(tz_time, '%m'))
            # day = int(datetime.strftime(tz_time, '%d'))
            today = calendar.weekday(year, month, day)
            check_shift =  [r.options for r in schedule.day_of_wewks_ids.filtered(lambda x: int(x.day_of_week) == today)]
            holiday_shift = True if check_shift[0] == 'holiday' else False
            holiday_year = emp_obj.check_holiday(self.cr, self.uid, employee, tz_time)
            day_option = 'N'
            if holiday_shift and not holiday_year:
                day_option = 'H'
            elif (holiday_shift and holiday_year) or (not holiday_shift and holiday_year):
                day_option = 'HD'
            for ot in att_obj.overtime_ids:
                print('ot_end = ', ot.end_ot)
                ot_day = hftt(ot.ot_rounded)
                ot_type = ot.ot_type_id.name
                ot_start = hftt(ot.start_ot)
                ot_end = hftt(ot.end_ot)
                ot_len = hftt(ot.end_ot - ot.start_ot)
                ot_one = hftt(ot.rate_one) if ot.rate_one > 0.00 else ''
                ot_one_half = hftt(ot.rate_one_half) if ot.rate_one_half > 0.00 else ''
                ot_double = hftt(ot.rate_double) if ot.rate_double > 0.00 else ''
                ot_triple = hftt(ot.rate_triple) if ot.rate_triple > 0.00 else ''
                # ot_length = hftt(ot.ot_length)
                result = {
                    'type': ot_type,
                    'date': day_name,
                    'schedule': att_obj.shift_id.name,
                    'day': day_option,
                    'sign_in': sign_in,
                    'sign_out': sign_out,
                    'one': ot_one,
                    'one_half': ot_one_half,
                    'double': ot_double,
                    'triple': ot_triple,
                    'ot_start': ot_start,
                    'ot_end': ot_end,
                    'ot_one': hftt(ot.rate_one),
                    'ot_one_half': hftt(ot.rate_one_half),
                    'ot_double': hftt(ot.rate_double),
                    'ot_triple': hftt(ot.rate_triple),
                    'ot_total': ot_len
                    # 'ot_length': ot_length

                }
                data.append(result)
        if data:
            return data
        else:
            return {}

        # for day in all_day['all_day']:
        #     self.cr.execute("SELECT name + INTERVAL '8 HOURS', sign_out + INTERVAL '8 HOURS' \
        #                     FROM sisb_attendance \
        #                     WHERE name + INTERVAL '8 HOURS' >= %s \
        #                     AND name + INTERVAL '8 HOURS' <= %s \
        #                     AND employee_id = %s\
        #                     AND overtime = true\
        #                     ",(day + ' 00:00:00', day + ' 23:59:59', employee_id))

    def _get_employee_ot(self, form):
        data = []
        emp_obj = self.pool.get('hr.employee')
        date_from = form['date_from']
        print('date_from = ', date_from)
        date_to = form['date_to']
        company = form['company_id']
        all_employee = self.pool.get('hr.employee').search(self.cr, self.uid, [('company_id', '=', company[0]),('active','=', True)])
        all_employee_id = self.pool.get('hr.employee').browse(self.cr, self.uid, all_employee)
        all_day = emp_obj.delta_days(self.cr, self.uid, date_from, date_to)
        for employee in all_employee_id:
            name = ''
            if employee.gender == 'male':
                name = employee.employee_no + ' ' + 'Mr.' + employee.name
            if employee.gender == 'female':
                name = employee.employee_no + ' ' + 'Mrs.' + employee.name
            if not employee.gender:
                name = employee.employee_no + ' ' + employee.name
            print('employee_id = ', employee.id)
            self.cr.execute("SELECT id FROM hr_attendance\
                            WHERE name + INTERVAL '8 HOURS' >= %s\
                            AND name + INTERVAL '8 HOURS' <= %s\
                            AND sign_out IS NOT NULL \
                            AND employee_id = %s \
                            AND overtime = true \
                            ORDER BY name ASC \
                            ",(date_from + ' 00:00:00', date_to + ' 23:59:59', employee.id))
            emp_ot = self.cr.fetchall()
            if not emp_ot:
                continue
            emp_ot_id = []
            total_one = total_one_half = total_double = total_triple = 0.00
            one = one_half = double = triple = ''
            for ot_id in emp_ot:
                if ot_id not in emp_ot_id:
                    emp_ot_id.append(ot_id[0])
            emp_att_id = self.pool.get('hr.attendance').browse(self.cr, self.uid, emp_ot_id)
            for ot_list in emp_att_id:
                for ot in ot_list.overtime_ids:
                    total_one += ot.rate_one
                    total_one_half += ot.rate_one_half
                    total_double += ot.rate_double
                    total_triple += ot.rate_triple
            if total_one > 0.00:
                one = hftt(total_one)
            if total_one_half > 0.00:
                one_half = hftt(total_one_half)
            if total_double > 0.00:
                double = hftt(total_double)
            if total_triple > 0.00:
                triple = hftt(total_triple)

            result = {
                'name': name,
                'ot_one': one,
                'ot_one_half': one_half,
                'ot_double': double,
                'ot_triple': triple,
                'employee_id': employee.id,
                'date_from': date_from,
                'date_to': date_to,
                'attendance_id': emp_ot_id
            }
            data.append(result)
        if data:
            return data
        else:
            return {}

class report_overtime_work_inspection(models.AbstractModel):
    _name = 'report.sisb_hr.employee_overtime_work_inspection'
    _inherit = 'report.abstract_report'
    _template = 'sisb_hr.employee_overtime_work_inspection'
    _wrapped_report_class = employee_overtime_work_inspection_report


'''
'''
from datetime import date
class Employee:
	def __init__(self, name, age, salary, employment_year):
		self.name = name
		self.age = age
		self.salary = salary
		self.employment_year = employment_year

	def get_working_years(self):
		current_year = date.today().year
		working_years = current_year - int(self.employment_year)
		return working_years
		# return date.today().year - self.employment_year

	def __str__(self):
		return f"Name: {self.name}, Age:{self.age}, Salary:{self.salary}, Working Years: {self.get_working_years()}"

class Manager(Employee):
	def __init__(self, name, age, salary, employment_year, bonus):
		super().__init__(name, age, salary, employment_year)
		self.bonus = bonus

	def get_bounus(self):
		return float(self.bonus) * float(self.salary)

	def __str__(self):
		return f"Name: {self.name}, Age:{self.age}, Salary:{self.salary}, Working Years{self.get_working_years()}, Bounus: {self.get_bounus()}"

employees_list = []
managers_list = []

print("Welcome to HR Pro 2020")
print("Options:")
print("\t1. Show Employees")
print("\t2. Show Managers")
print("\t3. Add An Employee")
print("\t4.Add A Manager")
print("\t5.Exit")
print("\t6. Go crazy")
print()
choice = int(input("What would you like to do:"))

while choice != 5:
	if choice == 1:
		for x in employees_list:
			print(x)
	elif choice == 2:
		for y in managers_list:
			print(y)
	elif choice == 3:
		name = input("Name: ")
		age = input("Age: ")
		salary = input("Salary: ")
		employment_year = input("Employment Year: ")
		employee_obj = Employee(name , age, salary, employment_year)
		employees_list.append(employee_obj)

	elif choice == 4:
		name = input("Name: ")
		age = input("Age: ")
		salary = input("Salary: ")
		employment_year = input("Employment Year: ")
		bonus = input("Bounus Percentage:")
		manager_obj = Manager(name , age, salary, employment_year, bonus)
		managers_list.append(manager_obj)
	print("Welcome to HR Pro 2020")
	print("Options:")
	print("\t1. Show Employees")
	print("\t2. Show Managers")
	print("\t3. Add An Employee")
	print("\t4.Add A Manager")
	print("\t5.Exit")
	print("\t6. Go crazy")
	print()
	choice = int(input("What would you like to do:"))


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 14 10:12:36 2019
@author: alexl
"""

import FeatureEngineerer as fe
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn.pipeline import make_pipeline
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import PolynomialFeatures, MinMaxScaler, PowerTransformer

#Supress ALL Warnings
import warnings
warnings.filterwarnings("ignore")

data = pd.read_csv('Data/train.csv', index_col=0, parse_dates = True)
X_pred = pd.read_csv('Data/test.csv', index_col=0, parse_dates = True)
#randomise the order of data
data = data.sample(len(data))

def share_results(a,b):
    iterater = 0
    for i in a:
        print(f'coefficient of {X.columns[iterater]}: {i}')
        iterater +=1
    print(f'constant is: {b}')

def XFeatures(data):
    #----------- Feature Engineering and Selection -----------#
    if 'count' in data.columns:
        X = data.drop(['count','registered','casual'], axis = 1)
    else:
        X = data

    #----------- Add year, month, day, hour to df -----------#
    X = fe.FeatureAdder(X)

    #----------- windspeed = 0 to RFclassifier -----------#
    #X = fe.FillWindspeed(X)

    #----------- buckets -----------#
    X['temp'] = pd.qcut(X['temp'], 3, labels = [0, 1, 2])
    #X = fe.OneHotEncoder(X,'temp').rename(columns = {0:'Cold',1:'Nice',2:'Hot'})

    #----------- one hot encode -----------#
    X = fe.OneHotEncoder(X,'season').rename(columns = {1:'Spring',2:'Summer',3:'Autumn',4:'Winter'})
    X = fe.OneHotEncoder(X,'weekday_hour').rename(columns = {0:'wh0',1:'wh1',2:'wh2',3:'wh3',4:'wh4',5:'wh5',
                        6:'wh6',7:'wh7',8:'wh8',9:'wh9',10:'wh10',11:'wh11',12:'wh12',13:'wh13',14:'wh14',15:'wh15',
                        16:'wh16',17:'wh17',18:'wh18',19:'wh19',20:'wh20',21:'wh21',22:'wh22',23:'wh23',24:'wh24'})

    X = fe.OneHotEncoder(X,'weekend_hour').rename(columns = {0:'we0',1:'we1',2:'we2',3:'we3',4:'we4',5:'we5',
                        6:'we6',7:'we7',8:'we8',9:'we9',10:'we10',11:'we11',12:'we12',13:'we13',14:'we14',15:'we15',
                        16:'we16',17:'we17',18:'we18',19:'we19',20:'we20',21:'we21',22:'we22',23:'we23',24:'we24'})
    X = fe.OneHotEncoder(X,'weather')

    #----------- Feature Selection -----------#
#    print(X.columns)
    X = X.drop([1,'we0','wh0', 'Autumn','atemp','humidity',
            'hour','month','year','day',
            'workingday','holiday'], axis = 1)

    return X

#y = data['count']
y = np.log10(data['count'])

X = XFeatures(data)
X_test = XFeatures(X_pred)

pipeline = make_pipeline(
        PolynomialFeatures(),
        MinMaxScaler(),
        PowerTransformer(copy=True, method = 'yeo-johnson',standardize=True),
        Ridge()
        )

g = GridSearchCV(pipeline, cv = 2, param_grid = {
        'polynomialfeatures__degree':[2],
        'ridge__alpha':[0.01,0.1,0.09,0.11]
        })

g.fit(X,y)

#predict test data
ypred = g.predict(X) #data from training set
y_pred = g.predict(X_test) #kaggle predictions

print(f'the best params are: g.best_params_')
print(f'Model score was: {g.best_score_}')
share_results(g.best_estimator_.named_steps['ridge'].coef_,
              g.best_estimator_.named_steps['ridge'].intercept_)

#re-evaluate negatives to 0
print(f'Sum of ypredictions < 0: {ypred[ypred < 0].sum()}')
print(f'Sum of kaggle ys < 0: {y_pred[y_pred < 0].sum()}')
ypred[ypred < 0] = 0
y_pred[y_pred < 0] = 0

#validate results
fe.Validator(X, y, ypred)

#prediction compare graph
#fe.PredictionCompare(data,ypred)

#unlog and export predictions for Kaggle
y_pred = 10**y_pred
export = pd.DataFrame(y_pred, columns = ['count'], index = X_test.index) ##################
export.to_csv(f'Data/count_predictions2.csv')

#dir(g)
#g.best_params_
#g.best_score_

#Gsearch coef and intercept of best fit
#g.best_estimator_.named_steps['ridge'].coef_
#g.best_estimator_.named_steps['ridge'].intercept_


import pytest
from model.Action import Action

from pysc2.lib.actions import FUNCTIONS, _Functions

class _Command:
    def __init__(self, function):
        self.function = function

class TestAction:
    def test_equal(self):
        act1 = Action()
        act2 = Action()
        act3 = Action()

        act1.addCommand(FUNCTIONS.select_point("select", [1, 2]))
        act2.addCommand(FUNCTIONS.Train_SCV_quick("now"))
        act3.addCommand(FUNCTIONS.select_point("select", [3, 4]))

        assert act1 != act2
        assert act1 == act3

    @pytest.mark.parametrize("test_com, expected_id", [
        ("Command_1", None),
        (FUNCTIONS.Train_SCV_quick("now"), 490),
    ])
    def test_add_command(self, test_com, expected_id):
        act = Action()

        act.addCommand(test_com)

        assert act.commands == [test_com]
        assert act.ids == [expected_id]

    def test_reset(self):
        act = Action()

        act.addCommand("Command_1")
        act.addCommand("Command_2")

        assert act.next() == "Command_1"
        assert act.next() == "Command_2"

        act.reset()
        assert act.next() == "Command_1"

    def test_repr(self):
        act = Action("abc")
        act.addCommand(FUNCTIONS.select_point("select", [1, 2]))

        assert act.__repr__() == "Action<group=abc, ids={}>".format([_Functions['select_point']])


from functools import partial
import numpy as np
from auxiliary import make_individual, to_phenotype, local_search_knowledge_compilation_based
from crossover import clause_crossover_1x, uniform_crossover, matched_uniform_crossover, uniform_clause_crossover, \
    scramble_clause_crossover, avoid_duplicate_clauses_scramble_clause_crossover, smart_clause_crossover_dispatch
from evaluation import evaluate, evaluate_use_infeasibility
from hassle_gen import HassleGen
from mutation import mutate_hardness, mutate_weight, mutate_clause, mutate_to_neighbour,\
    mutate_clause_smart, mutate_clause_smarter

def learn_max_sat_model(n,
                        k,
                        examples,
                        population_size,
                        tournament_size=None,
                        use_crowding=False,
                        crowding_variant="semantic_relative",
                        generations=10,
                        prob_crossover=0.9,
                        crossover_operators=None,
                        mutation_operators=None,
                        use_local_search=False,
                        cutoff_time=None,
                        cutoff_time_flags=None,
                        use_knowledge_compilation_caching=True,
                        knowledge_compilation_variant=4,
                        use_diagram_for_instance_evaluation=True,
                        conjunctive_contexts=0,
                        variable_absence_bias=1,
                        use_infeasibility=False,
                        use_clause_bitvector_cache=True,
                        observers=None):
    """
    Configures HASSLE-GEN as desired and subsequently runs the algorithm.
    :param n: The number of variables
    :param k: The number of constraints
    :param examples: A list of examples, where each example is a context-instance-label tuple
    :param population_size: The population size
    :param tournament_size: The tournament size - only relevant when not using crowding
    :param use_crowding: A Boolean that denotes whether to use crowding
    :param crowding_variant: What crowding variant to use: "semantic_relative", "semantic_absolute", "syntactic". Only
    relevant when use_crowding is True.
    :param generations: The maximum number of generations
    :param prob_crossover: The probability of applying a crossover operator on a pair of parents
    :param crossover_operators: A list of crossover operators to use. Each operator is itself a list of which the first
    element denotes the name of the crossover operator, and optional additional elements are used to configure the
    operator
    :param mutation_operators: A list of mutation operators to use. Each operator is itself a list of which the first
    element denotes the name of the mutation operator, and optional additional elements are used to configure the
    operator.
    :param use_local_search: Whether to apply a HASSLE-SLS local search step to each individual in each iteration
    :param cutoff_time: The maximum number of seconds to run the algorithm for
    :param cutoff_time_flags: A list of strings that denotes what operations should be counted in the runtime of the
    algorithm. Options are: "selection", "crossover", "mutation", "local search" and "evaluation". By default all
    are included.
    :param use_knowledge_compilation_caching: A Boolean that denotes whether to use knowledge-compilation-based caching
    :param knowledge_compilation_variant: What variant of knowledge-compilation-based caching to use. Options are:
    1, 2, 3 and 4. Only relevant when use_knowledge_compilation_caching is True.
    :param use_diagram_for_instance_evaluation: A Boolean that denotes whether to use the knowledge-compiled
    representation not just for computing the optimal value in a context, but also to evaluate the instances themselves.
    Only relevant when use_knowledge_compilation_caching is True.
    :param conjunctive_contexts: A Boolean that denotes whether the contexts occurring in the examples should be
    interpreted as conjunctions. If False, the contexts are interpreted as disjunctions.
    :param variable_absence_bias: A float that can be used to determine the length of constraints in the initial
    population. In initializing a model, in each constraint, each variable can occur as a negative
    literal, as a positive literal, or be absent from the constraint. With variable_absence_bias = 1, any variable
    has the same probability of occurring (either positively or negatively) as being absent in any constraint. With
    variable_absence_bias = 2, any variable is twice as likely to be absent as to occur, in any constraint.
    :param use_infeasibility: A Boolean that denotes whether to make use of the distinction between infeasible and
    suboptimal negative examples. If False, a negative example is simply interpreted as being negative, and no
    distinction between infeasibility and suboptimality is made.
    :param use_clause_bitvector_cache: A Boolean that denotes whether to cache coverage bitvectors computed during
    execution. Relevant when using smart mutation, smarter mutation or smart crossover.
    :param observers: A list of Observer objects to report aggregate data to in every generation.
    :return: A tuple consisting of (i) The best training set accuracy achieved, (ii) the phenotype that achieved said
    training set accuracy and (iii) the runtime of the algorithm (including only the runtime of the operations
    specified in cutoff_time_flags).
    """

    if not use_crowding and tournament_size is None:
        raise ValueError("Specify a tournament size when not using crowding")

    ga = HassleGen(
        partial(make_individual, n, k, variable_absence_bias=variable_absence_bias),
        partial(evaluate, examples=examples, use_knowledge_compilation_caching=use_knowledge_compilation_caching,
                knowledge_compilation_variant=knowledge_compilation_variant,
                use_diagram_for_instance_evaluation=use_diagram_for_instance_evaluation,
                conjunctive_contexts=conjunctive_contexts) if use_infeasibility == False else
        partial(evaluate_use_infeasibility, examples=examples,
                use_knowledge_compilation_caching=use_knowledge_compilation_caching,
                knowledge_compilation_variant=knowledge_compilation_variant,
                use_diagram_for_instance_evaluation=use_diagram_for_instance_evaluation,
                conjunctive_contexts=conjunctive_contexts),
        population_size=population_size,
        prob_crossover=prob_crossover,
        use_crowding=use_crowding,
        crowding_variant=crowding_variant,
        tournament_size=tournament_size
    )

    # Make smart clause mutation and smart clause crossover more efficient by caching
    if use_clause_bitvector_cache:
        clause_bitvector_cache = dict()
    else:
        clause_bitvector_cache = None

    # Add crossover operators
    if crossover_operators is None:
        # Use defaults
        ga.add_crossover(scramble_clause_crossover)
        ga.add_crossover(uniform_crossover)
    else:
        # Use the requested operators
        for crossover_operator in crossover_operators:
            if crossover_operator[0] == "clause_crossover_1x":
                ga.add_crossover(clause_crossover_1x)
            elif crossover_operator[0] == "uniform_crossover":
                ga.add_crossover(uniform_crossover)
            elif crossover_operator[0] == "matched_uniform_crossover":
                ga.add_crossover(matched_uniform_crossover)
            elif crossover_operator[0] == "uniform_clause_crossover":
                ga.add_crossover(uniform_clause_crossover)
            elif crossover_operator[0] == "scramble_clause_crossover":
                ga.add_crossover(scramble_clause_crossover)
            elif crossover_operator[0] == "avoid_duplicate_clauses_scramble_clause_crossover":
                ga.add_crossover(avoid_duplicate_clauses_scramble_clause_crossover)
            elif crossover_operator[0] == "smart_clause_crossover":
                greedy = crossover_operator[1]
                probability_variant = crossover_operator[2]
                temperature = crossover_operator[3]
                ga.add_crossover(partial(smart_clause_crossover_dispatch, examples=examples, greedy=greedy,
                                         probability_variant=probability_variant, temperature=temperature,
                                         clause_bitvector_cache=clause_bitvector_cache,
                                         use_infeasibility=use_infeasibility))
                # With smart clause crossover, 2 parents produce only a single offspring
                ga.single_offspring_per_parent_couple = True
            else:
                raise Exception("A valid crossover operator should be chosen")

    # Add mutation operators
    if mutation_operators is None:
        # Use defaults
        ga.add_mutation(mutate_hardness, trigger_prob=1, inner_prob=0.05)
        ga.add_mutation(mutate_clause, trigger_prob=1, inner_prob=0.05)
        ga.add_mutation(mutate_weight, trigger_prob=1, inner_prob=0.05)
    else:
        # Use the requested operators
        for mutation_operator in mutation_operators:
            if mutation_operator[0] == "mutate_hardness":
                trigger_prob = mutation_operator[1]
                inner_prob = mutation_operator[2]
                ga.add_mutation(partial(mutate_hardness, trigger_prob=trigger_prob, inner_prob=inner_prob))
            elif mutation_operator[0] == "mutate_clause":
                trigger_prob = mutation_operator[1]
                inner_prob = mutation_operator[2]
                ga.add_mutation(partial(mutate_clause, trigger_prob=trigger_prob, inner_prob=inner_prob))
            elif mutation_operator[0] == "mutate_weight":
                trigger_prob = mutation_operator[1]
                inner_prob = mutation_operator[2]
                ga.add_mutation(partial(mutate_weight, trigger_prob=trigger_prob, inner_prob=inner_prob))
            elif mutation_operator[0] == "mutate_to_neighbour":
                trigger_prob = mutation_operator[1]
                contexts = [e[0] for e in examples]
                data = np.array([e[1] for e in examples])
                labels = [e[2] for e in examples]
                boolean_labels = np.array([True if l == 1 else False for l in labels])
                if use_infeasibility:
                    inf = [True if l == -1 else False for l in labels]
                else:
                    inf = None
                ga.add_mutation(partial(mutate_to_neighbour, trigger_prob=trigger_prob, contexts=contexts, data=data,
                                        boolean_labels=boolean_labels, inf=inf))
            elif mutation_operator[0] == "mutate_clause_smart":
                trigger_prob = mutation_operator[1]
                temperature = None
                if len(mutation_operator) > 2:
                    temperature = mutation_operator[2]
                ga.add_mutation(partial(mutate_clause_smart, trigger_prob=trigger_prob, examples=examples,
                                        clause_bitvector_cache=clause_bitvector_cache,
                                        use_infeasibility=use_infeasibility,
                                        temperature=temperature))
            elif mutation_operator[0] == "mutate_clause_smarter":
                trigger_prob = mutation_operator[1]
                temperature = None
                if len(mutation_operator) > 2:
                    temperature = mutation_operator[2]
                ga.add_mutation(partial(mutate_clause_smarter, trigger_prob=trigger_prob, examples=examples,
                                        clause_bitvector_cache=clause_bitvector_cache,
                                        use_infeasibility=use_infeasibility,
                                        temperature=temperature))
            else:
                raise Exception("A valid mutation operator should be chosen")

    # Add method that converts genotype to phenotype representation
    ga.to_phenotype = to_phenotype

    if use_local_search:
        # Perform a local search step to every individual in every generation
        if use_infeasibility:
            contexts = [e[0] for e in examples]
            data = np.array([e[1] for e in examples])
            labels = [e[2] for e in examples]
            inf = [True if l == -1 else False for l in labels]
        else:
            inf = None
        ga.local_search_function = partial(local_search_knowledge_compilation_based, examples,
                                           conjunctive_contexts=conjunctive_contexts, inf=inf)

    if observers is not None:
        # Add observers
        for observer in observers:
            ga.observers.append(observer)

    # Run HASSLE-GEN
    return ga.run(generations, 1, cutoff_time, cutoff_time_flags)


import bpy
from ..utils.geo import get_blf_text_dims
from ..graphics.load import load_image_file
from ..graphics.draw import render_text
from ..window.panel.widget.layout.grid.elements.text import Text_Element
from ..window.panel.widget.layout.grid.elements.background import Background_Element
from ..window.panel.widget.layout.grid.elements.image import Image_Element
from ..window.panel.widget.layout.grid.elements.border import Border_Element

class Fast_UI():

    def __init__(self, db):

        self.db = db
        self.show = True

        # Start up data
        self.start_mouse_pos = None

        # Dims
        self.y_offset_mouse = 60 * self.db.scale_factor
        self.panel_padding = self.db.prefs.ui.Hops_modal_fast_ui_padding * self.db.scale_factor
        self.font_size = 12
        self.cell_padding = 12 #* self.db.scale_factor
        self.text_padding = 8 # * self.db.scale_factor

        # Switches
        self.first_in = True

        # Help
        self.show_help = self.db.prefs.ui.Hops_modal_auto_show_help
        self.show_global_help = False
        if self.show_help == False:
            self.show_help = self.db.prefs.ui.Hops_modal_help_left_open

        # Mods
        self.start_mod_hidden = False
        self.show_mods = self.db.prefs.ui.Hops_modal_auto_show_mods
        if self.show_mods == False:
            self.show_mods = self.db.prefs.ui.Hops_modal_mods_left_open

        self.location_option = 1    # NOTE: 1 = Center, 2 = Follow Mouse, 3 = Stays at mouse Pos

        # Drawing
        self.help = Help(db=self.db)
        self.boxes = []
        self.image = None
        self.mods = Mods(db=self.db)

    def clear(self):

        self.boxes = []

    def build_main(self, win_list=[], help_list=[], image="", mods_list=[], active_mod_name="", mods_label_text="Press M", number_mods=True):

        # Overrides
        self.mods.label_text = mods_label_text
        self.mods.number_mods = number_mods

        if self.start_mouse_pos == None:
            self.start_mouse_pos = self.db.event.mouse_pos

        # Override example in : Adjust Viewport
        if self.start_mod_hidden:
            if self.db.event.m_key_pressed:
                self.show_mods = True
                self.start_mod_hidden = False
            self.show_mods = False

        # Hot key checks
        self.hot_key_toggles()

        # Mods Panels
        self.mods.mod_items = mods_list
        if self.show_mods or self.db.prefs.ui.Hops_modal_mods_left_open:
            if mods_list != []:
                self.mods.active_mod_name = active_mod_name
                self.mods.already_sliced = False

        # Help Panel
        if self.show_help or self.db.prefs.ui.Hops_modal_help_left_open:
            self.help.help_items = help_list

        # Main Panel
        if win_list != []:

            bottom_left = (0, 0)

            prefs_y_offset = self.db.prefs.ui.Hops_modal_fast_ui_main_y_offset

            # Main banner is centered
            if self.location_option == 1:
                width = get_text_width(win_list=win_list, font_size=self.font_size) + 32 * self.db.scale_factor
                bottom_left = ((self.db.event.screen_width * .5) - (width * .5), self.panel_padding + prefs_y_offset)

            # Main banner will follow the mouse
            elif self.location_option == 2:
                bottom_left = (self.db.event.mouse_pos[0], self.db.event.mouse_pos[1] - self.y_offset_mouse)

            # Main banner will follow the mouse
            elif self.location_option == 3:
                bottom_left = (self.start_mouse_pos[0], self.start_mouse_pos[1] - self.y_offset_mouse)

            # Add image box
            bottom_left = self.setup_image_box(image=image, bottom_left=bottom_left)

            # Add text boxes
            self.setup_text_boxes(win_list=win_list, bottom_left=bottom_left)

    def setup_text_boxes(self, win_list=[], bottom_left=(0,0)):

        text_height = get_blf_text_dims(text="SAMPLE", size=self.font_size)[1]

        for item in win_list:

            font_dims = get_blf_text_dims(text=str(item), size=self.font_size)

            # Text box setup
            box = Box(db=self.db)

            # Dims
            box.bottom_left = bottom_left

            box.bottom_right = (
                bottom_left[0] + font_dims[0] + (self.text_padding * 2),
                bottom_left[1])

            box.top_left = (
                bottom_left[0],
                bottom_left[1]  + text_height + (self.text_padding * 2))

            box.top_right = (
                bottom_left[0] + font_dims[0] + (self.text_padding * 2),
                bottom_left[1]  + text_height + (self.text_padding * 2))

            # Text
            box.text = str(item)

            # Build elements
            box.setup(text_height=text_height)

            # Add to the drawing list
            self.boxes.append(box)

            # Add to bottom left
            bottom_left = (bottom_left[0] + self.cell_padding + font_dims[0] + (self.text_padding * 2),
                        bottom_left[1])

    def setup_image_box(self, image="", bottom_left=(0,0)):

        # Add image box
        if image != "":

            if self.image == None:
                self.image = load_image_file(filename=image)

            # Font size sample
            font_dims = get_blf_text_dims(text="SAMPLE", size=self.font_size)

            # Box setup
            box = Box(db=self.db)

            box.using_text = False
            box.image = self.image
            box.force_fit = True

            # Dims
            box.bottom_left = bottom_left

            box.bottom_right = (
                bottom_left[0] + font_dims[1] + (self.text_padding * 2),
                bottom_left[1])

            box.top_left = (
                bottom_left[0],
                bottom_left[1] + font_dims[1] + (self.text_padding * 2))

            box.top_right = (
                bottom_left[0] + font_dims[1] + (self.text_padding * 2),
                bottom_left[1] + font_dims[1] + (self.text_padding * 2))

            # Build elements
            box.setup()

            # Add to the drawing list
            self.boxes.append(box)

            # Add to bottom left
            bottom_left = (bottom_left[0] + self.cell_padding + font_dims[1] + (self.text_padding * 2),
                            bottom_left[1])

        return bottom_left

    def hot_key_toggles(self):

        # Toggle Help
        if self.db.event.h_key_pressed:
            if not self.db.event.shift_pressed:
                self.show_help = not self.show_help
                self.db.prefs.ui.Hops_modal_help_left_open = self.show_help
                self.show_global_help = False
            if self.show_help and self.db.event.shift_pressed:
                self.show_global_help = not self.show_global_help

        # Toggle Mods
        if self.db.event.m_key_pressed == True and not self.db.event.shift_pressed:
            self.show_mods = not self.show_mods
            self.db.prefs.ui.Hops_modal_mods_left_open = self.show_mods

        # Toggle UI Follow
        if self.db.event.accent_grave_pressed == True and not self.db.event.shift_pressed:

            if self.db.prefs.ui.Hops_modal_fast_ui_loc_options == 1:
                self.db.prefs.ui.Hops_modal_fast_ui_loc_options = 2

            elif self.db.prefs.ui.Hops_modal_fast_ui_loc_options == 2:
                self.db.prefs.ui.Hops_modal_fast_ui_loc_options = 3

            elif self.db.prefs.ui.Hops_modal_fast_ui_loc_options == 3:
                self.db.prefs.ui.Hops_modal_fast_ui_loc_options = 1

        self.location_option = self.db.prefs.ui.Hops_modal_fast_ui_loc_options

    def draw(self, context):

        # Main banner
        for box in self.boxes:
            box.draw()

        # Help
        if self.show_help:
            if self.show_global_help:
                self.help.draw(as_global=True)
            else:
                self.help.draw()
        elif self.db.prefs.ui.Hops_modal_help_show_label:
            self.help.draw_label()

        # Mods
        if self.show_mods:
            self.mods.draw(context)
            self.mods.already_sliced = True
        elif self.db.prefs.ui.Hops_modal_mods_show_label:
            self.mods.draw_label(context)

    def destroy(self):

        # Unload images
        if self.image != None:
            try:
                bpy.data.images.remove(self.image)
            except:
                pass

def get_text_width(win_list=[], font_size=12):

    if win_list != []:

        total_characters = 0
        for item in win_list:
            item = str(item)
            total_characters += len(item)

        return get_blf_text_dims(text=("X" * total_characters), size=font_size)[0]

    return 0

class Box():

    def __init__(self, db):

        self.db = db

        # Dims
        self.top_left = (0, 0)
        self.top_right = (0, 0)
        self.bottom_left = (0, 0)
        self.bottom_right = (0, 0)

        # Drawing
        self.using_text = True
        self.text = ""
        self.image = None
        self.text_element = Text_Element()
        self.background_element = Background_Element()
        self.image_element = Image_Element()
        self.border_element = Border_Element()
        self.setup_elements()

    def setup_elements(self):

        self.text_element.db = self.db
        self.background_element.db = self.db
        self.image_element.db = self.db
        self.border_element.db = self.db
        self.border_element.line_width = 1

    def setup(self, text_height=0):

        # Border
        if self.db.prefs.ui.Hops_modal_cell_border:
            self.border_element.top_left = self.top_left
            self.border_element.top_right = self.top_right
            self.border_element.bottom_left = self.bottom_left
            self.border_element.bottom_right = self.bottom_right

        # Background Element
        self.background_element.primary = False
        self.background_element.top_left = self.top_left
        self.background_element.top_right = self.top_right
        self.background_element.bottom_left = self.bottom_left
        self.background_element.bottom_right = self.bottom_right

        # Setup for text
        if self.using_text:

            # Text Element
            self.text_element.text = self.text
            self.text_element.force_fit_text = False
            self.text_element.color_select = 1
            self.text_element.set_y_external = True
            center = self.bottom_left[1] + ((self.top_left[1] - self.bottom_left[1]) * .5)
            center -= text_height * .5
            self.text_element.external_y = center
            self.text_element.top_left = self.top_left
            self.text_element.top_right = self.top_right
            self.text_element.bottom_left = self.bottom_left
            self.text_element.bottom_right = self.bottom_right

        # Setup for image
        else:

            # Image Element
            self.image_element.image = self.image
            self.image_element.force_fit = False
            self.image_element.top_left = self.top_left
            self.image_element.top_right = self.top_right
            self.image_element.bottom_left = self.bottom_left
            self.image_element.bottom_right = self.bottom_right

    def draw(self):

        self.background_element.draw()

        # Border
        if self.db.prefs.ui.Hops_modal_cell_border:
            self.border_element.draw()

        if self.using_text:
            self.text_element.draw()

        else:
            self.image_element.draw()

class Mods():

    def __init__(self, db):

        self.db = db

        # Text Dims
        self.padding_LR = 8 # * self.db.scale_factor
        self.padding_TB = 8 # * self.db.scale_factor
        self.panel_padding = self.db.prefs.ui.Hops_modal_fast_ui_padding * self.db.scale_factor

        # Font
        self.font_size = int(12 * self.db.prefs.ui.Hops_modal_fast_ui_mods_size)

        # Mods list
        self.already_sliced = False
        self.active_mod_name = ""
        self.mod_items = []

        # Label
        self.label_text = "Press M"

        # Options
        self.number_mods = True

        # Drawing
        self.background_element = Background_Element()
        self.border_element = Border_Element()
        self.setup_elements()

    def setup_elements(self):

        self.background_element.db = self.db
        self.background_element.primary = False

        self.border_element.db = self.db
        self.border_element.line_width = 1

    def draw_label(self, context):

        if self.mod_items != []:
            prefs_x_offset = self.db.prefs.ui.Hops_modal_fast_ui_mods_offset[0]
            prefs_y_offset = self.db.prefs.ui.Hops_modal_fast_ui_mods_offset[1]

            text_dims = get_blf_text_dims(text=self.label_text, size=12)
            total_width = text_dims[0] + self.padding_LR
            total_height = text_dims[1] + self.padding_TB
            self.draw_elements(total_width=total_width, total_height=total_height)
            color = self.db.colors.Hops_UI_secondary_text_color

            render_text(text=self.label_text, position=(self.panel_padding + prefs_x_offset, self.panel_padding + self.padding_TB + prefs_y_offset), size=12, color=color)

    def draw(self, context):

        if self.mod_items != []:

            self.slice_mods()

            # Dims
            offset_y = 0
            key_max_width = 0
            desc_max_width = 0
            total_height = 0
            total_width = 0

            # Get text offsets
            largest_item = ""
            index = 0

            for item in self.mod_items:
                key, desc = item[0], item[1]

                # Text dims
                key_dims = get_blf_text_dims(text=key, size=self.font_size)
                desc_dims = get_blf_text_dims(text=desc, size=self.font_size)

                if key_dims[0] > key_max_width:
                    key_max_width = key_dims[0]
                    largest_item = key

                if desc_dims[0] > desc_max_width:
                    desc_max_width = desc_dims[0]

                offset_y = key_dims[1]
                index += 1

            offset_y += self.padding_TB
            total_height = (len(self.mod_items) * offset_y) + self.padding_TB
            key_max_width += self.padding_LR
            desc_max_width += self.padding_LR

            total_width = key_max_width + desc_max_width

            self.draw_elements(total_width, total_height)
            self.draw_text(key_max_width, offset_y, key_dims)

    def slice_mods(self):

        if not self.already_sliced:

            # Number mods
            if self.number_mods:

                numbering = 0
                for item in reversed(self.mod_items):

                    if item[0] == self.active_mod_name:
                        item[0] = str(numbering + 1) + " : " + item[0]
                        self.active_mod_name = item[0]

                    else:
                        item[0] = str(numbering + 1) + " : " + item[0]

                    numbering += 1

            # Get mod index
            mod_index = 0
            for item in self.mod_items:
                if item[0] == self.active_mod_name:
                    break
                else:
                    mod_index += 1

            # Slice
            start_slice = 0
            if mod_index >= self.db.prefs.ui.Hops_modal_mod_count_fast_ui:
                start_slice = mod_index - self.db.prefs.ui.Hops_modal_mod_count_fast_ui

            end_slice = len(self.mod_items)
            if len(self.mod_items) - mod_index >= self.db.prefs.ui.Hops_modal_mod_count_fast_ui:
                end_slice = mod_index + self.db.prefs.ui.Hops_modal_mod_count_fast_ui

            self.mod_items = self.mod_items[start_slice : end_slice]

    def draw_elements(self, total_width, total_height):

        prefs_x_offset = self.db.prefs.ui.Hops_modal_fast_ui_mods_offset[0]
        prefs_y_offset = self.db.prefs.ui.Hops_modal_fast_ui_mods_offset[1]

        # Background
        self.background_element.bottom_left =  (prefs_x_offset + self.panel_padding - self.padding_LR, self.panel_padding + prefs_y_offset)
        self.background_element.top_left =     (prefs_x_offset + self.panel_padding - self.padding_LR, total_height + self.panel_padding + self.padding_TB + prefs_y_offset)
        self.background_element.bottom_right = (prefs_x_offset + self.panel_padding + total_width, self.panel_padding + prefs_y_offset)
        self.background_element.top_right =    (prefs_x_offset + self.panel_padding + total_width, total_height + self.panel_padding + self.padding_TB + prefs_y_offset)
        self.background_element.draw()

        # Border
        if self.db.prefs.ui.Hops_modal_cell_border:
            self.border_element.bottom_left =  (prefs_x_offset + self.panel_padding - self.padding_LR, self.panel_padding + prefs_y_offset)
            self.border_element.top_left =     (prefs_x_offset + self.panel_padding - self.padding_LR, total_height + self.panel_padding + self.padding_TB + prefs_y_offset)
            self.border_element.bottom_right = (prefs_x_offset + self.panel_padding + total_width, self.panel_padding + prefs_y_offset)
            self.border_element.top_right =    (prefs_x_offset + self.panel_padding + total_width, total_height + self.panel_padding + self.padding_TB + prefs_y_offset)
            self.border_element.draw()

    def draw_text(self, key_max_width, offset_y, key_dims):

        prefs_x_offset = self.db.prefs.ui.Hops_modal_fast_ui_mods_offset[0]
        prefs_y_offset = self.db.prefs.ui.Hops_modal_fast_ui_mods_offset[1]

        # Draw text
        for index, item in enumerate(self.mod_items):
            key, desc = item[0], item[1]

            color = (0,0,0,0)
            if key == self.active_mod_name or str(index) == self.active_mod_name:
                color = self.db.colors.Hops_UI_mods_highlight_color
            else:
                color = self.db.colors.Hops_UI_secondary_text_color

            render_text(text=key, position=(prefs_x_offset + self.panel_padding, offset_y + self.panel_padding + prefs_y_offset - self.padding_TB), size=self.font_size, color=color)
            render_text(text=desc, position=(prefs_x_offset + self.panel_padding + key_max_width, offset_y + self.panel_padding + prefs_y_offset - self.padding_TB), size=self.font_size, color=color)

            offset_y += key_dims[1] + self.padding_TB

class Help():

    def __init__(self, db):

        self.db = db

        # Font
        self.font_size = int(12 * db.prefs.ui.Hops_modal_fast_ui_help_size)

        # Items
        self.help_items = []

        # Window BG
        self.window_background = Background_Element()
        self.window_background.db = db
        self.window_background.primary = False

        self.window_border = Border_Element()
        self.window_border.db = db
        self.window_border.line_width = 1

        # Label BG
        self.label_background = Background_Element()
        self.label_background.db = db
        self.label_background.primary = False

        self.label_border = Border_Element()
        self.label_border.db = db
        self.label_border.line_width = 1

        # Dimensions
        self.font_padding = 8 * db.scale_factor
        self.prefs_offset = db.prefs.ui.Hops_modal_fast_ui_help_offset
        self.panel_padding = db.prefs.ui.Hops_modal_fast_ui_padding * db.scale_factor

    def draw_label(self):

        text_dims = get_blf_text_dims(text="Press H", size=12)

        total_width = text_dims[0] + self.font_padding
        total_height = text_dims[1] + self.font_padding

        offset_x = self.db.event.screen_width - total_width - self.panel_padding
        offset_x += self.prefs_offset[0]
        bot_y = self.panel_padding + self.prefs_offset[1]
        top_y = self.panel_padding + self.prefs_offset[1] + total_height + self.font_padding

        if self.db.prefs.ui.Hops_modal_background:
            self.label_background.bottom_left  = (offset_x - self.font_padding, bot_y)
            self.label_background.bottom_right = (offset_x + total_width      , bot_y)
            self.label_background.top_left     = (offset_x - self.font_padding, top_y)
            self.label_background.top_right    = (offset_x + total_width      , top_y)
            self.label_background.draw()

        if self.db.prefs.ui.Hops_modal_cell_border:
            self.label_border.bottom_left  = (offset_x - self.font_padding, bot_y)
            self.label_border.bottom_right = (offset_x + total_width      , bot_y)
            self.label_border.top_left     = (offset_x - self.font_padding, top_y)
            self.label_border.top_right    = (offset_x + total_width      , top_y)
            self.label_border.draw()

        color = self.db.colors.Hops_UI_secondary_text_color
        render_text(text="Press H", position=(offset_x, self.panel_padding + self.font_padding + self.prefs_offset[1]), size=12, color=color)

    def draw(self, as_global=False):

        if not self.help_items: return

        help_items = []
        if as_global:
            if "GLOBAL" not in self.help_items: return
            help_items = self.help_items["GLOBAL"]
        else:
            if "STANDARD" not in self.help_items: return
            help_items = self.help_items["STANDARD"]
        if not help_items:
            help_items = [("Nothing available", "")]

        offset_y = 0
        offset_x = 0
        key_max_width = 0
        desc_max_width = 0
        total_height = 0
        prefs_x_offset = self.db.prefs.ui.Hops_modal_fast_ui_help_offset[0]
        prefs_y_offset = self.db.prefs.ui.Hops_modal_fast_ui_help_offset[1]

        longest_key_string = ""
        longest_val_string = ""

        for item in help_items:
            key, desc = item[0], item[1]

            if len(key) > len(longest_key_string):
                longest_key_string = key

            if len(desc) > len(longest_val_string):
                longest_val_string = desc

        key_dims = get_blf_text_dims(text=longest_key_string, size=self.font_size)
        desc_dims = get_blf_text_dims(text=longest_val_string, size=self.font_size)
        key_max_width = key_dims[0] + self.font_padding
        desc_max_width = desc_dims[0] + self.font_padding
        offset_y = key_dims[1] + self.font_padding

        total_height = (len(help_items) * offset_y) + self.font_padding * 2
        total_width = key_max_width + desc_max_width + self.font_padding

        offset_x = self.db.event.screen_width - total_width - self.panel_padding
        offset_x += prefs_x_offset

        if self.db.prefs.ui.Hops_modal_background:
            self.window_background.bottom_left =  (offset_x - self.font_padding, self.panel_padding + prefs_y_offset)
            self.window_background.bottom_right = (offset_x + total_width, self.panel_padding + prefs_y_offset)
            self.window_background.top_left =     (offset_x - self.font_padding, total_height + self.panel_padding + prefs_y_offset)
            self.window_background.top_right =    (offset_x + total_width, total_height + self.panel_padding + prefs_y_offset)
            self.window_background.draw()

        if self.db.prefs.ui.Hops_modal_cell_border:
            self.window_border.bottom_left =  (offset_x - self.font_padding, self.panel_padding + prefs_y_offset)
            self.window_border.bottom_right = (offset_x + total_width, self.panel_padding + prefs_y_offset)
            self.window_border.top_left =     (offset_x - self.font_padding, total_height + self.panel_padding + prefs_y_offset)
            self.window_border.top_right =    (offset_x + total_width, total_height + self.panel_padding + prefs_y_offset)
            self.window_border.draw()

        for item in help_items:

            key, desc = item[0], item[1]
            color = self.db.colors.Hops_UI_secondary_text_color
            render_text(text=key, position=(offset_x, offset_y + self.panel_padding + prefs_y_offset - self.font_padding), size=self.font_size, color=color)
            render_text(text=desc, position=(offset_x + key_max_width, offset_y + self.panel_padding + prefs_y_offset - self.font_padding), size=self.font_size, color=color)

            offset_y += key_dims[1] + self.font_padding

        msg = "Standard (Shift H)" if as_global else "Global (Shift H)"
        self.__draw_window_label(msg=msg, pos=(offset_x - self.font_padding, offset_y))

    def __draw_window_label(self, msg="", pos=(0,0)):

        text_dims = get_blf_text_dims(text=msg, size=12)

        total_width = text_dims[0] + self.font_padding * 2
        total_height = text_dims[1] + self.font_padding * 2

        x, y = pos
        y += self.font_padding * 4

        if self.db.prefs.ui.Hops_modal_background:
            self.label_background.bottom_left  = (x , y)
            self.label_background.bottom_right = (x + total_width, y)
            self.label_background.top_left     = (x, y + total_height)
            self.label_background.top_right    = (x + total_width, y + total_height)
            self.label_background.draw()

        if self.db.prefs.ui.Hops_modal_cell_border:
            self.label_border.bottom_left  = (x , y)
            self.label_border.bottom_right = (x + total_width, y)
            self.label_border.top_left     = (x, y + total_height)
            self.label_border.top_right    = (x + total_width, y + total_height)
            self.label_border.draw()

        y += self.font_padding
        x += self.font_padding
        color = self.db.colors.Hops_UI_secondary_text_color
        render_text(text=msg, position=(x, y), size=12, color=color)


# See https://stackoverflow.com/a/19521297/3187068
import matplotlib
matplotlib.use('pdf')
font = {'size': 16}
matplotlib.rc('font', **font)

import argparse
import itertools
import matplotlib.pyplot as plt
import os.path

MARKERS = itertools.cycle(['o', '*', '^', 's', 'P', 'x', '1'])

def plot_one_throughput_vs_num_replicas(args) -> None:
    fig, ax = plt.subplots(1, 1, figsize=(1 * 6.4, 4.8))

    for fw in [0, 0.01, 0.02, 0.05, 0.1, 0.25, 1]:
        fr = 1 - fw
        ns = list(range(2, 31))
        throughputs = [(n * args.alpha) / (n*fw + fr) / 1000000 for n in ns]
        ax.plot(ns, throughputs, '.-', marker=next(MARKERS),
                label=f'{int(fr * 100)}% reads')

    ax.set_xlabel('Number of replicas')
    ax.grid()
    ax.set_ylabel('Peak throughput\n(millions of commands per second)')
    ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
    output_filename = os.path.join(args.output_dir,
                                   'theory_one_tput_vs_replicas.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_throughput_vs_num_replicas(args) -> None:
    fig, ax = plt.subplots(1, 3, figsize=(3 * 6.4, 4.8))

    for fw in [0, 0.01, 0.02, 0.05, 0.1, 0.25, 0.5, 1]:
        fr = 1 - fw
        ns = list(range(2, 31))
        throughputs = [(n * args.alpha) / (n*fw + fr) for n in ns]
        write_throughputs = [fw * t for t in throughputs]
        read_throughputs = [fr * t for t in throughputs]
        ax[0].plot(ns, throughputs, '.-', label=f'{int(fw * 100)}% writes')
        ax[1].plot(ns, write_throughputs, '.-')
        ax[2].plot(ns, read_throughputs, '.-')

    for a in ax:
        a.set_xlabel('Number of replicas')
        a.grid()
    ax[0].set_ylabel('Peak throughput')
    ax[1].set_ylabel('Peak write throughput')
    ax[2].set_ylabel('Peak read throughput')
    ax[0].legend(loc='best')
    output_filename = os.path.join(args.output_dir, 'theory_tput_vs_replicas.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_throughput_vs_write_ratio(args) -> None:
    fig, ax = plt.subplots(1, 3, figsize=(3 * 6.4, 4.8))

    fws = [i / 100 for i in range(0, 100, 2)] + [1]
    frs = [1 - fw for fw in fws]
    for n in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
        throughputs = [(n * args.alpha) / (n*fw + fr) for (fw, fr) in zip(fws, frs)]
        write_throughputs = [fw * t for (fw, t) in zip(fws, throughputs)]
        read_throughputs = [fr * t for (fr, t) in zip(frs, throughputs)]
        ax[0].plot(frs, throughputs, '.-', label=f'{n} replicas')
        ax[1].plot(frs, write_throughputs, '.-')
        ax[2].plot(frs, read_throughputs, '.-')

    fws = [i / 100 for i in range(10, 100, 2)] + [1]
    frs = [1 - fw for fw in fws]
    throughputs = [args.alpha / fw for fw in fws]
    write_throughputs = [fw * t for (fw, t) in zip(fws, throughputs)]
    read_throughputs = [fr * t for (fr, t) in zip(frs, throughputs)]
    ax[0].plot(frs, throughputs, '.-', label='infinite replicas')
    ax[1].plot(frs, write_throughputs, '.-')
    ax[2].plot(frs, read_throughputs, '.-')

    for a in ax:
        a.set_xlabel('Read fraction')
        a.grid()
    ax[0].set_ylabel('Peak throughput')
    ax[1].set_ylabel('Peak write throughput')
    ax[2].set_ylabel('Peak read throughput')
    ax[0].legend(loc='best')
    output_filename = os.path.join(args.output_dir, 'theory_tput_vs_fraction.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_nice_throughput_vs_num_replicas(args) -> None:
    fig, ax = plt.subplots(2, 1, figsize=(6.4, 2 * 4.8))

    for alpha_w in [f*args.alpha for f in [0, 0.1, 0.2, 0.3, 0.4, 0.5,
                                      0.75, 0.9, 0.95, 1]]:
        ns = list(range(2, 31))
        throughputs = [alpha_w + n*(args.alpha - alpha_w) for n in ns]
        write_throughputs = [alpha_w] * len(throughputs)
        read_throughputs = [t - alpha_w for t in throughputs]
        frs = [r / (r + w)
               for (r, w) in zip(read_throughputs, write_throughputs)]
        ax[0].plot(ns, throughputs, '.-', label=f'{int(alpha_w)} writes')
        ax[1].plot(ns, frs, '.-')

    for a in ax:
        a.set_xlabel('Number of replicas')
        a.grid()
    ax[0].set_ylabel('Peak throughput')
    ax[1].set_ylabel('Read fraction')
    ax[0].legend(loc='best')
    output_filename = os.path.join(args.output_dir,
                                   'theory_nice_tput_vs_replicas.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def plot_nice_throughput_vs_writes(args) -> None:
    fig, ax = plt.subplots(1, 1, figsize=(6.4, 4.8))

    alpha_ws = [args.alpha * i / 50 for i in range(0, 51)]
    for n in [1, 2, 3, 4, 5, 6, 7, 8, 9]:
        throughputs = [w + n*(args.alpha - w) for w in alpha_ws]
        read_throughputs = [t - w for (t, w) in zip(throughputs, alpha_ws)]
        ax.plot(alpha_ws, throughputs, '.-', label=f'{n} replicas')

    ax.set_xlabel('Writes')
    ax.grid()
    ax.set_ylabel('Peak throughput')
    ax.legend(loc='best')
    output_filename = os.path.join(args.output_dir,
                                   'theory_nice_tput_vs_writes.pdf')
    fig.savefig(output_filename, bbox_inches='tight')
    print(f'Wrote plot to {output_filename}.')

def main(args) -> None:
    plot_one_throughput_vs_num_replicas(args)
    # plot_throughput_vs_num_replicas(args)
    # plot_throughput_vs_write_ratio(args)
    # plot_nice_throughput_vs_num_replicas(args)
    # plot_nice_throughput_vs_writes(args)

def get_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser()
    parser.add_argument('--output_dir',
                        type=str,
                        default='.',
                        help='Output directory.')
    parser.add_argument('--alpha',
                        type=int,
                        default=100000,
                        help='Peak throughput of a single server.')
    return parser

if __name__ == '__main__':
    parser = get_parser()
    main(parser.parse_args())