import logging
import os
import time
import inspect
import shutil
import threading
import traceback

from six.moves import queue

from ray.tune import TuneError, session
from ray.tune.trainable import Trainable, TrainableUtil
from ray.tune.result import (TIME_THIS_ITER_S, RESULT_DUPLICATE,
                             SHOULD_CHECKPOINT)

logger = logging.getLogger(__name__)

# Time between FunctionRunner checks when fetching
# new results after signaling the reporter to continue
RESULT_FETCH_TIMEOUT = 0.2

ERROR_REPORT_TIMEOUT = 10
ERROR_FETCH_TIMEOUT = 1

class StatusReporter:
    """Object passed into your function that you can report status through.

    Example:
        >>> def trainable_function(config, reporter):
        >>>     assert isinstance(reporter, StatusReporter)
        >>>     reporter(timesteps_this_iter=1)
    """

    def __init__(self,
                 result_queue,
                 continue_semaphore,
                 trial_name=None,
                 trial_id=None,
                 logdir=None):
        self._queue = result_queue
        self._last_report_time = None
        self._continue_semaphore = continue_semaphore
        self._trial_name = trial_name
        self._trial_id = trial_id
        self._logdir = logdir
        self._last_checkpoint = {}
        self._fresh_checkpoint = False

    def __call__(self, **kwargs):
        """Report updated training status.

        Pass in `done=True` when the training job is completed.

        Args:
            kwargs: Latest training result status.

        Example:
            >>> reporter(mean_accuracy=1, training_iteration=4)
            >>> reporter(mean_accuracy=1, training_iteration=4, done=True)

        Raises:
            StopIteration: A StopIteration exception is raised if the trial has
                been signaled to stop.
        """

        assert self._last_report_time is not None, (
            "StatusReporter._start() must be called before the first "
            "report __call__ is made to ensure correct runtime metrics.")

        # time per iteration is recorded directly in the reporter to ensure
        # any delays in logging results aren't counted
        report_time = time.time()
        if TIME_THIS_ITER_S not in kwargs:
            kwargs[TIME_THIS_ITER_S] = report_time - self._last_report_time
        self._last_report_time = report_time

        # add results to a thread-safe queue
        self._queue.put(kwargs.copy(), block=True)

        # This blocks until notification from the FunctionRunner that the last
        # result has been returned to Tune and that the function is safe to
        # resume training.
        self._continue_semaphore.acquire()

    def make_checkpoint_dir(self, step=None):
        checkpoint_dir = TrainableUtil.make_checkpoint_dir(
            self.logdir, index=step)
        logger.debug("Making checkpoint dir at %s", checkpoint_dir)
        return checkpoint_dir

    def save_checkpoint(self, checkpoint):
        if isinstance(checkpoint, str):
            try:
                TrainableUtil.find_checkpoint_dir(checkpoint)
            except FileNotFoundError:
                logger.error("Checkpoint must be created with path given from "
                             "make_checkpoint_dir.")
                raise
        self._last_checkpoint = checkpoint
        self._fresh_checkpoint = True

    def has_new_checkpoint(self):
        return self._fresh_checkpoint

    def get_checkpoint(self):
        self._fresh_checkpoint = False
        return self._last_checkpoint

    def _start(self):
        self._last_report_time = time.time()

    @property
    def logdir(self):
        return self._logdir

    @property
    def trial_name(self):
        """Trial name for the corresponding trial of this Trainable."""
        return self._trial_name

    @property
    def trial_id(self):
        """Trial id for the corresponding trial of this Trainable."""
        return self._trial_id

class _RunnerThread(threading.Thread):
    """Supervisor thread that runs your script."""

    def __init__(self, entrypoint, error_queue):
        threading.Thread.__init__(self)
        self._entrypoint = entrypoint
        self._error_queue = error_queue
        self.daemon = True

    def run(self):
        try:
            self._entrypoint()
        except StopIteration:
            logger.debug(
                ("Thread runner raised StopIteration. Interperting it as a "
                 "signal to terminate the thread without error."))
        except Exception as e:
            logger.exception("Runner Thread raised error.")
            try:
                # report the error but avoid indefinite blocking which would
                # prevent the exception from being propagated in the unlikely
                # case that something went terribly wrong
                err_tb_str = traceback.format_exc()
                self._error_queue.put(
                    err_tb_str, block=True, timeout=ERROR_REPORT_TIMEOUT)
            except queue.Full:
                logger.critical(
                    ("Runner Thread was unable to report error to main "
                     "function runner thread. This means a previous error "
                     "was not processed. This should never happen."))
            raise e

class FunctionRunner(Trainable):
    """Trainable that runs a user function reporting results.

    This mode of execution does not support checkpoint/restore."""

    _name = "func"

    def setup(self, config):
        # Semaphore for notifying the reporter to continue with the computation
        # and to generate the next result.
        self._continue_semaphore = threading.Semaphore(0)

        # Queue for passing results between threads
        self._results_queue = queue.Queue(1)

        # Queue for passing errors back from the thread runner. The error queue
        # has a max size of one to prevent stacking error and force error
        # reporting to block until finished.
        self._error_queue = queue.Queue(1)

        self._status_reporter = StatusReporter(
            self._results_queue,
            self._continue_semaphore,
            trial_name=self.trial_name,
            trial_id=self.trial_id,
            logdir=self.logdir)
        self._last_result = {}

        session.init(self._status_reporter)
        self._runner = None
        self._restore_tmpdir = None
        self.default_checkpoint_dir = None

    def _trainable_func(self):
        """Subclasses can override this to set the trainable func."""

        raise NotImplementedError

    def _start(self):
        def entrypoint():
            return self._trainable_func(self.config, self._status_reporter,
                                        self._status_reporter.get_checkpoint())

        # the runner thread is not started until the first call to _train
        self._runner = _RunnerThread(entrypoint, self._error_queue)
        # if not alive, try to start
        self._status_reporter._start()
        try:
            self._runner.start()
        except RuntimeError:
            # If this is reached, it means the thread was started and is
            # now done or has raised an exception.
            pass

    def step(self):
        """Implements train() for a Function API.

        If the RunnerThread finishes without reporting "done",
        Tune will automatically provide a magic keyword __duplicate__
        along with a result with "done=True". The TrialRunner will handle the
        result accordingly (see tune/trial_runner.py).
        """
        if self._runner and self._runner.is_alive():
            # if started and alive, inform the reporter to continue and
            # generate the next result
            self._continue_semaphore.release()
        else:
            self._start()

        result = None
        while result is None and self._runner.is_alive():
            # fetch the next produced result
            try:
                result = self._results_queue.get(
                    block=True, timeout=RESULT_FETCH_TIMEOUT)
            except queue.Empty:
                pass

        # if no result were found, then the runner must no longer be alive
        if result is None:
            # Try one last time to fetch results in case results were reported
            # in between the time of the last check and the termination of the
            # thread runner.
            try:
                result = self._results_queue.get(block=False)
            except queue.Empty:
                pass

        # check if error occured inside the thread runner
        if result is None:
            # only raise an error from the runner if all results are consumed
            self._report_thread_runner_error(block=True)

            # Under normal conditions, this code should never be reached since
            # this branch should only be visited if the runner thread raised
            # an exception. If no exception were raised, it means that the
            # runner thread never reported any results which should not be
            # possible when wrapping functions with `wrap_function`.
            raise TuneError(
                ("Wrapped function ran until completion without reporting "
                 "results or raising an exception."))

        else:
            if not self._error_queue.empty():
                logger.warning(
                    ("Runner error waiting to be raised in main thread. "
                     "Logging all available results first."))

        # This keyword appears if the train_func using the Function API
        # finishes without "done=True". This duplicates the last result, but
        # the TrialRunner will not log this result again.
        if "__duplicate__" in result:
            new_result = self._last_result.copy()
            new_result.update(result)
            result = new_result

        self._last_result = result
        if self._status_reporter.has_new_checkpoint():
            result[SHOULD_CHECKPOINT] = True
        return result

    def execute(self, fn):
        return fn(self)

    def create_default_checkpoint_dir(self):
        self.default_checkpoint_dir = TrainableUtil.make_checkpoint_dir(
            self.logdir, index="default")
        return self.default_checkpoint_dir

    def save(self, checkpoint_path=None):
        if checkpoint_path:
            raise ValueError(
                "Checkpoint path should not be used with function API.")

        checkpoint = self._status_reporter.get_checkpoint()
        state = self.get_state()

        if not checkpoint:
            state.update(iteration=0, timesteps_total=0, episodes_total=0)
            parent_dir = self.create_default_checkpoint_dir()
        elif isinstance(checkpoint, dict):
            parent_dir = TrainableUtil.make_checkpoint_dir(
                self.logdir, index=self.training_iteration)
        else:
            parent_dir = TrainableUtil.find_checkpoint_dir(checkpoint)
        checkpoint_path = TrainableUtil.process_checkpoint(
            checkpoint, parent_dir, state)
        return checkpoint_path

    def save_to_object(self):
        checkpoint_path = self.save()
        obj = TrainableUtil.checkpoint_to_object(checkpoint_path)
        return obj

    def load_checkpoint(self, checkpoint):
        # This should be removed once Trainables are refactored.
        if "tune_checkpoint_path" in checkpoint:
            del checkpoint["tune_checkpoint_path"]
        self._status_reporter.save_checkpoint(checkpoint)

    def restore_from_object(self, obj):
        if self.default_checkpoint_dir is not None and os.exists(
                self.default_checkpoint_dir):
            shutil.rmtree(self.default_checkpoint_dir)
            logger.debug("Clearing default checkpoint: %s",
                         self.default_checkpoint_dir)

        checkpoint_dir = self.create_default_checkpoint_dir()
        checkpoint_path = TrainableUtil.create_from_pickle(obj, checkpoint_dir)
        self.restore(checkpoint_path)

    def cleanup(self):
        # If everything stayed in synch properly, this should never happen.
        if not self._results_queue.empty():
            logger.warning(
                ("Some results were added after the trial stop condition. "
                 "These results won't be logged."))

        # Check for any errors that might have been missed.
        self._report_thread_runner_error()
        session.shutdown()

    def _report_thread_runner_error(self, block=False):
        try:
            err_tb_str = self._error_queue.get(
                block=block, timeout=ERROR_FETCH_TIMEOUT)
            raise TuneError(("Trial raised an exception. Traceback:\n{}"
                             .format(err_tb_str)))
        except queue.Empty:
            pass

def detect_checkpoint_function(train_func, abort=False):
    """Use checkpointing if any arg has "checkpoint_dir" and args = 2"""
    argspec = inspect.getfullargspec(train_func)
    func_args = argspec.args
    func_kwargs = argspec.kwonlyargs
    validated = len(func_args) == 2 and any("checkpoint_dir" in arg
                                            for arg in func_args)
    validated = validated or (len(func_args) == 1) and any(
        "checkpoint_dir" in arg for arg in func_kwargs)
    if abort and not validated:
        raise ValueError(
            "Provided training function must have 2 args "
            "in the signature, and the latter arg must "
            "contain `checkpoint_dir`. For example: "
            "`func(config, checkpoint_dir=None)`. Got {}".format(func_args))
    return validated

def wrap_function(train_func):
    if hasattr(train_func, "__mixins__"):
        inherit_from = train_func.__mixins__ + (FunctionRunner, )
    else:
        inherit_from = (FunctionRunner, )

    class ImplicitFunc(*inherit_from):
        _name = train_func.__name__ if hasattr(train_func, "__name__") \
            else "func"

        def _trainable_func(self, config, reporter, checkpoint_dir):
            func_args = inspect.getfullargspec(train_func).args
            if len(func_args) > 1:  # more arguments than just the config
                if "reporter" not in func_args and (
                        not detect_checkpoint_function(train_func)):
                    raise ValueError(
                        "Unknown argument found in the Trainable function. "
                        "Arguments other than the 'config' arg must be one "
                        "of ['reporter', 'checkpoint_dir']. Found: {}".format(
                            func_args))
            use_reporter = "reporter" in func_args
            use_checkpoint = detect_checkpoint_function(train_func)
            if not use_checkpoint and not use_reporter:
                logger.warning(
                    "Function checkpointing is disabled. This may result in "
                    "unexpected behavior when using checkpointing features or "
                    "certain schedulers. To enable, set the train function "
                    "arguments to be `func(config, checkpoint_dir=None)`.")
                output = train_func(config)
            elif use_checkpoint:
                output = train_func(config, checkpoint_dir=checkpoint_dir)
            else:
                output = train_func(config, reporter)

            # If train_func returns, we need to notify the main event loop
            # of the last result while avoiding double logging. This is done
            # with the keyword RESULT_DUPLICATE -- see tune/trial_runner.py.
            reporter(**{RESULT_DUPLICATE: True})
            return output

    return ImplicitFunc


from time import sleep
from typing import Dict, List

import boto3
from botocore.exceptions import ClientError

from api.common.config.aws import (
    AWS_REGION,
    GLUE_CATALOGUE_DB_NAME,
    GLUE_TABLE_PRESENCE_CHECK_INTERVAL,
    GLUE_TABLE_PRESENCE_CHECK_RETRY_COUNT,
)
from api.common.custom_exceptions import (
    AWSServiceError,
    TableAlreadyExistsError,
    TableCreationError,
    TableDoesNotExistError,
)
from api.common.logger import AppLogger
from api.domain.dataset_metadata import DatasetMetadata
from api.domain.schema import Schema

class GlueAdapter:
    def __init__(
        self,
        glue_client=boto3.client("glue", region_name=AWS_REGION),
        glue_catalogue_db_name=GLUE_CATALOGUE_DB_NAME,
    ):
        self.glue_client = glue_client
        self.glue_catalogue_db_name = glue_catalogue_db_name

    def create_table(self, schema: Schema):
        try:
            self.glue_client.create_table(
                DatabaseName=self.glue_catalogue_db_name,
                TableInput={
                    "Name": schema.metadata.glue_table_name(),
                    "Owner": "hadoop",
                    "StorageDescriptor": {
                        "Columns": schema.get_non_partition_columns_for_glue(),
                        "Location": schema.metadata.s3_file_location(),
                        "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
                        "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
                        "Compressed": False,
                        "SerdeInfo": {
                            "SerializationLibrary": "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
                            "Parameters": {"serialization.format": "1"},
                        },
                        "NumberOfBuckets": -1,
                        "StoredAsSubDirectories": False,
                    },
                    "PartitionKeys": schema.get_partition_columns_for_glue(),
                    "TableType": "EXTERNAL_TABLE",
                    "Parameters": {
                        "classification": "parquet",
                        "typeOfData": "file",
                        "compressionType": "none",
                        "EXTERNAL": "TRUE",
                    },
                },
                PartitionIndexes=[
                    {
                        "Keys": [
                            col["Name"]
                            for col in schema.get_partition_columns_for_glue()
                        ],
                        "IndexName": "rapid-creation-partition",
                    },
                ]
                if schema.get_partition_columns_for_glue()
                else [],
            )
        except ClientError as error:
            self._handle_table_create_error(error)

    def _handle_table_create_error(self, error: ClientError):
        if error.response["Error"]["Code"] == "AlreadyExistsException":
            raise TableAlreadyExistsError("Table already exists with same name")
        else:
            raise TableCreationError("Table creation error")

    def get_table_when_created(self, table_name: str) -> Dict:
        for _ in range(GLUE_TABLE_PRESENCE_CHECK_RETRY_COUNT):
            try:
                return self._get_table(table_name)
            except TableDoesNotExistError:
                AppLogger.info(
                    f"Waiting {GLUE_TABLE_PRESENCE_CHECK_INTERVAL}s for table [{table_name}] to be created"
                )
                sleep(GLUE_TABLE_PRESENCE_CHECK_INTERVAL)
                continue
        raise AWSServiceError(
            f"[{table_name}] was not created after {GLUE_TABLE_PRESENCE_CHECK_RETRY_COUNT * GLUE_TABLE_PRESENCE_CHECK_INTERVAL}s"
        )  # noqa: E501

    def get_tables_for_dataset(self, dataset: DatasetMetadata) -> List[str]:
        tables = [
            item["Name"] for item in self._search_tables(dataset.glue_table_prefix())
        ]
        return tables

    def delete_tables(self, table_names: List[str]):
        try:
            self.glue_client.batch_delete_table(
                DatabaseName=GLUE_CATALOGUE_DB_NAME, TablesToDelete=table_names
            )
        except ClientError:
            raise AWSServiceError("Failed to delete tables")

    def _get_table(self, table_name: str) -> Dict:
        try:
            table = self.glue_client.get_table(
                DatabaseName=self.glue_catalogue_db_name, Name=table_name
            )
            return table
        except ClientError as error:
            if error.response["Error"]["Code"] == "EntityNotFoundException":
                raise TableDoesNotExistError(f"The table [{table_name}] does not exist")

    def _search_tables(self, search_term: str) -> Dict:
        try:
            paginator = self.glue_client.get_paginator("get_tables")
            page_iterator = paginator.paginate(
                DatabaseName=GLUE_CATALOGUE_DB_NAME,
            )
            tables = []
            for page in page_iterator:
                tables.extend(page["TableList"])

            return [table for table in tables if table["Name"].startswith(search_term)]
        except ClientError:
            raise AWSServiceError(f"Failed to search tables with term={search_term}")


from proc import *
from model import *
# from test import *

import numpy as np
import matplotlib.pyplot as plt

#n = eval(input("\n\nEnter number of iterations: ")) #1000
#l = eval(input("Enter learning rate: ")) #0.001
#p = bool(eval(input("Enter 1 to display cost else 0: "))) #False

n=6000
l=0.0001971
p=0

d = model(train_set_x, train_set_y, eval_set_x, eval_set_y, num_iterations = n, learning_rate = l, print_cost = p)

#retrieving essential data about model from dictionary 'd'
costs = d["costs"]
learning_rate = d["learning_rate"]

#saving the trained model for future use for evaling
np.save('trained_model12.npy',d)

#ploting cost vs iteration graph
plt.plot((costs))
plt.ylabel('cost')
plt.xlabel('iterations (per tens) ')
plt.title("Learning rate = " + str(learning_rate))
plt.show()

# test()


#!/usr/bin/env python2
from multiprocessing import Process
import os

import portals
from download_ckan import download as ckan
from download_socrata import download as socrata
from download_opendatasoft import download as opendatasoft

ROOT_DIR = 'portals'

def create(portal_type, func, portal_urls):
    '''
    Args:
        portal_type: "ckan", "opendatasoft" or "socrata"
        func: function to download all datasets given a portal and directory
        portal_urls: List of portal domains
    Returns:
        A list of processes
    '''
    processes = {}

    for portal in portal_urls:
        args = (portal, os.path.join(ROOT_DIR, portal_type))
        processes[(portal_type, portal)] = Process(target = func, args = args)

    return processes

def start(processes):
    'Start all of the processes.'
    for process in processes.values():
        process.start()

def join(processes):
    'Wait for all of the processes to end.'
    for process in processes.values():
        process.join()

def kill(processes):
    'Kill all of the processes.'
    for process in processes.values():
        process.terminate()

def main():
    import signal
    import sys

    # p = create('ckan', ckan, portals.ckan)
    # p = create('opendatasoft', opendatasoft, portals.opendatasoft)
    p = create('socrata', socrata, portals.socrata)

    def signal_handler(signal, frame):
        kill(p)
        sys.exit(0)

    signal.signal(signal.SIGINT, signal_handler)

    start(p)
    join(p)

if __name__ == '__main__':
    main()


from flask import Flask
from flask import request
from flask_cors import CORS, cross_origin
from blink1.blink1 import Blink1

app = Flask(__name__)
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'

b1 = Blink1()

def rgb_to_hex(rgb):
    return "#" + str('%02x%02x%02x' % rgb)

def call_blink_function(method, color_value, transition_time=1000):
    if method == "fade":
        b1.fade_to_color(transition_time, color_value)
    if method == "blink":
        b1.write_pattern_line(transition_time, color_value, 1)
        b1.write_pattern_line(transition_time, '#000000', 2)
        b1.play(1,2)

    return

def handle_color(method, color_value, transition_time=1000):
    error_output = ""

    try:
        call_blink_function(method, color_value, transition_time)

        return f"Blink changed with color {color_value} with method {method}."
    except:
        pass

    try:
        tempval = color_value
        red_value = tempval[0:tempval.index(",")]
        tempval = tempval[(tempval.index(",") + 1): len(tempval)]
        green_value = tempval[0:tempval.index(",")]
        tempval = tempval[(tempval.index(",") + 1): len(tempval)]
        blue_value = tempval

        red_value = int(red_value)
        green_value = int(green_value)
        blue_value = int(blue_value)

        tempval = rgb_to_hex((red_value, green_value, blue_value))

        call_blink_function(method, tempval, transition_time)

        return f"Blink changed with rgb value ({red_value}, {green_value}, {blue_value}) with method {method}."
    except:
        pass

    try:
        hexval = color_value
        if len(hexval) == 7:
            hexval = hexval[1: len(hexval)]
        hexval = "#" + hexval

        call_blink_function(method, hexval, transition_time)

        return f"Blink changed with hex value {hexval} with method {method}."
    except:
        pass

    return f"You've provided an invalid color\n{error_output}"

@app.route("/")
@cross_origin()
def home():
    return f"To change the light color, query /fade. To have it blink, /blink. To turn it off, /off"

@app.route("/off")
@cross_origin()
def off():
    b1.off()
    return f"Blink turned off"

@app.route("/fade")
@cross_origin()
def fade():
    method = "fade"

    transition_time = request.args.get('time')
    if transition_time is not None:
        transition_time = int(transition_time)
    else:
        transition_time = 1000

    color_value = request.args.get('color')
    if color_value is None:
        return f"Please specify a color value. Example: /{method}?color=red or /{method}?color=5523f2 or /{method}?color=13,22,192. You can also specify an optional time argument."

    return handle_color(method, color_value, transition_time)

@app.route("/blink")
@cross_origin()
def blink():
    method = "blink"

    transition_time = request.args.get('time')
    if transition_time is not None:
        transition_time = int(transition_time)
    else:
        transition_time = 1000

    color_value = request.args.get('color')
    if color_value is None:
        return f"Please specify a color value. Example: /{method}?color=red or /{method}?color=5523f2 or /{method}?color=13,22,192. You can also specify an optional time argument."

    return handle_color(method, color_value, transition_time)

if __name__ == "__main__":
    app.run(port=5000, host="0.0.0.0")


import numpy as np

from lib.data import get_dataset
from lib.classifier import SVM
from lib.bag_of_words import get_features
from lib.descriptors import get_keypoints_orb
from lib.mask import mask_apple_tomato

def get_data_and_voc(infos, val_ratio, voc_path1, voc_path2):
    (data, y), (data_test, y_test) = get_dataset('dataset/fruits',
                                                infos, val_ratio=val_ratio)
    vocabulary1 = np.load(voc_path1)
    vocabulary2 = np.load(voc_path2)
    vocabulary = np.concatenate((vocabulary1, vocabulary2), axis=0)
    return (data, y), (data_test, y_test), vocabulary

def random_cross_validation(n_folds, X, y):
    mean_acc = 0.0
    for _ in range(n_folds):
        val_idx = np.random.choice(len(X), int(len(X) * data_prop),
                                replace=False)
        train_idx = np.array([i for i in range(len(X)) if i not
                                in val_idx])
        X_val, y_val = X[val_idx], y[val_idx]
        X_train, y_train = X[train_idx], y[train_idx]
        svm = SVM(C=C, kernel='rbf', gamma=gamma, verbose=False)
        svm.fit(X_train, y_train)
        preds = svm.predict(X_val)
        mean_acc += np.mean(preds == y_val) / n_folds
    return mean_acc

def test_classifier(svm, X_train, y_train, X_test, y_test):
    svm.fit(X_train, y_train)
    preds = svm.predict(X_test)
    return np.mean(preds == y_test)

if __name__ == '__main__':
    ## Configs
    # Dataset / vocabulary / masking function
    infos = {'apple_a': (0, 500), 'apple_b': (1, 500), 'apple_c': (1, 0),
                    'tomato': (1, 0)}
    val_ratio = 0.15
    voc_path1 = 'vocabulary/fruits/apple_a_50_orb.npy'
    voc_path2 = 'vocabulary/fruits/apple_b_50_orb.npy'
    mask_func = mask_apple_tomato
    # Descriptors
    n_desc = 500
    dist = 'euclidian'
    # Cross validation
    data_prop = 0.2
    n_folds = 20
    C_range = [1, 10, 100, 1000]
    gamma_range = [1e-5, 1e-4, 1e-3, 1e-2, 0.1, 1]
    # Test
    test = True
    test_svm = SVM(C=100, kernel='rbf', gamma=0.1, verbose=False)

    (data, y), (data_test, y_test), vocabulary = get_data_and_voc(
        infos, val_ratio, voc_path1, voc_path2
        )
    print('Train & validation set:', len(data), 'samples.')
    print('Test set:', len(data_test), 'samples.')

    # Get descriptors and make bag of words features
    desc, _ = get_keypoints_orb(data, select=n_desc,
                                    mask_func=mask_func)
    desc_test, _ = get_keypoints_orb(data_test, select=n_desc,
                                    mask_func=mask_func)
    X = get_features(desc, vocabulary)
    X_test = get_features(desc_test, vocabulary, dist=dist)
    print('Features extracted.')

    # Cross-validation and gridsearch for SVM
    for C in C_range:
        for gamma in gamma_range:
            mean_acc = random_cross_validation(n_folds, X, y)
            print(f'C={C}, gamma={gamma:.6f} - val acc={100 * mean_acc:.2f}%')

    if test:
        # Test SVM on test set
        accuracy = test_classifier(test_svm, X, y, X_test, y_test)
        print(f'test accuracy: {100 * accuracy:.2f}%')


"""

Deprecated. Nullified by new PySAM code and will be taken out in Beta release.

"""

import requests as req
import numpy as np
import pandas as pd
import codecs
import json
import csv
import logging
logging.getLogger("requests").setLevel(logging.WARNING)

#%%
# Load configuration file, if one exists.
def load_config_params(config_file_name):
    '''
    Each user should fill in a config_template.json file.
    '''

    config = json.load(open('config.json','r'))

    return config

#%%
class Tariff:
    """
    Tariff Attributes:
        urdb_id: id for utility rate database. US, not international.
        eia_id: The EIA assigned ID number for the utility associated with this tariff
        name: tariff name
        utility: Name of utility this tariff is associated with
        fixed_charge: Fixed monthly charge in $/mo.
        peak_kW_capacity_max: The annual maximum kW of demand that a customer can have and still be on this tariff
        peak_kW_capacity_min: The annula minimum kW of demand that a customer can have and still be on this tariff
        kWh_useage_max: The maximum kWh of average monthly consumption that a customer can have and still be on this tariff
        kWh_useage_min: The minimum kWh of average monthly consumption that a customer can have and still be on this tariff
        sector: residential, commercial, or industrial
        comments: comments from the urdb
        description: tariff description from urdb
        source: uri for the source of the tariff
        uri: link the the urdb page
        voltage_category: secondary, primary, transmission
        d_flat_exists: Boolean of whether there is a flat (not tou) demand charge component. Flat demand is also called monthly or seasonal demand.
        d_flat_n: Number of unique flat demand period constructions. Does NOT correspond to width of d_flat_x constructs.
        d_flat_prices: The prices of each tier/period combination for flat demand. Rows are tiers, columns are months. Differs from TOU, where columns are periods.
        d_flat_levels: The limit (total kW) of each of each tier/period combination for flat demand. Rows are tiers, columns are months. Differs from TOU, where columns are periods.
        d_tou_exists = Boolean of whether there is a tou (not flat) demand charge component
        d_tou_n = Number of unique tou demand periods. Minimum of 1, since I'm counting no-charge periods still as a period.
        d_tou_prices = The prices of each tier/period combination for tou demand. Rows are tiers, columns are periods.
        d_tou_levels = The limit (total kW) of each of each tier/period combination for tou demand. Rows are tiers, columns are periods.
        e_exists = Boolean of whether there is a flat (not tou) demand charge component
        e_tou_exists = Boolean of whether there is a flat (not tou) demand charge component
        e_n = Number of unique energy periods. Minimum of 1, since I'm counting no-charge periods still as a period.
        e_prices = The prices of each tier/period combination for flat demand. Rows are tiers, columns are periods.
        e_levels = The limit (total kWh) of each of each tier/period combination for energy. Rows are tiers, columns are periods.
        e_wkday_12by24: 12 by 24 period definition for weekday energy. Rows are months, columns are hours.
        e_wkend_12by24: 12 by 24 period definition for weekend energy. Rows are months, columns are hours.
        d_wkday_12by24: 12 by 24 period definition for weekday energy. Rows are months, columns are hours.
        d_wkend_12by24: 12 by 24 period definition for weekend energy. Rows are months, columns are hours.
        d_tou_8760
        e_tou_8760
        e_prices_no_tier
        e_max_difference: The maximum energy price differential within any single day
        energy_rate_unit: kWh or kWh/day - for guiding the bill calculations later
        demand_rate_unit: kW or kW/day - for guiding the bill calculations later
    """

    def __init__(self, start_day=6, urdb_id=None, json_file_name=None, dict_obj=None, api_key=None):

        #######################################################################
        ##### If given no urdb id or csv file name, create blank tariff #######
        #######################################################################

        if urdb_id==None and json_file_name==None and isinstance(dict_obj,type(None)):
            # Default values for a blank tariff
            self.urdb_id = 'No urdb id given'
            self.name = 'User defined tariff - no name specified'
            self.utility = 'User defined tariff - no name specified'
            self.fixed_charge = 0
            self.peak_kW_capacity_max = 1e99
            self.peak_kW_capacity_min = 0
            self.kWh_useage_max = 1e99
            self.kWh_useage_min = 0
            self.sector = 'No sector specified'
            self.comments = 'No comments given'
            self.description = 'No description given'
            self.source = 'No source given'
            self.uri = 'No uri given'
            self.voltage_category = 'No voltage category given'
            self.eia_id = 'No eia id given'
            self.demand_rate_unit = 'kW'
            self.energy_rate_unit = 'kWh'
            self.start_day = 6

            ###################### Blank Flat Demand Structure ########################
            self.d_flat_exists = False
            self.d_flat_n = 0
            self.d_flat_prices = np.zeros([1, 12])
            self.d_flat_levels = np.zeros([1, 12])
            self.d_flat_levels[:,:] = 1e9

            #################### Blank Demand TOU Structure ###########################
            self.d_tou_exists = False
            self.d_tou_n = 1
            self.d_tou_prices = np.zeros([1, 1])
            self.d_tou_levels = np.zeros([1, 1])

            ################ Blank Coincident Peak Structure ##################
            self.coincident_peak_exists = False

            ######################## Blank Energy Structure ###########################
            self.e_exists = False
            self.e_tou_exists = False
            self.e_n = 1
            self.e_prices = np.zeros([1, 1])
            self.e_levels = np.zeros([1, 1])

            ######################## Blank Schedules ###########################
            self.e_wkday_12by24 = np.zeros([12,24], int)
            self.e_wkend_12by24 = np.zeros([12,24], int)
            self.d_wkday_12by24 = np.zeros([12,24], int)
            self.d_wkend_12by24 = np.zeros([12,24], int)

            ################### Blank 12x24s as 8760s Schedule ########################
            self.d_tou_8760 = np.zeros(8760, int)
            self.e_tou_8760 = np.zeros(8760, int)

            ######################## Precalculations ######################################
            self.e_prices_no_tier = np.zeros([1, 1])
            self.e_max_difference = np.zeros([1, 1])

        #######################################################################
        # If given a urdb_id input argument, obtain and reshape that tariff through the URDB API
        #######################################################################
        elif urdb_id != None:

            if api_key == None:
                print("No URDB API key defined.")

            input_params = {'version':3,
                        'format':'json',
                        'detail':'full',
                        'getpage':urdb_id,
                        'api_key':api_key}

            r = req.get('http://api.openei.org/utility_rates?', params=input_params)

            content = r.content
            tariff_original = json.loads(content, strict=False)['items'][0]

            if 'demandrateunit' in tariff_original: self.demand_rate_unit = tariff_original['demandrateunit']
            else: self.demand_rate_unit = 'kW'

            if 'eiaid' in tariff_original: self.eia_id = tariff_original['eiaid']
            else: self.eia_id = 'No eia id given'

            if 'label' in tariff_original: self.urdb_id = tariff_original['label']
            else: self.urdb_id = 'No urdb id given'

            if 'name' in tariff_original: self.name = tariff_original['name']
            else: self.name = 'No name specified'

            if 'utility' in tariff_original: self.utility = tariff_original['utility']
            else: self.utility = 'No utility specified'

            if 'fixedmonthlycharge' in tariff_original: self.fixed_charge = tariff_original['fixedmonthlycharge']
            else: self.fixed_charge = 0

            if 'peakkwcapacitymax' in tariff_original: self.peak_kW_capacity_max = tariff_original['peakkwcapacitymax']
            else: self.peak_kW_capacity_max = 1e99

            if 'peakkwcapacitymin' in tariff_original: self.peak_kW_capacity_min = tariff_original['peakkwcapacitymin']
            else: self.peak_kW_capacity_min = 0

            if 'peakkwhusagemax' in tariff_original: self.kWh_useage_max = tariff_original['peakkwhusagemax']
            else: self.kWh_useage_max = 1e99

            if 'peakkwhusagemin' in tariff_original: self.kWh_useage_min = tariff_original['peakkwhusagemin']
            else: self.kWh_useage_min = 0

            if 'sector' in tariff_original: self.sector = tariff_original['sector']
            else: self.sector = 'No sector given'

            if 'basicinformationcomments' in tariff_original: self.comments = tariff_original['basicinformationcomments']
            else: self.comments = 'No comments'

            if 'description' in tariff_original: self.description = tariff_original['description']
            else: self.description = 'No description'

            if 'source' in tariff_original: self.source = tariff_original['source']
            else: self.source = 'No source given'

            if 'uri' in tariff_original: self.uri = tariff_original['uri']
            else: self.uri = 'No uri given'

            if 'voltage_category' in tariff_original: self.voltage_category = tariff_original['voltage_category']
            else: self.voltage_category = 'No voltage category given'

            ###################### Repackage Flat Demand Structure ########################
            if 'flatdemandstructure' in tariff_original:
                self.d_flat_exists = True
                d_flat_structure = tariff_original['flatdemandstructure']
                d_flat_month_indicies = tariff_original['flatdemandmonths']
                self.d_flat_n = np.min([len(np.unique(tariff_original['flatdemandmonths'])), len(d_flat_structure)])

                # Clip indicies so they are within the given array size (only occurs when tariff was entered into URDB incorrectly)
                d_flat_month_indicies = np.clip(d_flat_month_indicies, 0, self.d_flat_n-1)

                # Determine the maximum number of tiers in the demand structure
                max_tiers = 1
                for period in range(self.d_flat_n):
                    n_tiers = len(d_flat_structure[period])
                    if n_tiers > max_tiers: max_tiers = n_tiers

                # Repackage Energy TOU Structure
                self.d_flat_prices = np.zeros([max_tiers, 12])
                self.d_flat_levels = np.zeros([max_tiers, 12])
                self.d_flat_levels[:,:] = 1e9
                for month in range(12):
                    for tier in range(len(d_flat_structure[period])):
                        self.d_flat_levels[tier, month] = d_flat_structure[d_flat_month_indicies[month]][tier].get('max', 1e9)
                        self.d_flat_prices[tier, month] = d_flat_structure[d_flat_month_indicies[month]][tier].get('rate', 0) + d_flat_structure[d_flat_month_indicies[month]][tier].get('adj', 0)
            else:
                self.d_flat_exists = False
                self.d_flat_n = 1
                self.d_flat_prices = np.zeros([1, 12])
                self.d_flat_levels = np.zeros([1, 12])
                self.d_flat_levels[:,:] = 1e9

            #################### Repackage Demand TOU Structure ###########################
            if 'demandratestructure' in tariff_original:
                demand_structure = tariff_original['demandratestructure']
                self.d_tou_n = len(demand_structure)
                if self.d_tou_n > 1: self.d_tou_exists = True
                else:
                    self.d_tou_exists = False
                    self.d_flat_exists = True

                # Determine the maximum number of tiers in the demand structure
                max_tiers = 1
                for period in range(self.d_tou_n):
                    n_tiers = len(demand_structure[period])
                    if n_tiers > max_tiers: max_tiers = n_tiers

                # Repackage Demand TOU Structure
                self.d_tou_prices = np.zeros([max_tiers, self.d_tou_n])
                self.d_tou_levels = np.zeros([max_tiers, self.d_tou_n])
                self.d_tou_levels[:,:] = 1e9
                for period in range(self.d_tou_n):
                    for tier in range(len(demand_structure[period])):
                        self.d_tou_levels[tier, period] = demand_structure[period][tier].get('max', 1e9)
                        self.d_tou_prices[tier, period] = demand_structure[period][tier].get('rate', 0) + demand_structure[period][tier].get('adj', 0)
            else:
                self.d_tou_exists = False
                self.d_tou_n = 1
                self.d_tou_prices = np.zeros([1, 1])
                self.d_tou_levels = np.zeros([1, 1])

            ######################## No Coincident Peak from URDB #############
            self.coincident_peak_exists = False

            ######################## Repackage Energy Structure ###########################
            if 'energyratestructure' in tariff_original:
                self.e_exists = True
                energy_structure = tariff_original['energyratestructure']
                self.energy_rate_unit = energy_structure[0][0].get('unit','kWh')
                self.e_n = len(energy_structure)
                if self.e_n > 1: self.e_tou_exists = True
                else: self.e_tou_exists = False

                # Determine the maximum number of tiers in the demand structure
                max_tiers = 1
                for period in range(self.e_n):
                    n_tiers = len(energy_structure[period])
                    if n_tiers > max_tiers: max_tiers = n_tiers

                # Repackage Energy TOU Structure
                self.e_prices = np.zeros([max_tiers, self.e_n])
                self.e_levels = np.zeros([max_tiers, self.e_n])
                self.e_levels[:,:] = 1e9
                for period in range(self.e_n):
                    for tier in range(len(energy_structure[period])):
                        self.e_levels[tier, period] = energy_structure[period][tier].get('max', 1e9)
                        self.e_prices[tier, period] = energy_structure[period][tier].get('rate', 0) + energy_structure[period][tier].get('adj', 0)
            else:
                self.e_exists = False
                self.e_tou_exists = False
                self.e_n = 0
                self.e_prices = np.zeros([1, 1])
                self.e_levels = np.zeros([1, 1])
                self.energy_rate_unit = 'kWh'

            ######################## Repackage Energy Schedule ###########################
            self.e_wkday_12by24 = np.zeros([12,24], int)
            self.e_wkend_12by24 = np.zeros([12,24], int)

            if 'energyweekdayschedule' in tariff_original:
                for month in range(12):
                    self.e_wkday_12by24[month, :] = tariff_original['energyweekdayschedule'][month]
                    self.e_wkend_12by24[month, :] = tariff_original['energyweekendschedule'][month]

            # If the urdb 12by24 has a period that isn't defined in the tiers,
            # set that period to the 0th tier.
            max_e_period_in_matrix = np.max([self.e_wkday_12by24, self.e_wkend_12by24])
            max_e_period_in_prices = np.shape(self.e_prices)[1]
            for period in np.arange(max_e_period_in_prices, max_e_period_in_matrix+1, 1):
                self.e_wkday_12by24[self.e_wkday_12by24==period] = 0
                self.e_wkend_12by24[self.e_wkend_12by24==period] = 0

            ######################## Repackage Demand Schedule ###########################
            self.d_wkday_12by24 = np.zeros([12,24], int)
            self.d_wkend_12by24 = np.zeros([12,24], int)

            if 'demandweekdayschedule' in tariff_original:
                for month in range(12):
                    self.d_wkday_12by24[month, :] = tariff_original['demandweekdayschedule'][month]
                    self.d_wkend_12by24[month, :] = tariff_original['demandweekendschedule'][month]

            # If the urdb 12by24 has a period that isn't defined in the tiers,
            # set that period to the 0th tier.
            max_d_period_in_matrix = np.max([self.d_wkday_12by24, self.d_wkend_12by24])
            max_d_period_in_prices = np.shape(self.d_tou_prices)[1]
            for period in np.arange(max_d_period_in_prices, max_d_period_in_matrix+1, 1):
                self.d_wkday_12by24[self.d_wkday_12by24==period] = 0
                self.d_wkend_12by24[self.d_wkend_12by24==period] = 0

            ################### Repackage 12x24s as 8760s Schedule ########################
            self.start_day = start_day
            self.d_tou_8760 = build_8760_from_12by24s(self.d_wkday_12by24, self.d_wkend_12by24, self.start_day)
            self.e_tou_8760 = build_8760_from_12by24s(self.e_wkday_12by24, self.e_wkend_12by24, self.start_day)

            ######################## Precalculations ######################################
            # Collapse the tiered price matrix down to just the maximum cost
            # in each tier, to be used during dispatch.
            self.e_prices_no_tier = np.max(self.e_prices, 0)

            # Determine the maximum differential in energy price within a day.
            e_12by24_max_prices_wkday = self.e_prices_no_tier[self.e_wkday_12by24]
            e_12by24_max_prices_wkend = self.e_prices_no_tier[self.e_wkend_12by24]
            e_max_price_differential_wkday = np.max(e_12by24_max_prices_wkday, 1) - np.min(e_12by24_max_prices_wkday, 1)
            e_max_price_differential_wkend = np.max(e_12by24_max_prices_wkend, 1) - np.min(e_12by24_max_prices_wkend, 1)
            self.e_max_difference = np.max([e_max_price_differential_wkday, e_max_price_differential_wkend])

        #######################################################################
        # If given a json input argument, construct a tariff from that file
        #######################################################################
        elif json_file_name != None:

            obj_text = codecs.open(json_file_name, 'r', encoding='utf-8').read()
            d = json.loads(obj_text)
            for fieldname in list(d.keys()):
                if isinstance(d[fieldname], list):
                    d[fieldname] = np.array(d[fieldname])

            if 'urdb_id' in d: self.urdb_id = d['urdb_id']
            if 'name' in d: self.name = d['name']
            if 'utility' in d: self.utility = d['utility']
            if 'fixed_charge' in d: self.fixed_charge = d['fixed_charge']
            if 'peak_kW_capacity_max' in d: self.peak_kW_capacity_max = d['peak_kW_capacity_max']
            if 'peak_kW_capacity_min' in d: self.peak_kW_capacity_min = d['peak_kW_capacity_min']
            if 'kWh_useage_max' in d: self.kWh_useage_max = d['kWh_useage_max']
            if 'kWh_useage_min' in d: self.kWh_useage_min = d['kWh_useage_min']
            if 'sector' in d: self.sector = d['sector']
            if 'comments' in d: self.comments = d['comments']
            if 'description' in d: self.description = d['description']
            if 'source' in d: self.source = d['source']
            if 'uri' in d: self.uri = d['uri']
            if 'source' in d: self.source = d['source']
            if 'voltage_category' in d: self.voltage_category = d['voltage_category']
            if 'eia_id' in d: self.eia_id = d['eia_id']
            if 'energy_rate_unit' in d: self.energy_rate_unit = d['energy_rate_unit']
            if 'max_demand_charge' in d: self.max_demand_charge = d['max_demand_charge']

            ###################### Blank Flat Demand Structure ########################
            if 'd_flat_exists' in d: self.d_flat_exists = d['d_flat_exists']
            if 'd_flat_prices' in d: self.d_flat_prices = d['d_flat_prices']
            if 'd_flat_levels' in d: self.d_flat_levels = d['d_flat_levels']
            if 'd_flat_n' in d: self.d_flat_n = d['d_flat_n']

            #################### Blank Demand TOU Structure ###########################
            if 'd_tou_exists' in d: self.d_tou_exists = d['d_tou_exists']
            if 'd_tou_n' in d: self.d_tou_n = d['d_tou_n']
            if 'd_tou_prices' in d: self.d_tou_prices = d['d_tou_prices']
            if 'd_tou_levels' in d: self.d_tou_levels = d['d_tou_levels']

            #################### Coincident Peak Structure ###########################
            if 'coincident_peak_exists' in d: self.coincident_peak_exists = d['coincident_peak_exists']
            else: self.coincident_peak_exists = False

            if 'coincident_style' in d: self.coincident_style = d['coincident_style']
            if 'coincident_hour_def' in d: self.coincident_hour_def = d['coincident_hour_def']
            if 'coincident_prices' in d: self.coincident_prices = d['coincident_prices']
            if 'coincident_levels' in d: self.coincident_levels = d['coincident_levels']
            if 'coincident_monthly_periods' in d: self.coincident_monthly_periods = d['coincident_monthly_periods']

            ######################## Blank Energy Structure ###########################
            if 'e_exists' in d: self.e_exists = d['e_exists']
            if 'e_tou_exists' in d: self.e_tou_exists = d['e_tou_exists']
            if 'e_n' in d: self.e_n = d['e_n']
            if 'e_prices' in d: self.e_prices = d['e_prices']
            if 'e_levels' in d: self.e_levels = d['e_levels']

            ######################## Blank Schedules ###########################
            if 'e_wkday_12by24' in d: self.e_wkday_12by24 = d['e_wkday_12by24']
            if 'e_wkend_12by24' in d: self.e_wkend_12by24 = d['e_wkend_12by24']
            if 'd_wkday_12by24' in d: self.d_wkday_12by24 = d['d_wkday_12by24']
            if 'd_wkend_12by24' in d: self.d_wkend_12by24 = d['d_wkend_12by24']

            ################### Blank 12x24s as 8760s Schedule ########################
            if 'd_tou_8760' in d: self.d_tou_8760 = d['d_tou_8760']
            if 'e_tou_8760' in d: self.e_tou_8760 = d['e_tou_8760']

            ######################## Precalculations ######################################
            if 'e_prices_no_tier' in d: self.e_prices_no_tier = d['e_prices_no_tier']
            if 'e_max_difference' in d: self.e_max_difference = d['e_max_difference']
            if 'start_day' in d: self.start_day = d['start_day']

        #######################################################################
        # If given a dict input, construct a tariff from that object
        #######################################################################
        elif not isinstance(dict_obj,type(None)):
            if 'start_day' in dict_obj: self.start_day = dict_obj['start_day']
            else: self.start_day = 6

            if 'urdb_id' in dict_obj: self.urdb_id = dict_obj['urdb_id']
            if 'name' in dict_obj: self.name = dict_obj['name']
            if 'utility' in dict_obj: self.utility = dict_obj['utility']
            if 'sector' in dict_obj: self.sector = dict_obj['sector']
            if 'comments' in dict_obj: self.comments = dict_obj['comments']
            if 'description' in dict_obj: self.description = dict_obj['description']
            if 'source' in dict_obj: self.source = dict_obj['source']
            if 'uri' in dict_obj: self.uri = dict_obj['uri']
            if 'voltage_category' in dict_obj: self.voltage_category = dict_obj['voltage_category']
            if 'fixed_charge' in dict_obj: self.fixed_charge = dict_obj['fixed_charge']
            if 'peak_kW_capacity_max' in dict_obj: self.peak_kW_capacity_max = dict_obj['peak_kW_capacity_max']
            if 'peak_kW_capacity_min' in dict_obj: self.peak_kW_capacity_min = dict_obj['peak_kW_capacity_min']
            if 'kWh_useage_max' in dict_obj: self.kWh_useage_max = dict_obj['kWh_useage_max']
            if 'kWh_useage_min' in dict_obj: self.kWh_useage_min = dict_obj['kWh_useage_min']
            if 'eia_id' in dict_obj: self.eia_id = dict_obj['eia_id']
            if 'demand_rate_unit' in dict_obj: self.demand_rate_unit = dict_obj['demand_rate_unit']
            if 'energy_rate_unit' in dict_obj: self.energy_rate_unit = dict_obj['energy_rate_unit']

            ###################### Flat Demand Structure ########################
            if 'd_flat_exists' in dict_obj: self.d_flat_exists = dict_obj['d_flat_exists']
            if 'd_flat_n' in dict_obj: self.d_flat_n = dict_obj['d_flat_n']
            if 'd_flat_prices' in dict_obj:
                self.d_flat_prices = np.array(dict_obj['d_flat_prices'])
                if 'd_flat_levels' in dict_obj: self.d_flat_levels = np.array(dict_obj['d_flat_levels'])
                else: self.d_flat_levels = np.zeros([1,np.shape(self.d_flat_prices)[1]]) + 1e9

            #################### Demand TOU Structure ###########################
            if 'd_tou_exists' in dict_obj: self.d_tou_exists = dict_obj['d_tou_exists']
            if 'd_tou_n' in dict_obj: self.d_tou_n = dict_obj['d_tou_n']
            if 'd_tou_prices' in dict_obj:
                self.d_tou_prices = np.array(dict_obj['d_tou_prices'])
                if 'd_tou_levels' in dict_obj: self.d_tou_levels = np.array(dict_obj['d_tou_levels'])
                else: self.d_tou_levels = np.zeros([1,np.shape(self.d_tou_prices)[1]]) + 1e9

            #################### Coincident Peak Structure ###########################
            if 'coincident_style' in dict_obj: self.coincident_style = dict_obj['coincident_style']
            if 'coincident_hour_def' in dict_obj: self.coincident_hour_def = dict_obj['coincident_hour_def']
            if 'coincident_prices' in dict_obj: self.coincident_prices = dict_obj['coincident_prices']
            if 'coincident_levels' in dict_obj: self.coincident_levels = dict_obj['coincident_levels']
            if 'coincident_monthly_periods' in dict_obj: self.coincident_monthly_periods = dict_obj['coincident_monthly_periods']

            ######################## Energy Structure ###########################
            if 'e_exists' in dict_obj: self.e_exists = dict_obj['e_exists']
            if 'e_tou_exists' in dict_obj: self.e_tou_exists = dict_obj['e_tou_exists']
            if 'e_n' in dict_obj: self.e_n = dict_obj['e_n']
            if 'e_prices' in dict_obj:
                self.e_prices = np.array(dict_obj['e_prices'])
                if 'e_levels' in dict_obj: self.e_levels = np.array(dict_obj['e_levels'])
                else: self.e_levels = np.zeros([1,np.shape(self.e_prices)[1]]) + 1e9

            ######################## Schedules ###########################
            if 'e_wkday_12by24' in dict_obj: self.e_wkday_12by24 = np.array(dict_obj['e_wkday_12by24'])
            if 'e_wkend_12by24' in dict_obj: self.e_wkend_12by24 = np.array(dict_obj['e_wkend_12by24'])
            if 'd_wkday_12by24' in dict_obj: self.d_wkday_12by24 = np.array(dict_obj['d_wkday_12by24'])
            if 'd_wkend_12by24' in dict_obj: self.d_wkend_12by24 = np.array(dict_obj['d_wkend_12by24'])

            # If the 12by24 has a period that isn't defined in the tiers,
            # set that period to the 0th tier.
            if 'e_wkday_12by24' in dict_obj:
                max_e_period_in_matrix = np.max([self.e_wkday_12by24, self.e_wkend_12by24])
                max_e_period_in_prices = np.shape(self.e_prices)[1]
                for period in np.arange(max_e_period_in_prices, max_e_period_in_matrix+1, 1):
                    self.e_wkday_12by24[self.e_wkday_12by24==period] = 0
                    self.e_wkend_12by24[self.e_wkend_12by24==period] = 0

            # If the 12by24 has a period that isn't defined in the tiers,
            # set that period to the 0th tier.
            if 'd_wkday_12by24' in dict_obj:
                max_d_period_in_matrix = np.max([self.d_wkday_12by24, self.d_wkend_12by24])
                max_d_period_in_prices = np.shape(self.d_tou_prices)[1]
                for period in np.arange(max_d_period_in_prices, max_d_period_in_matrix+1, 1):
                    self.d_wkday_12by24[self.d_wkday_12by24==period] = 0
                    self.d_wkend_12by24[self.d_wkend_12by24==period] = 0

            ################### 12x24s as 8760s Schedule ########################
            # Build 8760's. Note that any ingested 8760's will be ignored
            if 'd_wkday_12by24' in dict_obj:
                self.d_tou_8760 = build_8760_from_12by24s(self.d_wkday_12by24, self.d_wkend_12by24, self.start_day)

            if 'e_wkday_12by24' in dict_obj:
                self.e_tou_8760 = build_8760_from_12by24s(self.e_wkday_12by24, self.e_wkend_12by24, self.start_day)

            ######################## Precalculations ######################################
            if 'e_prices' in dict_obj: self.e_max_difference = np.max(self.e_prices) - np.min(self.e_prices)
            if 'e_prices' in dict_obj: self.e_prices_no_tier = np.max(self.e_prices, 0)

    #######################################################################
    # Write the current class object to a json file
    #######################################################################
    def write_json(self, json_file_name):

        d = self.__dict__

        d_prep_for_json = d.copy()

        # change ndarray dtypes to lists, since json doesn't know ndarrays
        for fieldname in list(d_prep_for_json.keys()):
            if isinstance(d_prep_for_json[fieldname], np.ndarray):
                d_prep_for_json[fieldname] = d_prep_for_json[fieldname].tolist()

        with open(json_file_name, 'w') as fp:
            json.dump(d_prep_for_json, fp)

    #######################################################################
    # Define TOU demand charge periods, levels, and prices
    #######################################################################
    def define_d_tou(self, d_wkday_12by24, d_wkend_12by24, d_tou_levels, d_tou_prices):

        self.d_tou_levels = d_tou_levels
        self.d_tou_prices = d_tou_prices
        self.d_wkday_12by24 = d_wkday_12by24
        self.d_wkend_12by24 = d_wkend_12by24
        self.d_tou_n = int(np.shape(d_tou_levels)[1])
        if np.count_nonzero(d_tou_prices) == 0: self.d_tou_exists = False
        else: self.d_tou_exists = True

        self.d_tou_8760 = build_8760_from_12by24s(d_wkday_12by24, d_wkend_12by24, self.start_day)

    #######################################################################
    # Define Flat demand charge periods, levels, and prices
    #######################################################################
    def define_d_flat(self, d_flat_levels, d_flat_prices):

        # If it is only handed one value for levels and prices, it assumes that
        # value applies to all months
        if np.size(d_flat_prices) == 1:
            self.d_flat_levels = np.array([[d_flat_levels]]).repeat(12).reshape(1,12)
            self.d_flat_prices = np.array([[d_flat_prices]]).repeat(12).reshape(1,12)
            self.d_flat_n = 1
        else:
            self.d_flat_levels = d_flat_levels
            self.d_flat_prices = d_flat_prices
            self.d_flat_n = np.size(np.unique(d_flat_prices))

        if np.all(d_flat_prices==0): self.d_flat_exists = False
        else: self.d_flat_exists = True

    #######################################################################
    # Define energy periods, levels, and prices
    #######################################################################
    def define_e(self, e_wkday_12by24, e_wkend_12by24, e_levels, e_prices):

        self.e_levels = e_levels
        self.e_prices = e_prices
        self.e_wkday_12by24 = e_wkday_12by24
        self.e_wkend_12by24 = e_wkend_12by24
        self.e_n = int(np.shape(e_levels)[1])

        if np.count_nonzero(e_prices) == 0: self.e_exists = False
        else: self.e_exists = True

        if np.any(e_wkday_12by24 != np.repeat(e_wkday_12by24[:,0],24).reshape(12,24)): # TODO: add weekends
            self.e_tou_exists = True
        else:
            self.d_tou_exists = False

        self.e_tou_8760 = build_8760_from_12by24s(e_wkday_12by24, e_wkend_12by24, self.start_day)

        ######################## Precalculations ######################################
        # Collapse the tiered price matrix down to just the maximum cost
        # in each tier, to be used during dispatch.
        self.e_prices_no_tier = np.max(self.e_prices, 0)

        # Determine the maximum differential in energy price within a day.
        e_12by24_max_prices_wkday = self.e_prices_no_tier[self.e_wkday_12by24]
        e_12by24_max_prices_wkend = self.e_prices_no_tier[self.e_wkend_12by24]
        e_max_price_differential_wkday = np.max(e_12by24_max_prices_wkday, 1) - np.min(e_12by24_max_prices_wkday, 1)
        e_max_price_differential_wkend = np.max(e_12by24_max_prices_wkend, 1) - np.min(e_12by24_max_prices_wkend, 1)
        self.e_max_difference = np.max([e_max_price_differential_wkday, e_max_price_differential_wkend])

    #######################################################################
    # Identify the maximum demand charge for this tariff
    #######################################################################
    def identify_max_demand_charge(self):

        # identify the max demand charge for each period for both flat and tou
        max_d_flat_prices = np.max(self.d_flat_prices, 0)
        max_d_tou_prices = np.max(self.d_tou_prices, 0)

        # recast the 12by24 in terms of the demand charge within each cell
        d_wkday_12by24_prices = max_d_tou_prices[self.d_wkday_12by24]
        d_wkend_12by24_prices = max_d_tou_prices[self.d_wkend_12by24]

        # add each month's flat charge to the corresponding row of the 12by24
        d_wkday_12by24_prices += max_d_flat_prices.reshape(12,1)
        d_wkend_12by24_prices += max_d_flat_prices.reshape(12,1)

        # determine the max between wkends and wkdays
        d_12by24_prices = np.maximum(d_wkday_12by24_prices, d_wkend_12by24_prices)

        # determine max demand charge
        self.max_demand_charge = np.max(d_12by24_prices)

#%%
class Export_Tariff:
    """
    Structure of compensation for exported generation. Currently only two
    styles: full-retail NEM, and instantanous TOU energy value.
    """

    def __init__(self, full_retail_nem=True,
                 prices = np.zeros([1, 1], float),
                 levels = np.zeros([1, 1], float),
                 periods_8760 = np.zeros(8760, int),
                 period_tou_n = 1):

        self.full_retail_nem = full_retail_nem
        self.prices = prices
        self.levels = levels
        self.periods_8760 = periods_8760
        self.period_tou_n = period_tou_n

    def set_constant_sell_price(self, price):
        self.full_retail_nem = False
        self.prices = np.array([[price]], float)
        self.levels = np.array([[9999999]], float)
        self.periods_8760 = np.zeros(8760, int)
        self.period_tou_n = 1

#%%
def tiered_calc_vec(values, levels, prices):
    # Vectorized piecewise function calculator
    values = np.asarray(values)
    levels = np.asarray(levels)
    prices = np.asarray(prices)
    y = np.zeros(values.shape)

    # Credit at tier 1 for negative values
    y = y + ((values < 0)) * (values*prices[:][:][0])

    # Tier 1
    y = y + ((values >= 0) & (values < levels[:][:][0])) * (values*prices[:][:][0])

    # Tiers 2 and beyond
    for tier in np.arange(1,np.size(levels,0)):
        y = y +  ((values >= levels[:][:][tier-1]) & (values < levels[:][:][tier])) * (
            ((values-levels[:][:][tier-1])*prices[:][:][tier]) + levels[:][:][tier-1]*prices[:][:][tier-1])

    return y

#%%

def bill_calculator(load_profile, tariff, export_tariff):
    """
    Deprecated. Nullified by new PySAM code but kept for reference.

    Parameters
    ----------
    load_profile : 8760 profile of agent
    tariff : :class:`python.tariff_functions.Tariff`
        Tariff class object
    export_tariff : :class:`python.tariff_functions.Export_Tariff`
        Export tariff class object
    """

    n_months = 12

    # Check if a window length is specified, assume hourly if none is given
    if hasattr(tariff, 'window_length_hours') == False: tariff.window_length_hours = 1.0

    # If load profile resolution is greater than necessary, average it.
    if len(load_profile) > int(8760 / tariff.window_length_hours):
        load_profile = np.array(load_profile).reshape(-1, int(len(load_profile)/int(8760 / tariff.window_length_hours))).mean(axis=1)

    # Note that if load profile resolution is lower than window resolution, it
    # will calculate the bill at the lower resolution
    n_timesteps = len(load_profile)

    # If necessary, adjust the resolution of the 8760 period vectors
    if hasattr(tariff, 'd_tou_8760'):
        if len(tariff.d_tou_8760) > n_timesteps: tariff.d_tou_8760 = np.array(tariff.d_tou_8760.reshape(-1, len(tariff.d_tou_8760)/n_timesteps).mean(axis=1), int)
        if len(tariff.d_tou_8760) < n_timesteps:
            temp_array = np.zeros([len(tariff.d_tou_8760), n_timesteps/len(tariff.d_tou_8760)], int)
            temp_array[:,:] = tariff.d_tou_8760.reshape(len(tariff.d_tou_8760),1)
            tariff.d_tou_8760 = temp_array.reshape(n_timesteps)

    if hasattr(tariff, 'e_tou_8760'):
        if len(tariff.e_tou_8760) > n_timesteps: tariff.e_tou_8760 = np.array(tariff.e_tou_8760.reshape(-1, len(tariff.e_tou_8760)/n_timesteps).mean(axis=1), int)
        if len(tariff.e_tou_8760) < n_timesteps:
            temp_array = np.zeros([len(tariff.e_tou_8760), n_timesteps/len(tariff.e_tou_8760)], int)
            temp_array[:,:] = tariff.e_tou_8760.reshape(len(tariff.e_tou_8760),1)
            tariff.e_tou_8760 = temp_array.reshape(n_timesteps)

    if hasattr(export_tariff, 'periods_8760'):
        if len(export_tariff.periods_8760) > n_timesteps: export_tariff.periods_8760 = np.array(export_tariff.periods_8760.reshape(-1, len(export_tariff.periods_8760)/n_timesteps).mean(axis=1), int)
        if len(export_tariff.periods_8760) < n_timesteps:
            temp_array = np.zeros([len(export_tariff.periods_8760), n_timesteps/len(export_tariff.periods_8760)], int)
            temp_array[:,:] = export_tariff.periods_8760.reshape(len(export_tariff.periods_8760),1)
            export_tariff.periods_8760 = temp_array.reshape(n_timesteps)

    # 8760 vector of month numbers
    month_hours = np.array([0, 744, 1416, 2160, 2880, 3624, 4344, 5088, 5832, 6552, 7296, 8016, 8760], int) * n_timesteps//8760
    month_index = np.zeros(n_timesteps, int)
    for month, hours in enumerate(month_hours):
        month_index[month_hours[month-1]:hours] = month-1

    # Temporary patch of daily kWh tiers
    if hasattr(tariff, 'energy_rate_unit'):
        if tariff.energy_rate_unit == 'kWh daily': tariff.e_levels = np.array(tariff.e_levels) * 30.0

    #=========================================================================#
    ################## Calculate TOU Demand Charges ###########################
    #=========================================================================#
    if hasattr(tariff, 'd_tou_8760'):
        # Cast the TOU periods into a boolean matrix
        d_tou_n = np.shape(tariff.d_tou_prices)[1]
        period_matrix = np.zeros([n_timesteps, d_tou_n*n_months], bool)
        period_matrix[list(range(n_timesteps)),tariff.d_tou_8760+month_index*d_tou_n] = True

        # Determine the max demand in each period of each month of each year
        load_distributed = load_profile[np.newaxis, :].T*period_matrix
        period_maxs = np.max(load_distributed, axis=0)

        # Calculate the cost of TOU demand charges
        d_TOU_period_charges = tiered_calc_vec(period_maxs, np.tile(tariff.d_tou_levels[:,0:d_tou_n], 12), np.tile(tariff.d_tou_prices[:,0:d_tou_n], 12))

        d_TOU_month_total_charges = np.zeros([n_months])
        for month in range(n_months):
            d_TOU_month_total_charges[month] = np.sum(d_TOU_period_charges[(month*d_tou_n):(month*d_tou_n + d_tou_n)])
    else:
        d_TOU_month_total_charges = np.zeros([n_months])
        period_maxs = np.zeros(0)

    #=========================================================================#
    ################# Calculate Flat Demand Charges ###########################
    #=========================================================================#
    if hasattr(tariff, 'd_flat_prices'):
        # Cast the seasons into a boolean matrix
        flat_matrix = np.zeros([n_timesteps, n_months], bool)
        flat_matrix[list(range(n_timesteps)),month_index] = True

        # Determine the max demand in each month of each year
        load_distributed = load_profile[np.newaxis, :].T*flat_matrix
        flat_maxs = np.max(load_distributed, axis=0)

        flat_charges = tiered_calc_vec(flat_maxs, tariff.d_flat_levels, tariff.d_flat_prices)
    else:
        flat_charges = np.zeros([n_months])
        flat_maxs = np.zeros(0)

    #=========================================================================#
    ############# Calculate Coincident Peak Demand Charges ####################
    #=========================================================================#
    if hasattr(tariff, 'coincident_style'):
        if tariff.coincident_style == 0:
            # Input is a n by m array. Each row is a peak period and each set
            # of columns are the hours that define that period. For example,
            # [[100,200],[5100,5200]] would have two periods that are defined
            # by the average demand of hours [100,200] and [5100,5200]
            # respectively.
            # Coincident_monthly_periods is a 12-length array that maps the
            # charges to the billing periods.
            coincident_demand_levels = np.average(load_profile[tariff.coincident_hour_def], 1)
            coincident_charges = tiered_calc_vec(coincident_demand_levels, tariff.coincident_levels, tariff.coincident_prices)
            coincident_monthly_charges = coincident_charges[tariff.coincident_monthly_periods]
    else:
        coincident_monthly_charges = np.zeros(12)
        coincident_demand_levels = None

    #=========================================================================#
    #################### Calculate Energy Charges #############################
    #=========================================================================#
    # Calculate energy charges without full retail NEM
    if hasattr(tariff, 'e_tou_8760'):
        e_n = np.shape(tariff.e_prices)[1]
        if export_tariff.full_retail_nem == False:
            imported_profile = np.clip(load_profile, 0, 1e99)
            exported_profile = np.clip(load_profile, -1e99, 0)

            # Calculate fixed schedule export_tariff
            # Cast the TOU periods into a boolean matrix
            e_period_export_matrix = np.zeros([len(export_tariff.periods_8760), export_tariff.period_tou_n*n_months], bool)
            e_period_export_matrix[list(range(len(export_tariff.periods_8760))),export_tariff.periods_8760+month_index*export_tariff.period_tou_n] = True

            # Determine the energy consumed in each period of each month of each year
            load_distributed = exported_profile[np.newaxis, :].T*e_period_export_matrix
            export_period_sums = np.sum(load_distributed, axis=0)

            # Calculate the cost of TOU demand charges
            export_period_credits = tiered_calc_vec(export_period_sums, np.tile(export_tariff.levels[:,0:export_tariff.period_tou_n], 12), np.tile(export_tariff.prices[:,0:export_tariff.period_tou_n], 12))

            export_month_total_credits = np.zeros([n_months])
            for month in range(n_months):
                export_month_total_credits[month] = np.sum(export_period_credits[(month*export_tariff.period_tou_n):(month*export_tariff.period_tou_n + export_tariff.period_tou_n)])

            # Calculate imported energy charges.
            # Cast the TOU periods into a boolean matrix
            e_period_import_matrix = np.zeros([len(tariff.e_tou_8760), e_n*n_months], bool)
            e_period_import_matrix[list(range(len(tariff.e_tou_8760))),tariff.e_tou_8760+month_index*e_n] = True

            # Determine the max demand in each period of each month of each year
            load_distributed = imported_profile[np.newaxis, :].T*e_period_import_matrix
            e_period_import_sums = np.sum(load_distributed, axis=0)

            # Calculate the cost of TOU demand charges
            e_period_import_charges = tiered_calc_vec(e_period_import_sums, np.tile(tariff.e_levels, 12), np.tile(tariff.e_prices, 12))

            e_month_import_total_charges = np.zeros([n_months])
            for month in range(n_months):
                e_month_import_total_charges[month] = np.sum(e_period_import_charges[(month*e_n):(month*e_n + e_n)])

            export_month_total_credits = np.negative(export_month_total_credits)
            e_month_total_net_charges = e_month_import_total_charges - export_month_total_credits

            # placeholder
            e_period_charges = "placeholder"
            e_period_sums = "placeholder"

        # Calculate energy charges with full retail NEM
        else:
            # Calculate imported energy charges with full retail NEM
            # Cast the TOU periods into a boolean matrix
            e_period_matrix = np.zeros([len(tariff.e_tou_8760), e_n*n_months], bool)
            e_period_matrix[list(range(len(tariff.e_tou_8760))),tariff.e_tou_8760+month_index*e_n] = True

            # Determine the energy consumed in each period of each month of each year netting exported electricity
            load_distributed = load_profile[np.newaxis, :].T*e_period_matrix
            e_period_sums = np.sum(load_distributed, axis=0)

            # Calculate the cost of TOU energy charges netting exported electricity
            e_period_charges = tiered_calc_vec(e_period_sums, np.tile(tariff.e_levels, 12), np.tile(tariff.e_prices, 12))

            e_month_total_net_charges = np.zeros([n_months])
            for month in range(n_months):
                e_month_total_net_charges[month] = np.sum(e_period_charges[(month*e_n):(month*e_n + e_n)])

            # Determine the value of NEM
            # Calculate imported energy charges with zero exported electricity
            imported_profile = np.clip(load_profile, 0, 1e99)

            # Determine the energy consumed in each period of each month of each year - without exported electricity
            imported_load_distributed = imported_profile[np.newaxis, :].T*e_period_matrix
            e_period_sums_imported = np.sum(imported_load_distributed, axis=0)

            # Calculate the cost of TOU energy charges without exported electricity
            e_period_imported_charges = tiered_calc_vec(e_period_sums_imported, np.tile(tariff.e_levels, 12), np.tile(tariff.e_prices, 12))

            e_month_total_import_charges = np.zeros([n_months])
            for month in range(n_months):
                e_month_total_import_charges[month] = np.sum(e_period_imported_charges[(month*e_n):(month*e_n + e_n)])

            # Determine how much  the exported electricity was worth by comparing
            # bills where it was netted against those where it wasn't
            export_month_total_credits = e_month_total_net_charges - e_month_total_import_charges

            e_period_import_sums = 'placeholder'
    else:
        e_month_total_net_charges = np.zeros(12)
        export_month_total_credits = np.zeros(12)
        e_period_charges = np.zeros(12*e_n)
        e_period_sums = np.zeros(12*e_n)
        e_period_import_sums = np.zeros(12*e_n)

    total_monthly_bills = d_TOU_month_total_charges + flat_charges + coincident_monthly_charges + e_month_total_net_charges + tariff.fixed_charge
    annual_bill = np.sum(total_monthly_bills)

    results_dict = {'annual_bill':annual_bill,
                    'd_charges':np.sum(d_TOU_month_total_charges + flat_charges),
                    'e_charges':np.sum(e_month_total_net_charges),
                    'fixed_charges':tariff.fixed_charge*12,
                    'monthly_total_bills':total_monthly_bills,
                    'monthly_d_charges':d_TOU_month_total_charges + flat_charges,
                    'monthly_d_tou_charges':d_TOU_month_total_charges,
                    'monthly_d_flat_charges':flat_charges,
                    'monthly_e_total_net_charges':e_month_total_net_charges,
                    'monthly_e_total_import_charges':e_month_total_net_charges-export_month_total_credits,
                    'monthly_e_total_export_credits':export_month_total_credits,
                    'period_kW_maxs':period_maxs,
                    'monthly_kW_maxs':flat_maxs,
                    'period_e_charges':e_period_charges,
                    'period_e_sums':e_period_sums,
                    'e_period_import_sums':e_period_import_sums,
                    'coincident_monthly_charges':coincident_monthly_charges,
                    'coincident_demand_levels':coincident_demand_levels
                    }

    return annual_bill, results_dict

#%%
# Bulk Downloader from URDB API
def download_tariffs_from_urdb(api_key, sector=None, utility=None, print_progress=False):
    '''
    API request for URDB rates.
    Each user should get their own URDB API key: http://en.openei.org/services/api/signup/
    Sectors: Residential, Commercial, Industrial, Lighting

    Parameters
    ----------
    api_key : str
        Each user should get their own URDB API key: http://en.openei.org/services/api/signup/
    sector : str
        One of Residential, Commercial, Industrial, Lighting
    utility : str, optional
    Return
    ------
    pandas.DataFrame
        Dataframe of URDB rates.

    '''

    fields = ['utility',
              'eiaid',
              'name',
              'label',
              'enddate',
              'demandrateunit',
              'flatdemandunit',
              'uri',
              'sector',
              'description',
              'source',
              'peakkwcapacitymax',
              'peakkwcapacitymin',
              'peakkwhuseagemax',
              'peakkwhuseagemin',
              'voltagecategory',
              'phasewiring']

    tariffs = pd.DataFrame(columns=fields)

    flag = True
    offset = 0
    chunk_count = 0

    input_params = {'version':3,
                'format':'json',
                'detail':'full',
                'limit':500,
                'api_key':api_key}

    if sector != None: input_params['sector'] = sector
    if utility != None: input_params['ratesforutility'] = utility

    while flag == True:
        input_params['offset'] = offset

        r = req.get('http://api.openei.org/utility_rates?', params=input_params)

        content = r.content
        tariff_list = json.loads(content, strict=False)['items']

        if len(tariff_list) == 0:
            flag = False
        else:
            tariff_chunk = pd.DataFrame(index=list(range(500)), columns=fields)
            for count, tariff in enumerate(tariff_list):
                if 'utility' in tariff: tariff_chunk.loc[count, 'utility'] = tariff['utility'].encode('utf-8')
                if 'eiaid' in tariff: tariff_chunk.loc[count, 'eiaid'] = tariff['eiaid']
                if 'name' in tariff: tariff_chunk.loc[count, 'name'] = tariff['name'].encode('utf-8')
                if 'label' in tariff: tariff_chunk.loc[count, 'label'] = tariff['label']
                if 'enddate' in tariff: tariff_chunk.loc[count, 'enddate'] = tariff['enddate']
                if 'demandrateunit' in tariff: tariff_chunk.loc[count, 'demandrateunit'] = tariff['demandrateunit']
                if 'flatdemandunit' in tariff: tariff_chunk.loc[count, 'flatdemandunit'] = tariff['flatdemandunit']
                if 'uri' in tariff: tariff_chunk.loc[count, 'uri'] = tariff['uri'].encode('utf-8')
                if 'sector' in tariff: tariff_chunk.loc[count, 'sector'] = tariff['sector'].encode('utf-8')
                if 'description' in tariff: tariff_chunk.loc[count, 'description'] = tariff['description'].encode('utf-8')
                if 'source' in tariff: tariff_chunk.loc[count, 'source'] = tariff['source'].encode('utf-8')
                if 'peakkwcapacitymax' in tariff: tariff_chunk.loc[count, 'peakkwcapacitymax'] = tariff['peakkwcapacitymax']
                if 'peakkwcapacitymin' in tariff: tariff_chunk.loc[count, 'peakkwcapacitymin'] = tariff['peakkwcapacitymin']
                if 'peakkwhuseagemax' in tariff: tariff_chunk.loc[count, 'peakkwhuseagemax'] = tariff['peakkwhuseagemax']
                if 'peakkwhuseagemin' in tariff: tariff_chunk.loc[count, 'peakkwhuseagemin'] = tariff['peakkwhuseagemin']
                if 'voltagecategory' in tariff: tariff_chunk.loc[count, 'voltagecategory'] = tariff['voltagecategory'].encode('utf-8')
                if 'phasewiring' in tariff: tariff_chunk.loc[count, 'phasewiring'] = tariff['phasewiring'].encode('utf-8')

            tariffs = tariffs.append(tariff_chunk.loc[:count, :], ignore_index=True)
            offset += len(tariff_list)

            chunk_count += len(tariff_list)
            if print_progress==True: print(chunk_count)

    return tariffs

#%%
# Filter tariff_df by a list of keywords in the tariff names, and unit types

def filter_tariff_df(tariff_df,
                     keyword_list=None,
                     keyword_list_file=None,
                     demand_units_to_exclude=['hp', 'kVA', 'kW daily', 'hp daily', 'kVA daily'],
                     remove_expired=True):

    """
    Filter tariffs based on inclusion (e.g. keywords), or exclusion (e.g. demand units)
    Parameters
    ----------
    tariff_df : pandas.DataFrame
        dataframe of URDB tariffs created by :func:`download_tariffs_from_urdb`.
    keyword_list : list of str, optional
        list of keywords to search for in rate structure.
    keyword_list_file : str
        filepath to .txt file containing keywords to search for.
    demand_units_to_exclude : list of str
        exclude rates from URDB database if the units are contained in this list. Default values are `hp`,`kVA`,`kW daily`,`hp daily`,`kVA daily`
    remove_expired : bool
        exclude expired rates. Default is `True`.
    """

    if keyword_list_file != None:
        keyword_list = []
        with open(keyword_list_file, 'rb') as f:
            reader = csv.reader(f)
            for item in reader:
                keyword_list = keyword_list + item
    elif keyword_list != None:
        keyword_list = keyword_list
    else:
        print('enter a keyword_list or keyword_list_file')

    tariffs_to_exclude = np.zeros(len(tariff_df), bool)
    keyword_count_df = pd.DataFrame(index=keyword_list)

    for keyword in keyword_list:
        tariffs_that_contain_keyword = tariff_df['name'].str.contains(keyword, case=False)
        tariffs_to_exclude = tariffs_that_contain_keyword + tariffs_to_exclude
        keyword_count_df.loc[keyword, 'num_of_tariffs_excluded'] = np.sum(tariffs_that_contain_keyword)

    for demand_unit in demand_units_to_exclude:
        tariffs_that_contain_unit = tariff_df['demandrateunit'] == demand_unit
        tariffs_that_contain_unit += tariff_df['flatdemandunit'] == demand_unit
        tariffs_to_exclude = tariffs_to_exclude + tariffs_that_contain_unit

    tariffs_with_an_end_date = pd.isnull(tariff_df['enddate']) == False

    tariffs_to_exclude = tariffs_to_exclude + tariffs_with_an_end_date

    excluded_tariffs = tariff_df[tariffs_to_exclude==True]
    included_tariffs = tariff_df[tariffs_to_exclude==False]

    return included_tariffs, excluded_tariffs, keyword_count_df

#%%
# Create 8760 from two 12x24's
def build_8760_from_12by24s(wkday_12by24, wkend_12by24, start_day=6):
    '''
    Construct long-df (8760) from a weekday and weekend 12by24

    Parameters
    ----------
    wkday_12by24 : numpy.ndarray
    wkend_12by24 : numpy.ndarray
    start_day : int
        Start day of 6 (default) equates to a Sunday.
    '''

    month_hours = np.array([0, 744, 1416, 2160, 2880, 3624, 4344, 5088, 5832, 6552, 7296, 8016, 8760], int)

    month_index = np.zeros(8760, int)
    for month, hours in enumerate(month_hours):
        month_index[month_hours[month-1]:hours] = month-1

    period_8760 = np.zeros(8760, int)
    hour = 0
    day = start_day # Start on 6 because the load profiles we are using start on a Sunday
    for h in range(8760):
        if day < 5:
            period_8760[h] = wkday_12by24[month_index[h], hour]
        else:
            period_8760[h] = wkend_12by24[month_index[h], hour]
        hour += 1
        if hour == 24: hour = 0; day += 1
        if day == 7: day = 0

    return period_8760

#%%
def design_tariff_for_portfolio(agent_df, avg_rev, peak_hour_indicies, summer_month_indicies, rev_f_d, rev_f_e, rev_f_fixed):

    '''
    Builds a tariff that would extract a given $/kWh from a portfolio of
    customers.

    Parameters
    ----------
    agent_df : 'pd.DataFrame'
        agents as loaded from the agent pkl file.

        Attributes
        ----------
        agent_df.load_profile : numpy.ndarray
        agent_df.weight : numpy.ndarray

    avg_rev : 'float'
        $/kWh that the tariff would extract from the given portfolio of customers.
    peak_hour_indicies : 'list'
        list of indices corresponding to the peak demand hours. Assumes peak hours are the same between
        demand and energy.
    summer_month_indicies : 'list'
        list of indices corresponding to the summer peak demand hours. Assumes peak hours only occur during the summer.
    rev_f_d : 'list'
        revenue strucutre for demand charges. Format is [fraction of total revenue, fraction that
        comes from tou charges, fraction that comes from flat charges] ex: [0.4875, 0.5, 0.5].
    rev_f_e : 'list'
        revenue strucutre for energy charges. Format is [fraction of total revenue, fraction that
        comes from off-peak hours, fraction that comes from on-peak hours] ex: [0.4875, 0.20, 0.8].
    rev_f_fixed : 'list'
        [fraction of revenue from fixed monthly charges]. ex: [0.025].

    Note
    ----
    1)  Peak hours are the same between demand and energy.
    2)  Peak hours only occur during the summer.

    Returns
    -------
    tarrif : 'class object'
        Returns tarrif, an object instantiated with the Tarrif class.
    '''

    # Construct the 12x24 matricies for the given peak hours
    d_wkend_12by24 = np.zeros([12,24], int)
    d_wkday_12by24 = np.zeros([12,24], int)
    e_wkend_12by24 = np.zeros([12,24], int)
    e_wkday_12by24 = np.zeros([12,24], int)
    for peak_hour in peak_hour_indicies:
        d_wkday_12by24[summer_month_indicies, peak_hour] = 1
        e_wkday_12by24[summer_month_indicies, peak_hour] = 1

    # Build an 8760 of peak hours
    d_tou_8760 = build_8760_from_12by24s(d_wkday_12by24, d_wkend_12by24, start_day=6)
    d_tou_n = 2

    # 8760 vector of month numbers
    month_hours = np.array([0, 744, 1416, 2160, 2880, 3624, 4344, 5088, 5832, 6552, 7296, 8016, 8760], int)
    month_index = np.zeros(8760, int)
    for month, hours in enumerate(month_hours):
        month_index[month_hours[month-1]:hours] = month-1

    period_matrix = np.zeros([8760, d_tou_n*12], bool)
    period_matrix[list(range(8760)),d_tou_8760+month_index*d_tou_n] = True

    # Define the dataframes that energy and demand values will be recorded in
    bld_peak_demands = pd.DataFrame()
    bld_flat_demands = pd.DataFrame()
    bld_peak_energy = pd.DataFrame()
    bld_offpeak_energy = pd.DataFrame()

    # Determine the peak demands and energy consumption for each building in
    # the portfolio
    for bld in list(agent_df.index):
        load_profile = agent_df.loc[bld, 'load_profile']
        load_distributed = load_profile[np.newaxis, :].T*period_matrix

        # Determine the max demands
        period_maxs = np.max(load_distributed, axis=0).reshape([2,12], order='F')
        bld_peak_demands[bld] = period_maxs[1,:]
        bld_flat_demands[bld] = np.max(period_maxs, axis=0)

        # Determine energy consumption
        period_sums = np.sum(load_distributed, axis=0).reshape([2,12], order='F')
        bld_peak_energy[bld] = period_sums[1,:]
        bld_offpeak_energy[bld] = period_sums[0,:]

    # Calculate the normalized revenue from each category
    normalized_consumption = np.sum(agent_df['f_in_this_portfolio'] * agent_df['aec'])
    norm_rev = normalized_consumption * avg_rev
    norm_rev_d_peak = norm_rev * rev_f_d[0] * rev_f_d[1]
    norm_rev_d_flat = norm_rev * rev_f_d[0] * rev_f_d[2]
    norm_rev_e_peak = norm_rev * rev_f_e[0] * rev_f_e[1]
    norm_rev_e_offpeak = norm_rev * rev_f_e[0] * rev_f_e[2]
    norm_rev_fixed = norm_rev * rev_f_fixed[0]

    # Calculate the prices that would result in the required revenue being
    # collected, for each category
    charge_d_peak = norm_rev_d_peak / np.sum(np.sum(bld_peak_demands)*agent_df['f_in_this_portfolio'])
    charge_d_flat = norm_rev_d_flat / np.sum(np.sum(bld_flat_demands)*agent_df['f_in_this_portfolio'])
    charge_e_peak = norm_rev_e_peak / np.sum(np.sum(bld_peak_energy)*agent_df['f_in_this_portfolio'])
    charge_e_offpeak = norm_rev_e_offpeak / np.sum(np.sum(bld_offpeak_energy)*agent_df['f_in_this_portfolio'])
    charge_fixed_monthly = norm_rev_fixed / np.sum(agent_df['f_in_this_portfolio']) / 12.0

    # Prepare variables for tariff definition
    d_tou_levels = np.array([[1e9, 1e9]])
    d_tou_prices = np.array([[0, charge_d_peak]])
    d_flat_levels = 1e9
    d_flat_prices = charge_d_flat
    e_levels = np.array([[1e9, 1e9]])
    e_prices = np.array([[charge_e_offpeak, charge_e_peak]]) # Check this!!! TODO

    # Define tariff
    tariff = Tariff()
    tariff.define_d_flat(d_flat_levels, d_flat_prices)
    tariff.define_d_tou(d_wkday_12by24, d_wkend_12by24, d_tou_levels, d_tou_prices)
    tariff.define_e(e_wkday_12by24, e_wkend_12by24, e_levels, e_prices)
    tariff.fixed_charge = charge_fixed_monthly

    export_tariff = Export_Tariff(full_retail_nem=True)
    for bld in list(agent_df.index):
        load_profile = agent_df.loc[bld, 'load_profile']
        original_bill, original_bill_results = bill_calculator(load_profile, tariff, export_tariff)

    return tariff