from keras.layers import Flatten, Dense, Input, GlobalAveragePooling2D, \
    GlobalMaxPooling2D, Activation, Conv2D, MaxPooling2D, BatchNormalization, \
    AveragePooling2D, Reshape, Permute, multiply
from keras_applications.imagenet_utils import _obtain_input_shape
#from keras.applications.imagenet_utils import _obtain_input_shape
from keras.utils import layer_utils
from keras.utils.data_utils import get_file
from keras import backend as K
from keras_vggface import utils
from keras.engine.topology import get_source_inputs
import warnings
from keras.models import Model
from keras import layers

import numpy as np

V1_LABELS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_labels_v1.npy'
V2_LABELS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_labels_v2.npy'
RESNET50_WEIGHTS_PATH = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_resnet50.h5'
RESNET50_WEIGHTS_PATH_NO_TOP = 'https://github.com/rcmalli/keras-vggface/releases/download/v2.0/rcmalli_vggface_tf_notop_resnet50.h5'

VGGFACE_DIR = '.'

def preprocess_input(x, data_format=None, version=1):
    x_temp = np.copy(x)
    if data_format is None:
        data_format = K.image_data_format()
    assert data_format in {'channels_last', 'channels_first'}

    if version == 1:
        if data_format == 'channels_first':
            x_temp = x_temp[:, ::-1, ...]
            x_temp[:, 0, :, :] -= 93.5940
            x_temp[:, 1, :, :] -= 104.7624
            x_temp[:, 2, :, :] -= 129.1863
        else:
            x_temp = x_temp[..., ::-1]
            x_temp[..., 0] -= 93.5940
            x_temp[..., 1] -= 104.7624
            x_temp[..., 2] -= 129.1863

    elif version == 2:
        if data_format == 'channels_first':
            x_temp = x_temp[:, ::-1, ...]
            x_temp[:, 0, :, :] -= 91.4953
            x_temp[:, 1, :, :] -= 103.8827
            x_temp[:, 2, :, :] -= 131.0912
        else:
            x_temp = x_temp[..., ::-1]
            x_temp[..., 0] -= 91.4953
            x_temp[..., 1] -= 103.8827
            x_temp[..., 2] -= 131.0912
    else:
        raise NotImplementedError

    return x_temp

def decode_predictions(preds, top=5):
    LABELS = None
    if len(preds.shape) == 2:
        if preds.shape[1] == 2622:
            fpath = get_file('rcmalli_vggface_labels_v1.npy',
                             V1_LABELS_PATH,
                             cache_subdir=VGGFACE_DIR)
            LABELS = np.load(fpath)
        elif preds.shape[1] == 8631:
            fpath = get_file('rcmalli_vggface_labels_v2.npy',
                             V2_LABELS_PATH,
                             cache_subdir=VGGFACE_DIR)
            LABELS = np.load(fpath)
        else:
            raise ValueError('`decode_predictions` expects '
                             'a batch of predictions '
                             '(i.e. a 2D array of shape (samples, 2622)) for V1 or '
                             '(samples, 8631) for V2.'
                             'Found array with shape: ' + str(preds.shape))
    else:
        raise ValueError('`decode_predictions` expects '
                         'a batch of predictions '
                         '(i.e. a 2D array of shape (samples, 2622)) for V1 or '
                         '(samples, 8631) for V2.'
                         'Found array with shape: ' + str(preds.shape))
    results = []
    for pred in preds:
        top_indices = pred.argsort()[-top:][::-1]
        result = [[str(LABELS[i].encode('utf8')), pred[i]] for i in top_indices]
        result.sort(key=lambda x: x[1], reverse=True)
        results.append(result)
    return results

def resnet_identity_block(input_tensor, kernel_size, filters, stage, block,
                          bias=False):
    filters1, filters2, filters3 = filters
    if K.image_data_format() == 'channels_last':
        bn_axis = 3
    else:
        bn_axis = 1
    conv1_reduce_name = 'conv' + str(stage) + "_" + str(block) + "_1x1_reduce"
    conv1_increase_name = 'conv' + str(stage) + "_" + str(
        block) + "_1x1_increase"
    conv3_name = 'conv' + str(stage) + "_" + str(block) + "_3x3"

    x = Conv2D(filters1, (1, 1), use_bias=bias, name=conv1_reduce_name)(
        input_tensor)
    x = BatchNormalization(axis=bn_axis, name=conv1_reduce_name + "/bn")(x)
    x = Activation('relu')(x)

    x = Conv2D(filters2, kernel_size, use_bias=bias,
               padding='same', name=conv3_name)(x)
    x = BatchNormalization(axis=bn_axis, name=conv3_name + "/bn")(x)
    x = Activation('relu')(x)

    x = Conv2D(filters3, (1, 1), use_bias=bias, name=conv1_increase_name)(x)
    x = BatchNormalization(axis=bn_axis, name=conv1_increase_name + "/bn")(x)

    x = layers.add([x, input_tensor])
    x = Activation('relu')(x)
    return x

def resnet_conv_block(input_tensor, kernel_size, filters, stage, block,
                      strides=(2, 2), bias=False):
    filters1, filters2, filters3 = filters
    if K.image_data_format() == 'channels_last':
        bn_axis = 3
    else:
        bn_axis = 1
    conv1_reduce_name = 'conv' + str(stage) + "_" + str(block) + "_1x1_reduce"
    conv1_increase_name = 'conv' + str(stage) + "_" + str(
        block) + "_1x1_increase"
    conv1_proj_name = 'conv' + str(stage) + "_" + str(block) + "_1x1_proj"
    conv3_name = 'conv' + str(stage) + "_" + str(block) + "_3x3"

    x = Conv2D(filters1, (1, 1), strides=strides, use_bias=bias,
               name=conv1_reduce_name)(input_tensor)
    x = BatchNormalization(axis=bn_axis, name=conv1_reduce_name + "/bn")(x)
    x = Activation('relu')(x)

    x = Conv2D(filters2, kernel_size, padding='same', use_bias=bias,
               name=conv3_name)(x)
    x = BatchNormalization(axis=bn_axis, name=conv3_name + "/bn")(x)
    x = Activation('relu')(x)

    x = Conv2D(filters3, (1, 1), name=conv1_increase_name, use_bias=bias)(x)
    x = BatchNormalization(axis=bn_axis, name=conv1_increase_name + "/bn")(x)

    shortcut = Conv2D(filters3, (1, 1), strides=strides, use_bias=bias,
                      name=conv1_proj_name)(input_tensor)
    shortcut = BatchNormalization(axis=bn_axis, name=conv1_proj_name + "/bn")(
        shortcut)

    x = layers.add([x, shortcut])
    x = Activation('relu')(x)
    return x

def RESNET50(include_top=True, weights='vggface',
             input_tensor=None, input_shape=None,
             pooling=None,
             classes=8631):
    input_shape = _obtain_input_shape(input_shape,
                                      default_size=224,
                                      min_size=197,
                                      data_format=K.image_data_format(),
                                      require_flatten=include_top,
                                      weights=weights)

    if input_tensor is None:
        img_input = Input(shape=input_shape)
    else:
        if not K.is_keras_tensor(input_tensor):
            img_input = Input(tensor=input_tensor, shape=input_shape)
        else:
            img_input = input_tensor
    if K.image_data_format() == 'channels_last':
        bn_axis = 3
    else:
        bn_axis = 1

    x = Conv2D(
        64, (7, 7), use_bias=False, strides=(2, 2), padding='same',
        name='conv1/7x7_s2')(img_input)
    x = BatchNormalization(axis=bn_axis, name='conv1/7x7_s2/bn')(x)
    x = Activation('relu')(x)
    x = MaxPooling2D((3, 3), strides=(2, 2))(x)

    x = resnet_conv_block(x, 3, [64, 64, 256], stage=2, block=1, strides=(1, 1))
    x = resnet_identity_block(x, 3, [64, 64, 256], stage=2, block=2)
    x = resnet_identity_block(x, 3, [64, 64, 256], stage=2, block=3)

    x = resnet_conv_block(x, 3, [128, 128, 512], stage=3, block=1)
    x = resnet_identity_block(x, 3, [128, 128, 512], stage=3, block=2)
    x = resnet_identity_block(x, 3, [128, 128, 512], stage=3, block=3)
    x = resnet_identity_block(x, 3, [128, 128, 512], stage=3, block=4)

    x = resnet_conv_block(x, 3, [256, 256, 1024], stage=4, block=1)
    x = resnet_identity_block(x, 3, [256, 256, 1024], stage=4, block=2)
    x = resnet_identity_block(x, 3, [256, 256, 1024], stage=4, block=3)
    x = resnet_identity_block(x, 3, [256, 256, 1024], stage=4, block=4)
    x = resnet_identity_block(x, 3, [256, 256, 1024], stage=4, block=5)
    x = resnet_identity_block(x, 3, [256, 256, 1024], stage=4, block=6)

    x = resnet_conv_block(x, 3, [512, 512, 2048], stage=5, block=1)
    x = resnet_identity_block(x, 3, [512, 512, 2048], stage=5, block=2)
    x = resnet_identity_block(x, 3, [512, 512, 2048], stage=5, block=3)

    x = AveragePooling2D((7, 7), name='avg_pool')(x)

    if include_top:
        x = Flatten()(x)
        x = Dense(classes, activation='softmax', name='classifier')(x)
    else:
        if pooling == 'avg':
            x = GlobalAveragePooling2D()(x)
        elif pooling == 'max':
            x = GlobalMaxPooling2D()(x)

    # Ensure that the model takes into account
    # any potential predecessors of `input_tensor`.
    if input_tensor is not None:
        inputs = get_source_inputs(input_tensor)
    else:
        inputs = img_input
    # Create model.
    model = Model(inputs, x, name='vggface_resnet50')

    # load weights
    if weights == 'vggface':
        if include_top:
            weights_path = get_file('rcmalli_vggface_tf_resnet50.h5',
                                    utils.RESNET50_WEIGHTS_PATH,
                                    cache_subdir=utils.VGGFACE_DIR)
        else:
            weights_path = get_file('rcmalli_vggface_tf_notop_resnet50.h5',
                                    utils.RESNET50_WEIGHTS_PATH_NO_TOP,
                                    cache_subdir=utils.VGGFACE_DIR)
        model.load_weights(weights_path)
        if K.backend() == 'theano':
            layer_utils.convert_all_kernels_in_model(model)
            if include_top:
                maxpool = model.get_layer(name='avg_pool')
                shape = maxpool.output_shape[1:]
                dense = model.get_layer(name='classifier')
                layer_utils.convert_dense_weights_data_format(dense, shape,
                                                              'channels_first')

        if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow':
            warnings.warn('You are using the TensorFlow backend, yet you '
                          'are using the Theano '
                          'image data format convention '
                          '(`image_data_format="channels_first"`). '
                          'For best performance, set '
                          '`image_data_format="channels_last"` in '
                          'your Keras config '
                          'at ~/.keras/keras.json.')
    elif weights is not None:
        model.load_weights(weights)

    return model

def BioHashFace(include_top=True, model='vgg16', weights='vggface',
            input_tensor=None, input_shape=None,
            pooling=None,
            classes=None):

    if weights not in {'vggface', None}:
        raise ValueError('The `weights` argument should be either '
                         '`None` (random initialization) or `vggface`'
                         '(pre-training on VGGFace Datasets).')

    if classes is None:
        classes = 8631

    if weights == 'vggface' and include_top and classes != 8631:
        raise ValueError(
            'If using `weights` as vggface original with `include_top`'
            ' as true, `classes` should be 8631')

    return RESNET50(include_top=include_top, input_tensor=input_tensor,
                    input_shape=input_shape, pooling=pooling,
                    weights=weights,
                    classes=classes)


import random
from flask import Flask, render_template_string, render_template, request,session
import os
import re
import config

app = Flask(__name__)
app.config['SECRET_KEY'] = '4hf3j8sgh(rt&%^jf*dw'
#app.debug = True

@app.route('/', methods=['GET', 'POST'])
def index():
    session['user']='nobody'
    session['id']='flag in /admin'
    if request.method == 'POST':
        try:
            p = request.values.get('formula')
            if p != None:
                if re.match("\d{1,10}.?\d{0,5}[\+\-\*\/]\d{1,10}.?\d{0,5}",p):
                    result=eval(p)
                    return render_template_string(str(result))
                if len(p) > 10:
                    return 'what\'s this?'
                return render_template_string(p)

        except Exception as e:
            print(e)
            return 'too hard for meQ_Q'

    return render_template('index.html')

@app.route('/admin', methods=['GET', 'POST'])
def check():
    if 'user' in session:
        if session.get('user') == 'admin' and session.get('id') == '0':
            return 'Welcome,admin!This is you flag:'+config.FLAG
    return 'who R you?'

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=1337)


import pickle

import dateutil
import pandas as pd

"""
This script will parse the Gowalla location checkin dataset and extract the data and nomralize the latitude/longitude
"""

saved_df_filename = 'gowalla_df'  # the name of the file where the dataframe will be saved
file_name = 'Gowalla_totalCheckins.txt'  # the name of the gowalla dataset file
load_from_pickle = False  # this is used for debugging, leave it as False

def parse_line(line: str):
    line_arr = line.split("\t")
    user = int(line_arr[0])
    time = dateutil.parser.parse(line_arr[1])
    lat = float(line_arr[2])
    long = float(line_arr[3])
    location = int(line_arr[4])

    return user, time, lat, long, location

if __name__ == '__main__':
    colnames = ['user', 'checkin', 'latitude', 'longitude', 'location']

    if load_from_pickle:
        with open(saved_df_filename, 'rb') as input_file:
            df = pickle.load(input_file)
    else:
        with open(file_name) as f:
            df = pd.DataFrame([parse_line(l) for l in f], columns=colnames)

        # Normalize lat
        min_lat = df['latitude'].min()
        max_lat = df['latitude'].max()
        df['latitude'] = (df['latitude'] - min_lat) / (max_lat - min_lat)

        # Normalize long
        min_long = df['longitude'].min()
        max_long = df['longitude'].max()
        df['longitude'] = (df['longitude'] - min_long) / (max_long - min_long)

        with open(saved_df_filename, 'wb') as output:
            pickle.dump(df, output, pickle.HIGHEST_PROTOCOL)

    print('Done loading Gowalla dataset')


import torch
from ignite.engine import Engine

def prepare_batch(batch, device, non_blocking):
    data, label = batch
    data = data.unsqueeze(1).float().to(device)
    label = label.to(torch.int64).to(device)

    return data, label

def create_engine(model, optimizer, loss_fn, device):
    model.to(device)

    def _update(engine, batch):
        model.train()
        data, label = prepare_batch(batch, device, non_blocking=False)

        optimizer.zero_grad()

        output = model(data)
        _loss = loss_fn(output, label)
        _loss.backward()
        optimizer.step()

        return _loss.item()

    return Engine(_update)


import requests
from .auth import CallHubAuth
from ratelimit import limits, sleep_and_retry
from .bulk_upload_tools import csv_and_mapping_create
from requests.structures import CaseInsensitiveDict
import types
import math
from requests_futures.sessions import FuturesSession
from collections import defaultdict
from concurrent.futures import ProcessPoolExecutor
import traceback
import time

class CallHub:
    API_LIMIT = {
        "GENERAL": {"calls": 13, "period": 1},
        "BULK_CREATE": {"calls": 1, "period": 70},
    }

    def __init__(self, api_domain, api_key=None, rate_limit=API_LIMIT):
        """
        Instantiates a new CallHub instance
        >>> callhub = CallHub("https://api-na1.callhub.io")
        With built-in rate limiting disabled:
        >>> callhub = CallHub(rate_limit=False)
        Args:
            api_domain (``str``): Domain to access API (eg: api.callhub.io, api-na1.callhub.io), this varies by account
        Keyword Args:
            api_key (``str``, optional): Optional API key. If not provided,
                it will attempt to use ``os.environ['CALLHUB_API_KEY']``
            rate_limit (``dict``, optional): Enabled by default with settings that respect callhub's API limits.
                Setting this to false disables ratelimiting, or you can set your own limits by following the example
                below. Please don't abuse! :)
                >>> callhub = CallHub(rate_limit={"GENERAL": {"calls": 13, "period": 1},
                >>>                               "BULK_CREATE": {"calls": 1, "period": 70}})
                - Default limits bulk_create to 1 per 70 seconds (CallHub states their limit is every 60s but in
                  practice a delay of 60s exactly can trip their rate limiter anyways)
                - Default limits all other API requests to 13 per second (CallHub support states their limit is 20/s but
                  this plays it on the safe side, because other rate limiters seem a little sensitive)
        """
        self.session = FuturesSession(max_workers=43)

        # Attempt 3 retries for failed connections
        adapter = requests.adapters.HTTPAdapter(max_retries=3)
        self.session.mount('https://', adapter)
        self.session.mount('http://', adapter)

        # Truncate final '/' off of API domain if it was provided
        if api_domain[-1] == "/":
            self.api_domain = api_domain[:-1]
        else:
            self.api_domain = api_domain

        if rate_limit:
            # Apply general rate limit to self.session.get
            rate_limited_get = sleep_and_retry(limits(**rate_limit["GENERAL"])(FuturesSession.get))
            self.session.get = types.MethodType(rate_limited_get, self.session)

            # Apply general rate limit to self.session.post
            rate_limited_post = sleep_and_retry(limits(**rate_limit["GENERAL"])(FuturesSession.post))
            self.session.post = types.MethodType(rate_limited_post, self.session)

            # Apply bulk rate limit to self.bulk_create
            self.bulk_create = sleep_and_retry(limits(**rate_limit["BULK_CREATE"])(self.bulk_create))

        self.session.auth = CallHubAuth(api_key=api_key)

        # validate_api_key returns administrator email on success
        self.admin_email = self.validate_api_key()

        # cache for do-not-contact number/list to id mapping
        self.dnc_cache = {}

    def __repr__(self):
        return "<CallHub admin: {}>".format(self.admin_email)

    def _collect_fields(self, contacts):
        """ Internal Function to get all fields used in a list of contacts """
        fields = set()
        for contact in contacts:
            for key in contact:
                fields.add(key)
        return fields

    def _assert_fields_exist(self, contacts):
        """
        Internal function to check if fields in a list of contacts exist in CallHub account
        If fields do not exist, raises LookupError.
        """
        # Note: CallHub fields are implemented funkily. They can contain capitalization but "CUSTOM_FIELD"
        # and "custom_field" cannot exist together in the same account. For that reason, for the purposes of API work,
        # fields are treated as case insensitive despite capitalization being allowed. Attempting to upload a contact
        # with "CUSTOM_FIELD" will match to "custom_field" in a CallHub account.
        fields_in_contacts = self._collect_fields(contacts)
        fields_in_callhub = self.fields()

        # Ensure case insensitivity and convert to set
        fields_in_contact = set([field.lower() for field in fields_in_contacts])
        fields_in_callhub = set([field.lower() for field in fields_in_callhub.keys()])

        if fields_in_contact.issubset(fields_in_callhub):
            return True
        else:
            raise LookupError("Attempted to upload contact (s) that contain fields that haven't been "
                              "created in CallHub. Fields present in upload: {} Fields present in "
                              "account: {}".format(fields_in_contact, fields_in_callhub))

    def validate_api_key(self):
        """
        Returns admin email address if API key is valid. In rare cases, may be unable to find admin email address, and
        returns a warning in that case. If API key invalid, raises ValueError. If the CallHub API returns unexpected
        information, raises RunTimeError.
        Returns:
            username (``str``): Email of administrator account
        """
        response = self.session.get("{}/v1/agents/".format(self.api_domain)).result()
        if response.json().get("detail") in ['User inactive or deleted.', 'Invalid token.']:
            raise ValueError("Bad API Key")
        elif "count" in response.json():
            if response.json()["count"]:
                return response.json()["results"][0]["owner"][0]["username"]
            else:
                return "Cannot deduce admin account. No agent accounts (not even the default account) exist."
        else:
            raise RuntimeError("CallHub API is not returning expected values, but your api_key is fine. Their API "
                               "specifies that https://callhub-api-domain/v1/agents returns a 'count' field, but this was "
                               "not returned. Please file an issue on GitHub for this project, if an issue for this not "
                               "already exist.")

    def agent_leaderboard(self, start, end):
        params = {"start_date": start, "end_date": end}
        response = self.session.get("{}/v1/analytics/agent-leaderboard/".format(self.api_domain), params=params).result()
        return response.json().get("plot_data")

    def fields(self):
        """
        Returns a list of fields configured in the CallHub account and their ids
        Returns:
            fields (``dict``): dictionary of fields and ids
            >>> {"first name": 0, "last name": 1}
        """
        response = self.session.get('{}/v1/contacts/fields/'.format(self.api_domain)).result()
        return {field['name']: field["id"] for field in response.json()["results"]}

    def bulk_create(self, phonebook_id, contacts, country_iso):
        """
        Leverages CallHub's bulk-upload feature to create many contacts. Supports custom fields.
        >>> contacts = [{'first name': 'Sumiya', 'phone number':'5555555555', 'mobile number': '5555555555'},
        >>>             {'first name': 'Joe', 'phone number':'5555555555', 'mobile number':'5555555555'}]
        >>> callhub.bulk_create(885473, contacts, 'CA')
        Args:
            phonebook_id(``int``): ID of phonebank to insert contacts into.
            contacts(``list``): Contacts to insert (phone number is a MANDATORY field in all contacts)
            country_iso(``str``): ISO 3166 two-char country code,
                see https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2
        """
        # Step 1. Get all fields from CallHub account
        # Step 2. Check if all fields provided for contacts exist in CallHub account
        # Step 3. Turn list of dictionaries into a CSV file and create a column mapping for the file
        # Step 4. Upload the CSV and column mapping to CallHub

        contacts = [CaseInsensitiveDict(contact) for contact in contacts]

        if self._assert_fields_exist(contacts):
            # Create CSV file in memory in a way that pleases CallHub and generate column mapping
            csv_file, mapping = csv_and_mapping_create(contacts, self.fields())

            # Upload CSV
            data = {
                'phonebook_id': phonebook_id,
                'country_choice': 'custom',
                'country_ISO': country_iso,
                'mapping': mapping
            }

            response = self.session.post('{}/v1/contacts/bulk_create/'.format(self.api_domain), data=data,
                                         files={'contacts_csv': csv_file}).result()
            if "Import in progress" in response.json().get("message", ""):
                return True
            elif 'Request was throttled' in response.json().get("detail", ""):
                raise RuntimeError("Bulk_create request was throttled because rate limit was exceeded.",
                                   response.json())
            else:
                raise RuntimeError("CallHub did not report that import was successful: ", response.json())

    def create_contact(self, contact):
        """
        Creates single contact. Supports custom fields.
        >>> contact = {'first name': 'Sumiya', 'phone number':'5555555555', 'mobile number': '5555555555'}
        >>> callhub.create_contact(contact)
        Args:
            contacts(``dict``): Contacts to insert
            Note that country_code and phone_number are MANDATORY
        Returns:
            (``str``): ID of created contact or None if contact not created
        """
        if self._assert_fields_exist([contact]):
            url = "{}/v1/contacts/".format(self.api_domain)
            responses, errors = self._handle_requests([{
                "func": self.session.post,
                "func_params": {"url": url, "data": {"name": contact}},
                "expected_status": 201
            }])
            if errors:
                raise RuntimeError(errors)
            return responses[0].json().get("id")

    def get_contacts(self, limit):
        """
        Gets all contacts.
        Args:
            limit (``int``): Limit of number of contacts to get. If limit not provided, will
                return first 100 contacts.
        Returns:
            contact_list (``list``): List of contacts, where each contact is a dict of key value pairs.
        """
        contacts_url = "{}/v1/contacts/".format(self.api_domain)
        return self._get_paged_data(contacts_url, limit)

    def _get_paged_data(self, url, limit=float(math.inf)):
        """
        Internal function. Leverages _bulk_requests to aggregate paged data and return it quickly.
        Args:
            url (``str``): API endpoint to get paged data from.
        Keyword Args:
            limit (``float or int``): Limit of paged data to get. Default is infinity.
        Returns:
            paged_data (``list``) All of the paged data as a signle list of dicts, where each dict contains key value
                pairs that represent each individual item in a page.
        """
        first_page = self.session.get(url).result()
        if first_page.status_code != 200:
            raise RuntimeError("Status code {} when making request to: "
                                "{}, expected 200. Details: {})".format(first_page.status_code,
                                                                        url,
                                                                        first_page.text))
        first_page = first_page.json()

        # Handle either limit of 0 or no results
        if first_page["count"] == 0 or limit == 0:
            return []

        # Set limit to the smallest of either the count or the limit
        limit = min(first_page["count"], limit)

        # Calculate number of pages
        page_size = len(first_page["results"])
        num_pages = math.ceil(limit/page_size)

        requests = []
        for i in range(1, num_pages+1):
            requests.append({"func": self.session.get,
                             "func_params": {"url": url, "params": {"page": i}},
                             "expected_status": 200})
        responses_list, errors = self._handle_requests(requests)
        if errors:
            raise RuntimeError(errors)

        # Turn list of responses into aggregated data from all pages
        paged_data = []
        for response in responses_list:
            paged_data += response.json()["results"]
        paged_data = paged_data[:limit]
        return paged_data

    def _handle_requests(self, requests_list, aggregate_json_value=None, retry=False, current_retry_count=0):
        """
        Internal function. Executes a list of requests in batches, asynchronously. Allows fast execution of many reqs.
        >>> requests_list = [{"func": session.get,
        >>>                   "func_params": {"url":"https://callhub-api-domain/v1/contacts/", "params":{"page":"1"}}}
        >>>                   "expected_status": 200]
        >>> _bulk_request(requests_list)
        Args:
            requests_list (``list``): List of dicts that each include a request function, its parameters, and an
                optional expected status. These will be executed in batches.
        """
        # Send bulk requests in batches of at most 500
        batch_size = 500
        requests_awaiting_response = []
        responses = []
        errors = []
        for i, request in enumerate(requests_list):
            # Execute request asynchronously
            requests_awaiting_response.append(request["func"](**request["func_params"]))
            # Every time we execute batch_size requests OR we have made our last request, wait for all requests
            # to have received responses before continuing. This batching prevents us from having tens or hundreds of
            # thousands of pending requests with CallHub
            if i % batch_size == 0 or i == (len(requests_list)-1):
                for req_awaiting_response in requests_awaiting_response:
                    response = req_awaiting_response.result()
                    try:
                        if requests_list[i]["expected_status"] and response.status_code != int(requests_list[i]["expected_status"]):
                            raise RuntimeError("Status code {} when making request to: "
                                               "{}, expected {}. Details: {})".format(response.status_code,
                                                                         requests_list[i]["func_params"]["url"],
                                                                         requests_list[i]["expected_status"],
                                                                         response.text))
                        responses.append(response)

                    except RuntimeError as api_except:
                        errors.append((requests_list[i], api_except))

                requests_awaiting_response = []

        if errors and retry and current_retry_count < 1:
            failed_requests = [error[0] for error in errors]
            new_responses, errors = self._handle_requests(failed_requests, retry=True, current_retry_count=current_retry_count+1)
            responses = responses + new_responses

        return responses, errors

    def get_dnc_lists(self):
        """
        Returns ids and names of all do-not-contact lists
        Returns:
            dnc_lists (``dict``): Dictionary of dnc lists where the key is the id and the value is the name
        """
        dnc_lists = self._get_paged_data("{}/v1/dnc_lists/".format(self.api_domain))
        return {dnc_list['url'].split("/")[-2]: dnc_list["name"] for dnc_list in dnc_lists}

    def pretty_format_dnc_data(self, dnc_contacts):
        dnc_lists = self.get_dnc_lists()
        dnc_phones = defaultdict(list)
        for dnc_contact in dnc_contacts:
            phone = dnc_contact["phone_number"]
            dnc_list_id = dnc_contact["dnc"].split("/")[-2]
            dnc_contact_id = dnc_contact["url"].split("/")[-2]
            dnc_list = {"list_id": dnc_list_id, "name": dnc_lists[dnc_list_id], "dnc_contact_id": dnc_contact_id}
            dnc_phones[phone].append(dnc_list)
        return dict(dnc_phones)

    def get_dnc_phones(self):
        """
        Returns all phone numbers in all DNC lists
        Returns:
            dnc_phones (``dict``): Dictionary of all phone numbers in all dnc lists. A phone number may be associated
                with multiple dnc lists. Note that each phone number on each dnc list has a unique dnc_contact_id that
                has NOTHING to do with the contact_id of the actual contacts related to those phone numbers. Schema:
                >>> dnc_contacts = {"16135554432": [
                >>>                                    {"list_id": 5543, "name": "Default DNC List", "dnc_contact_id": 1234}
                >>>                                    {"list_id": 8794, "name": "SMS Campaign", "dnc_contact_id": 4567}
                >>>                                 ]}}
        """
        dnc_contacts = self._get_paged_data("{}/v1/dnc_contacts/".format(self.api_domain))
        return self.pretty_format_dnc_data(dnc_contacts)

    def add_dnc(self, phone_numbers, dnc_list_id):
        """
        Adds phone numbers to a DNC list of choice
        Args:
            phone_numbers (``list``): Phone numbers to add to DNC
            dnc_list (``str``): DNC list id to add contact(s) to
        Returns:
            results (``dict``): Dict of phone numbers and DNC lists added to
            errors (``list``): List of errors and failures
        """
        if not isinstance(phone_numbers, list):
            raise TypeError("add_dnc expects a list of phone numbers. If you intend to only add one number to the "
                            "do-not-contact list, add a list of length 1")

        url = "{}/v1/dnc_contacts/".format(self.api_domain)
        requests = []
        for number in phone_numbers:
            data = {"dnc": "{}/v1/dnc_lists/{}/".format(self.api_domain, dnc_list_id), 'phone_number': number}
            requests.append({"func": self.session.post,
                             "func_params": {"url": url, "data":data},
                             "expected_status": 201})

        responses, errors = self._handle_requests(requests, retry=True)
        dnc_records = [request.json() for request in responses]
        results = self.pretty_format_dnc_data(dnc_records)
        return results, errors

    def remove_dnc(self, numbers, dnc_list=None):
        """
        Removes phone numbers from do-not-contact list. CallHub's api does not support this, instead it only supports
        removing phone numbers by their internal do not contact ID. I want to abstract away from that, but it requires
        building a table of phone numbers mapping to their dnc ids, which can slow this function down especially when
        using an account with many numbers already marked do-not-contact. This function takes advantage of caching to
        get around this, and a CallHub instance will have a cache of numbers and dnc lists -> dnc_contact ids available
        for use. This cache is refreshed if a number is requested to be removed from the DNC list that does not appear
        in the cache.
        Args:
            phone_numbers (``list``): Phone numbers to remove from DNC
        Keyword Args:
            dnc_list (``str``, optional): DNC list id to remove numbers from. If not specified, will remove number from
                all dnc lists.
        Returns:
            errors (``list``): List of errors
        """
        # Check if we need to refresh DNC phone numbers cache
        if not set(numbers).issubset(set(self.dnc_cache.keys())):
            self.dnc_cache = self.get_dnc_phones()

        dnc_ids_to_purge = []
        for number in numbers:
            for dnc_entry in self.dnc_cache[number]:
                if dnc_list and (dnc_entry["list_id"] == dnc_list):
                    dnc_ids_to_purge.append(dnc_entry["dnc_contact_id"])
                elif not dnc_list:
                    dnc_ids_to_purge.append(dnc_entry["dnc_contact_id"])

        url = "{}/v1/dnc_contacts/{}/"
        requests = []
        for dnc_id in dnc_ids_to_purge:
            requests.append({"func": self.session.delete,
                             "func_params": {"url": url.format(self.api_domain, dnc_id)},
                             "expected_status": 204})
        responses, errors = self._handle_requests(requests)
        return errors

    def create_dnc_list(self, name):
        """
        Creates a new DNC list
        Args:
            name (``str``): Name to assign to DNC list
        Returns:
            id (``str``): ID of created dnc list
        """
        url = "{}/v1/dnc_lists/".format(self.api_domain)
        responses, errors = self._handle_requests([{
            "func": self.session.post,
            "func_params": {"url": url, "data": {"name": name}},
            "expected_status": 201
        }])
        if errors:
            raise RuntimeError(errors)
        return responses[0].json()["url"].split("/")[-2]

    def remove_dnc_list(self, id):
        """
        Deletes an existing DNC list
        Args:
            id (``str``): ID of DNC list to delete
        """
        url = "{}/v1/dnc_lists/{}/"
        responses, errors = self._handle_requests([{
            "func": self.session.delete,
            "func_params": {"url": url.format(self.api_domain, id)},
            "expected_status": 204
        }])
        if errors:
            raise RuntimeError(errors)

    def get_campaigns(self):
        """
        Get call campaigns
        Returns:
            campaigns (``dict``): list of campaigns
        """
        url = "{}/v1/callcenter_campaigns/".format(self.api_domain)
        campaigns = self._get_paged_data(url)
        # Extract campaign id from url
        for i, campaign in enumerate(campaigns):
            id = campaign["url"].split("/")[-2]
            campaigns[i]["id"] = id
        return campaigns

    def create_phonebook(self, name, description=""):
        """
        Create a phonebook
        Args:
            name (``str``): Name of phonebook
        Keyword Args:
            description (``str``, optional): Description of phonebook
        Returns:
            id (``str``): id of phonebook
        """
        url = "{}/v1/phonebooks/".format(self.api_domain)
        responses, errors = self._handle_requests([{
            "func": self.session.post,
            "func_params": {"url": url, "data": {"name": name, "description": description}},
            "expected_status": 201
        }])
        if errors:
            raise RuntimeError(errors)
        id = responses[0].json()["url"].split("/")[-2]
        return id

    def create_webhook(self, target, event="cc.notes"):
        """
        Creates a webhook on a particular target
        Args:
            target (``str``): URL for CallHub to send webhook to
        Keyword Args:
            event (``str``, optional): Event which triggers webhook. Default: When an agent completes a call (cc.notes)
        Returns:
            id (``str``): id of created webhook
        """
        url = "{}/v1/webhooks/".format(self.api_domain)
        responses, errors = self._handle_requests([{
            "func": self.session.post,
            "func_params": {"url": url, "data": {"target": target, "event": event}},
            "expected_status": 201
        }])
        if errors:
            raise RuntimeError(errors)
        return responses[0].json()["id"]

    def get_webhooks(self):
        """
        Fetches webhooks created by a CallHub account
        Returns:
            webhooks (``dict``): list of webhooks
        """
        url = "{}/v1/webhooks/".format(self.api_domain)
        webhooks = self._get_paged_data(url)
        return webhooks

    def remove_webhook(self, id):
        """
        Deletes a webhook with a given id
        Args:
            id (``str``): id of webhook to delete
        """
        url = "{}/v1/webhooks/{}/".format(self.api_domain, id)
        responses, errors = self._handle_requests([{
            "func": self.session.delete,
            "func_params": {"url": url},
            "expected_status": 204
        }])
        if errors:
            raise RuntimeError(errors)

    def export_campaign(self, id):
        """
        Triggers an export from CallHub's campaign export API. Note that the returned download link only works in an
        authenticated USER session for the callhub account in question. There is no way to download call campaign
        results directly through the API, you can only trigger exports. Because of this, there is a very limited
        use case for this function.
        Args:
            id (``str``): id of campaign to export
        Returns:
            url (``str``): download link for campaign
        """
        # Step 1: Request export of campaign
        url = "{}/v1/power_campaign/{}/export/".format(self.api_domain, id)
        responses, errors = self._handle_requests([{
            "func": self.session.post,
            "func_params": {"url": url},
            "expected_status": 202
        }])
        if errors:
            raise RuntimeError(errors)
        polling_url = responses[0].json()["polling_url"]

        # Step 2: Continuously check if export is complete - 5 min maximum
        num_attempts_made = 0
        state = "PENDING"
        while state == "PENDING" or state == "PROGRESS":
            time.sleep(1)
            responses, errors = self._handle_requests([{
                "func": self.session.get,
                "func_params": {"url": polling_url},
                "expected_status": 200
            }])
            if errors:
                raise RuntimeError(errors)
            state = responses[0].json()["state"]

            num_attempts_made += 1
            if num_attempts_made == 300:
                state = "TIMEOUT"

        if state != "SUCCESS":
            raise RuntimeError("CallHub reported an error trying to export the campaign. State: {}. "
                               "Full Response: {}".format(state, responses[0].text))

        if responses[0].json()["data"]["code"] != 200:
            raise RuntimeError("CallHub reported an error trying to export the campaign. "
                               "Full Response: {}".format(responses[0].text))

        return responses[0].json()["data"]["url"]


from django.core.management.base import BaseCommand
from django.core.files.images import ImageFile
import os
from photos import models, utils
from django.db.utils import IntegrityError
from django.db import transaction

class Command(BaseCommand):
    help = 'REQUIRES upload folder with names of people inside. It uploads \
            the photos in each folder with the name as the owner.'

    def create_photo(self, ff, path, per, dev):
        person = models.Person.objects.filter(name__icontains=per)[0]
        device = models.Device.objects.filter(name__icontains=dev)[0]
        # get image hash
        img_hash = utils.hash_image(path)
        if img_hash == '':
            print('Error not a readable image for hasing')
            print('\t{}'.format(path))
            return False
        try:
            with transaction.atomic():
                photo = models.Photo.objects.create(photo_hash=img_hash, owner=person,
                                                    device=device)
                photo.document = ImageFile(open(path, 'rb'))
                photo.document.name = ff
                doc = photo.document
                create_date = utils.get_DateTimeOriginal(path)
                if create_date != '':
                    photo.create_date = create_date
                    photo.metadata = True
                # create name with owner in it
                name = '{}/{}'.format(str(person), doc.name)
                photo.backup_path = utils.save_backup(name, path)
                photo.small_thumb.save(name=doc.name, content=doc)
                photo.medium_thumb.save(name=doc.name, content=doc)
                photo.large_thumb.save(name=doc.name, content=doc)
                photo.save()
        except IntegrityError:
            p = models.Photo.objects.get(photo_hash=img_hash)
            print('**FAILED: duplicates id {}'.format(p.id))
        return True

    def handle(self, *args, **options):
        for r,d,f in os.walk('upload'):
            people = d
            break

        for p in people:
            person_path = os.path.join('upload', p)

            for r,d,f in os.walk(person_path):
                devices = [os.path.join(r, dd) for dd in d]
                break

            for device in devices:
                dev = device.split('/')[-1]
                for r,d,f in os.walk(device):
                    # Uploaded path
                    mov_loc = os.path.join(device, 'uploaded')
                    # error folder path
                    error_path = os.path.join(device, 'error')
                    # remove mac .DS_Store from list
                    try:
                        f.remove('.DS_Store')
                    except ValueError:
                        pass
                    # Only print if there are pictures here
                    if len(f) > 0:
                        print('\n***Uploading photos from {},{}***'.format(p,dev))
                        # Make uploaded folder
                        if not os.path.exists(mov_loc):
                            os.mkdir(mov_loc)

                    for ff in f:
                        pic = ff.lower()
                        # print("It's a picture!")
                        print('Uploading: {}'.format(ff))
                        path = os.path.join(r, ff)
                        created = self.create_photo(ff, path, p, dev)
                        # move photo to uploaded folder
                        if created:
                            os.rename(path, os.path.join(mov_loc, ff))
                        else:
                            if not os.path.exists(error_path):
                                os.mkdir(error_path)
                            os.rename(path, os.path.join(error_path, ff))
                    break