from sklearn import preprocessing
import pandas as pd

class CategoricalFeature:
    def __init__(self, df, categorical_features, encoding_type, handle_na=False):
        """
        :param df: pandas dataframe
        :param categorical_features: list of column names e.g. ['ord_1','ord_2'......]
        :param encoding_type: label,binary

        """
        self.df = df
        self.cat_features = categorical_features
        self.enc_type = encoding_type
        self.handle_na = handle_na
        self.label_encoders = dict()
        self.binary_encoders = dict()
        self.ohe = dict()

        if handle_na:
            for c in self.cat_features:
                self.df.loc[:, c] = self.df.loc[:, c].astype(str).fillna('-9999999999999999999')
            self.output_df = self.df.copy(deep=True)

    def _label_encoding_(self):

        for c in self.cat_features:
            lbl = preprocessing.LabelEncoder()
            lbl.fit(self.output_df[c].values)
            self.output_df.loc[:, c] = lbl.transform(self.output_df[c].values)

            self.label_encoders[c] = lbl

        if 'Unnamed: 0' in self.output_df.columns:
            self.output_df.drop(self.output_df.filter(regex="Unnamed"), axis=1, inplace=True)
        return self.output_df

    def _label_binarization(self):
        for c in self.cat_features:
            lbl = preprocessing.LabelBinarizer()
            lbl.fit(self.output_df[c].values)
            val = lbl.transform(self.output_df[c].values)  # Array
            self.output_df = self.output_df.drop(c, axis=1)
            for j in range(val.shape[1]):
                new_col_name = c + f"__bin_{j}"
                self.output_df[new_col_name] = val[:, j]
            self.binary_encoders[c] = lbl

        if 'Unnamed: 0' in self.output_df.columns:
            self.output_df.drop(self.output_df.filter(regex="Unnamed"), axis=1, inplace=True)
        return self.output_df

    def _one_hot(self):
        dummies = pd.get_dummies(self.output_df[self.cat_features], drop_first=True)
        dataframe = pd.concat([self.output_df, dummies], axis=1)
        dataframe.drop(self.cat_features, axis=1, inplace=True)
        if 'Unnamed: 0' in dataframe.columns:
            dataframe.drop(dataframe.filter(regex="Unnamed"), axis=1, inplace=True)
        return dataframe

    def fit_transform(self):
        if self.enc_type == 'label':
            return self._label_encoding_()
        elif self.enc_type == 'binary':
            return self._label_binarization()
        elif self.enc_type == 'ohe':
            return self._one_hot()
        else:
            raise Exception('Encoding type unknown')
    #
    # def transform(self, dataframe):
    #     if self.handle_na:
    #         for c in self.cat_features:
    #             dataframe.loc[:, c] = dataframe.loc[:, c].astype(str).fillna("-999999999")
    #     if self.enc_type == 'label':
    #         for c, lbl in self.label_encoders.items():
    #             dataframe.loc[:, c] = lbl.transform(dataframe[c].values)
    #         return dataframe
    #     elif self.enc_type == 'binary':
    #         print(self.binary_encoders.items())
    #         for c, lbl in self.binary_encoders.items():
    #             print(c)
    #             val = lbl.transform(dataframe[c].values)
    #             dataframe = dataframe.drop(c, axis=1)
    #             for j in range(val.shape[1]):
    #                 new_col_name = c + f"__bin_{j}"
    #                 self.dataframe[new_col_name] = val[:, j]
    #             print(dataframe)
    #         return dataframe
    #     else:
    #         raise Exception('Encoding type not understood')


import csv
import json
import os.path as osp

rows = []

# CSV got by https://www.alphavantage.co/query?function=TIME_SERIES_INTRADAY_EXTENDED&symbol=GME&interval=5min&slice=year1month2&apikey=ALPHA_VANTAGE_API_KEY
# Where ALPHA_VANTAGE_API_KEY is your API key.
with open('raw_data/GME_y1m1_5min.csv') as f:
	rows.extend(csv.DictReader(f))

per_day_data = {}

for row in rows:
	day, _, tm = row['time'].partition(' ')
	per_day_data.setdefault(day, []).append(row)

for day, dt in per_day_data.iteritems():
	dt.sort(key=lambda x:x['time'])
	list_of_rows = []
	for x in dt:
		list_of_rows.append(['', float(x['low']), float(x['open']), float(x['close']), float(x['high'])])
	with open(osp.join('real_stonks', day), 'w') as f:
		json.dump(list_of_rows, f)


import requests
from bs4 import BeautifulSoup
import csv

attri=[ 'Name', 'URL', 'Author', 'Price' , 'Number of Ratings' , 'Average Rating']

with open('in_book.csv','a') as fil:
          wriat= csv.writer(fil)
          wriat.writerow(attri)
          attri.clear()

for pg in range(1,5):
     infor=requests.get("https://www.amazon.in/gp/bestsellers/books/ref=zg_bs_pg_1?ie=UTF8&pg="+str(pg))
     data=infor.text
     soup=BeautifulSoup(data,'lxml')
     mov=soup.find_all('div',class_='zg_itemImmersion')

     for bst in range(20):
             try:
                   name=mov[bst].find_all('a',class_='a-link-normal')
                   attri.append(name[0].text)
             except:
                   attri.append("NOT AVAILABLE")
             try:
                   url=mov[bst].find_all('a',class_='a-section a-spacing-none p13n-asin')
                   attri.append(url[0].a['href'])
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   auth=mov[bst].find_all('a',class_='a-row a-size-small')
                   attri.append(auth[0].text)
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   ratnum=mov[bst].find_all('a',class_='a-link-normal a-text-normal')
                   attri.append("Rs"+str(name[0].text))
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   avgrat=mov[bst].find_all('a',class_='a-size-small a-link-normal')
                   attri.append(avgrat[0].text)
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   name=mov[bst].find_all('a',class_='a-link-normal')
                   attri.append(name[0].a['title'])
             except:
                  attri.append("NOT AVAILABLE")

             with open('in_book.csv','a') as fil:
                       wriat= csv.writer(fil)
                       wriat.writerow(attri)
                       attri.clear()


import brian2 as b2
from neurodynex.adex_model import AdEx
b2.defaultclock.dt = 1 * b2.ms

from neurodynex.tools import plot_tools, input_factory
import io
import math
import pdb
from numba import jit
import numpy as np
from .base import *
import quantities as qt
from quantities import mV, ms, s, us, ns, V
import matplotlib as mpl
SLOW_ZOOM = True
from neuronunit.capabilities import spike_functions as sf
mpl.use('Agg')
import matplotlib.pyplot as plt

getting_started = False
try:
    import asciiplotlib as apl
    fig = apl.figure()
    fig.plot([1,0], [0,1])
    ascii_plot = True
    import gc

except:
    ascii_plot = False
import numpy
try:
    brian2.clear_cache('cython')
except:
    pass
    #I_stim = stim, simulation_time=st)

from scipy.interpolate import interp1d

# This function implement Adaptive Exponential Leaky Integrate-And-Fire neuron model
def simulate_AdEx_neuron_local(
        tau_m=None,
        R=None,
        v_rest=None,#
        v_reset=None,#V_RESET,
        v_rheobase=None,#RHEOBASE_THRESHOLD_v_rh,
        a=None,#ADAPTATION_VOLTAGE_COUPLING_a,
        b=None,#SPIKE_TRIGGERED_ADAPTATION_INCREMENT_b,
        v_spike=None,#FIRING_THRESHOLD_v_spike,
        delta_T=None,#SHARPNESS_delta_T,
        tau_w=None,#ADAPTATION_TIME_CONSTANT_tau_w,
        I_stim=None,#input_factory.get_zero_current(),
        simulation_time=None):
    r"""
    code is from:
    /neurodynex/adex_model/AdEx.py

    Implementation of the AdEx model with a single adaptation variable w.

    The Brian2 model equations are:

    .. math::

        \frac{dv}{dt} = (-(v-v_rest) +delta_T*exp((v-v_rheobase)/delta_T)+ R * I_stim(t,i) - R * w)/(tau_m) : volt \\
        \frac{dw}{dt} = (a*(v-v_rest)-w)/tau_w : amp

    Args:
        tau_m (Quantity): membrane time scale
        R (Quantity): membrane restistance
        v_rest (Quantity): resting potential
        v_reset (Quantity): reset potential
        v_rheobase (Quantity): rheobase threshold
        a (Quantity): Adaptation-Voltage coupling
        b (Quantity): Spike-triggered adaptation current (=increment of w after each spike)
        v_spike (Quantity): voltage threshold for the spike condition
        delta_T (Quantity): Sharpness of the exponential term
        tau_w (Quantity): Adaptation time constant
        I_stim (TimedArray): Input current
        simulation_time (Quantity): Duration for which the model is simulated

    Returns:
        (state_monitor, spike_monitor):
        A b2.StateMonitor for the variables "v" and "w" and a b2.SpikeMonitor
    """
    v_spike_str = "v>{:f}*mvolt".format(v_spike / b2.mvolt)

    # EXP-IF
    eqs = """
        dv/dt = (-(v-v_rest) +delta_T*exp((v-v_rheobase)/delta_T)+ R * I_stim(t,i) - R * w)/(tau_m) : volt
        dw/dt=(a*(v-v_rest)-w)/tau_w : amp
        """

    neuron = b2.NeuronGroup(1, model=eqs, threshold=v_spike_str, reset="v=v_reset;w+=b", method="euler")

    # initial values of v and w is set here:
    neuron.v = v_rest
    neuron.w = 0.0 * b2.pA

    # Monitoring membrane voltage (v) and w
    state_monitor = b2.StateMonitor(neuron, ["v", "w"], record=True)
    spike_monitor = b2.SpikeMonitor(neuron)

    # running simulation
    b2.run(simulation_time)
    return state_monitor, spike_monitor

getting_started = False
class ADEXPBackend(Backend):

    name = 'adexp'

    def get_spike_count(self):
        return int(self.spike_monitor.count[0])
    def init_backend(self, attrs=None, cell_name='thembi',
                     current_src_name='spanner', DTC=None,
                     debug = False):
        super(ADEXPBackend,self).init_backend()
        self.debug = None
        self.model._backend.use_memory_cache = False
        self.current_src_name = current_src_name
        self.cell_name = cell_name
        self.vM = None
        self.attrs = attrs
        self.debug = debug
        self.temp_attrs = None
        self.n_spikes = None
        self.spike_monitor = None
        self.peak_v = 0.02
        self.verbose = False
        self.model.get_spike_count = self.get_spike_count

        if type(attrs) is not type(None):
            self.set_attrs(**attrs)
            self.sim_attrs = attrs

        if type(DTC) is not type(None):
            if type(DTC.attrs) is not type(None):
                self.set_attrs(**DTC.attrs)
            if hasattr(DTC,'current_src_name'):
                self._current_src_name = DTC.current_src_name
            if hasattr(DTC,'cell_name'):
                self.cell_name = DTC.cell_name

    def set_stop_time(self, stop_time = 650*pq.ms):
        """Sets the simulation duration
        stopTimeMs: duration in milliseconds
        """
        self.tstop = float(stop_time.rescale(pq.ms))

    def get_membrane_potential(self):
        """Must return a neo.core.AnalogSignal.
        And must destroy the hoc vectors that comprise it.
        """
        return self.vM

    def set_attrs(self, **attrs):

        self.AdEx = None
        self.AdEx = AdEx

        if len(attrs):
            self.AdEx.ADAPTATION_TIME_CONSTANT_tau_w = attrs['ADAPTATION_TIME_CONSTANT_tau_w']*AdEx.b2.units.ms
            self.AdEx.ADAPTATION_VOLTAGE_COUPLING_a = attrs['ADAPTATION_VOLTAGE_COUPLING_a']*AdEx.b2.units.nS
            self.AdEx.FIRING_THRESHOLD_v_spike = attrs['FIRING_THRESHOLD_v_spike']*AdEx.b2.units.mV*100
            self.AdEx.MEMBRANE_RESISTANCE_R = attrs['MEMBRANE_RESISTANCE_R']*AdEx.b2.units.Gohm
            self.AdEx.MEMBRANE_TIME_SCALE_tau_m = attrs['MEMBRANE_TIME_SCALE_tau_m']*AdEx.b2.units.ms
            self.AdEx.RHEOBASE_THRESHOLD_v_rh = attrs['RHEOBASE_THRESHOLD_v_rh']*AdEx.b2.units.mV*100
            self.AdEx.SHARPNESS_delta_T = attrs['SHARPNESS_delta_T']*AdEx.b2.units.mV*100
            self.AdEx.SPIKE_TRIGGERED_ADAPTATION_INCREMENT_b = attrs['SPIKE_TRIGGERED_ADAPTATION_INCREMENT_b']*AdEx.b2.units.pA
            self.AdEx.V_RESET = attrs['V_RESET']*AdEx.b2.units.mV
            self.AdEx.V_REST = attrs['V_REST']*AdEx.b2.units.mV*100

            if str('peak_v') in attrs:
                self.peak_v = attrs['peak_v']

            self.model.attrs.update(attrs)
        if attrs is None:
            #from neurodynex.adex_model import AdEx
            b2.defaultclock.dt = 1 * b2.ms

            self.AdEx =AdEx
    def finalize(self):
        '''
        Necessary for imputing missing sampling, simulating at high sample frequency is prohibitevely slow, with
        out significant difference in behavior.
        '''
        transform_function = interp1d([float(t) for t in self.vM.times],[float(v) for v in self.vM.magnitude])
        xnew = np.linspace(0, float(np.max(self.vM.times)), num=1004001, endpoint=True)
        vm_new = transform_function(xnew) #% generate the y values for all x values in xnew
        self.vM = AnalogSignal(vm_new,units = V,sampling_period = float(xnew[1]-xnew[0]) * pq.s)
        if self.verbose:

            print(len(self.vM))
        self.vM = AnalogSignal(vm_new,units = V,sampling_period = float(xnew[1]-xnew[0]) * pq.s)
        return self.vM
    def inject_square_current(self, current):#, section = None, debug=False):
        """Inputs: current : a dictionary with exactly three items, whose keys are: 'amplitude', 'delay', 'duration'
        Example: current = {'amplitude':float*pq.pA, 'delay':float*pq.ms, 'duration':float*pq.ms}}
        where \'pq\' is a physical unit representation, implemented by casting float values to the quanitities \'type\'.
        Description: A parameterized means of applying current injection into defined
        Currently only single section neuronal models are supported, the neurite section is understood to be simply the soma.

        """
        b2.defaultclock.dt = 1 * b2.ms
        self.state_monitor = None
        self.spike_monitor = None
        self.AdEx = None
        self.AdEx = AdEx
        attrs = copy.copy(self.model.attrs)
        if self.model.attrs is None or not len(self.model.attrs):
            self.AdEx = AdEx
        else:
            self.set_attrs(**attrs)
        if 'injected_square_current' in current.keys():
            c = current['injected_square_current']
        else:
            c = current

        #amplitude = c['amplitude'].simplified

        duration = int(c['duration'])#/dt#/dt.rescale('ms')
        delay = int(c['delay'])#/dt#.rescale('ms')
        pre_current = int(duration)+100
        amp = c['amplitude']#.rescale('uA')
        amplitude = amp.simplified
        if getting_started == False:
            stim = input_factory.get_step_current(delay, delay+duration, b2.ms, 1000.0*amplitude*b2.A)
            st = (duration+delay+200)* b2.ms
        else:
            stim = input_factory.get_step_current(10, 45, b2.ms, 7.2 * b2.uA)
            st = 70 * b2.ms

        if self.model.attrs is None or not len(self.model.attrs):
            #from neurodynex.adex_model import AdEx
            b2.defaultclock.dt = 1 * b2.ms

            self.AdEx = AdEx
            self.state_monitor, self.spike_monitor = simulate_AdEx_neuron_local(I_stim = stim, simulation_time=st)

        else:
            if self.verbose:
                #print(attrs)
                print(attrs['ADAPTATION_TIME_CONSTANT_tau_w'])

            if getting_started == True:

                stim = input_factory.get_step_current(10, 200, 1. * b2.ms, 65.0 * b2.pA)
                st = 300 * b2.ms
            self.set_attrs(**attrs)
            self.state_monitor, self.spike_monitor = self.AdEx.simulate_AdEx_neuron(
            tau_m = attrs['MEMBRANE_TIME_SCALE_tau_m']*AdEx.b2.units.ms,
            R = np.abs(attrs['MEMBRANE_RESISTANCE_R'])*AdEx.b2.units.Gohm,
            v_rest = attrs['V_REST']*AdEx.b2.units.V,
            v_reset = attrs['V_RESET']*AdEx.b2.units.V,
            v_rheobase = attrs['RHEOBASE_THRESHOLD_v_rh']*AdEx.b2.units.V,
            a = attrs['ADAPTATION_VOLTAGE_COUPLING_a']*AdEx.b2.units.nS,
            b =  attrs['b']*b2.pA,
            v_spike=attrs['FIRING_THRESHOLD_v_spike']*AdEx.b2.units.V,
            delta_T = attrs['SHARPNESS_delta_T']*AdEx.b2.units.V,
            tau_w = attrs['ADAPTATION_TIME_CONSTANT_tau_w']*AdEx.b2.units.ms,
            I_stim = stim, simulation_time=st)

        #self.state_monitor.clock.dt = 1 *b2.ms
        self.dt = self.state_monitor.clock.dt

        state_dic = self.state_monitor.get_states()
        vm = state_dic['v']
        vm = [ float(i) for i in vm ]

        self.vM = AnalogSignal(vm,units = mV,sampling_period = float(0.0001) * pq.s)

        tdic = self.spike_monitor.spike_trains()
        if str('peak_v') in attrs.keys():
            self.peak_v = attrs['peak_v']
        else:
            self.peak_v = 2
        if self.verbose:

            print(self.peak_v)
        for key,value in tdic.items():

            if len(value)==1:
                i = int(float(value)/0.001)
                self.vM[i] = self.peak_v*qt.mV
            else:
                for v in value:
                    i = int(float(v)/0.001)
                    self.vM[i] = self.peak_v*qt.mV

        self.n_spikes = int(self.spike_monitor.count[0])
        self.attrs = attrs

        if ascii_plot:
            if SLOW_ZOOM and self.get_spike_count()>=1 :
                from neuronunit.capabilities.spike_functions import get_spike_waveforms
                vm = get_spike_waveforms(self.vM)
            else:
                vm = self.vM
            try:
                t = [float(f) for f in vm.times]
                v = [float(f) for f in vm.magnitude]
                fig = apl.figure()
                fig.plot(t, v, label=str('brain adexp: ')+str(vm.units), width=100, height=20)
                fig.show()
                gc.collect()
                fig = None
            except:
                print('strange error')
        return self.vM

    def _backend_run(self):
        results = None
        results = {}

        results['vm'] = self.vM

        results['t'] = self.vM.times
        results['run_number'] = results.get('run_number',0) + 1
        return results


from django.urls import path
from . import views

urlpatterns = [
    path('',views.ArtilceListView.as_view(),name='home'),
    path(
        'articles/<int:pk>',
        views.ArticleDetailView.as_view(),
        name='article_detail'
    ),
    path(
        'articles/profile',
        views.ArtilceProfileListView.as_view(),
        name='profile_articles'
    ),
    path(
        'articles/create',
        views.ArticleCreateView.as_view(),
        name='article_create'
        ),
    path(
        'articles/<int:pk>/delete',
        views.ArticleDeleteView.as_view(),
        name='article_delete'
    ),
    path(
        'articles/<int:pk>/update',
        views.ArticleUpdateView.as_view(),
        name='article_update'
    ),
    path(
        'comment/add',
        views.CommentCreateView.as_view(),
        name='comment_create'
    ),
]


# -*- coding: utf-8 -*-
"""
Created on Mon Feb  4 17:49:56 2019

@author: tiwarisudhir
"""

import gym
import torch
import numpy as np
from collections import deque
from unityagents import UnityEnvironment
import qnetwork

# set device placement

SEED            = 0
MEM_BUFFER      = int(1e5)
MEM_BATCH       = 64
GAMMA           = 0.99
TAU             = 1e-3
LEARNING_RATE   = 5e-4
UPDATE_FREQ     = 4
CHECKPOINT_NAME = 'checkpoint.pth'
MIN_THRESHOLD   = 14.
device          = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def get_environment():
    '''
    Get gym environment and brain for the navigator
    '''

    env = UnityEnvironment(file_name="/data/Banana_Linux_NoVis/Banana.x86_64")

    return env

def get_agent(env):
    '''
    Get DQN agent and environment brain
    '''

    brain_name = env.brain_names[0]
    brain = env.brains[brain_name]
    env_info = env.reset(train_mode=True)[brain_name]

    states = len(env_info.vector_observations[0])
    actions = brain.vector_action_space_size

    agent = qnetwork.DQNAgent(states, actions, seed = SEED, device = device,
                              mem_buffer = MEM_BUFFER, mem_batch = MEM_BATCH,
                              gamma = GAMMA, tau = TAU,
                              learning_rate = LEARNING_RATE, update_freq = UPDATE_FREQ)

    return agent

def train_navigator(env, agent, episodes = 2000, max_steps = None, epsilon_max = 1., epsilon_min = 0.01, epsilon_decay = 0.997):
    '''
    Train a DQN for navigator to collect Bananas
    param episodes: number of episodes to train for
    param max_steps: max number of time steps per episode
    param epsilon_max: starting epsilon value for greedy policy selection
    param epsilon_end: min value of the epsilon for greedy policy selection
    param epsilon_decay: decay rate of epsilon per episode
    '''

    # monitor learning by keeping track of rewards
    epsilon             = epsilon_max
    best_avg_rewards    = 0.
    episode_rewards     = []
    window_rewards      = deque(maxlen = 100)

    for episode in range(episodes):

        step       = 0
        rewards    = 0.
        brain_name = env.brain_names[0]
        env_info   = env.reset(train_mode = True)[brain_name]
        state      = env_info.vector_observations[0]

        while (max_steps == None) or (step < max_steps):

            # get agent's action
            action = agent.select_action(state, epsilon)

            # take action that agent has selected
            env_info = env.step(action)[brain_name]

            # get the transitioned state and reward
            new_state = env_info.vector_observations[0]
            reward   = env_info.rewards[0]
            done     = env_info.local_done[0]

            # pass the reward to agent
            agent.experience(state, action, reward, new_state, done)

            # transition to new state
            state = new_state
            rewards += reward

            if done:
                break

            step += 1

        # decay epsilon
        epsilon = max(epsilon_min, epsilon * epsilon_decay)

        # record rewards
        episode_rewards.append(rewards)
        window_rewards.append(rewards)

        print('\r Episode-{}, Eps: {:.4f}, Avg Reward: {:.2f}'.format((episode + 1), epsilon, np.mean(window_rewards)), end = "")

        if (episode + 1) % 100 == 0:
            print('\r Episode-{}, Avg Reward: {:.2f}'.format((episode + 1), np.mean(window_rewards)))

        if (np.mean(window_rewards) >= MIN_THRESHOLD) and (np.mean(window_rewards) > best_avg_rewards):

            best_avg_rewards = np.mean(window_rewards)

            print('\n Agent has learned to solve the task. Episode-{}, Avg Reward: {:.2f}'.format((episode + 1), best_avg_rewards))

            agent.save_memory(checkpoint_name = CHECKPOINT_NAME)

    return episode_rewards

def restore_memory(agent):
    '''
    Restore agent's past learning
    '''

    agent.restore_memory(checkpoint_name = CHECKPOINT_NAME)


import cv2

cap = cv2.VideoCapture()
cap.open('Safety_Full_Hat_and_Vest.mp4')
# n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
# print(n_frames)
print(cap.get(cv2.CAP_PROP_FPS))
cv2.namedWindow('video')

# for i in range(n_frames):
#     ret, frame = cap.read()
#     cv2.imshow('video', frame)
#     if cv2.waitKey(1) == 27:
#         break
while cap.isOpened:
    ret, frame = cap.read()
    cv2.imshow('video', frame)
    if cv2.waitKey(30) == 27:
        break


import json
from sqlalchemy.inspection import inspect
import sqlalchemy.orm

from datetime import date, datetime

def jsonify(query):
    result = []

    for row in query:
        to_append = {}

        # get object columns and data
        for column in row.__table__.columns:
            column_header = str(column).split('.')[1]
            to_append[column_header] = getattr(row, column_header)

        # get objects relationship columns and data
        inspect_row = inspect(row).__dict__['class_'].__dict__.keys()

        for column_name in inspect_row:
            if not column_name.startswith('_'):
                if column_name not in row.__table__.columns:
                    if '_sa_adapter' in getattr(row, column_name).__dict__.keys():
                        # 'sqlalchemy.orm.collections.InstrumentedList'
                        item_collect = []

                        for relationship_row in getattr(row, column_name):
                            item = {}
                            for data in relationship_row.__table__.columns:
                                item[str(data).split('.')[1]] = getattr(relationship_row, str(data).split('.')[1])

                            item_collect.append(item)

                        to_append[column_name] = item_collect

                    else:
                        # 'sqlalchemy.orm.dynamic.AppenderBaseQuery'
                        item_collect = []

                        for relationship_row in getattr(row, column_name):
                            item = {}
                            for data in relationship_row.__table__.columns:
                                item[str(data).split('.')[1]] = getattr(relationship_row, str(data).split('.')[1])

                            item_collect.append(item)

                        to_append[column_name] = item_collect

        result.append(to_append)

    return result


#!/usr/bin/env python
# coding: utf-8

# # Targeting Direct Marketing
#
# ---
#
# ## Background
# Direct marketing, either through mail, email, phone, etc., is a common tactic to acquire customers. Because resources and a customer's attention are limited, the goal is to only target the subset of prospects who are likely to engage with a specific offer. Predicting those potential customers based on readily available information like demographics, past interactions, and environmental factors is a common machine learning problem.
#
# This notebook presents an example problem to predict if a customer will enroll for a term deposit at a bank after one or more phone calls. The steps include:
#
# * Preparing your Amazon SageMaker notebook
# * Downloading data from the internet into Amazon SageMaker
# * Investigating and transforming the data so that it can be fed to Amazon SageMaker algorithms
# * Estimating a model
# * Evaluating the effectiveness of the model
# * Setting the model up to make ongoing predictions
#
# ---
#
# ## Preparation
#
# _This notebook was created and tested on an ml.m4.xlarge notebook instance._
#
# Start by specifying:
#
# - The Amazon Simple Storage Service (Amazon S3) bucket and prefix that you want to use for training and model data. This should be within the same Region as the Notebook Instance, training, and hosting.
# - The AWS Identity and Access Management (IAM) role Amazon Resource Name (ARN) used to give training and hosting access to your data. See the documentation for how to create these.
#
# **Note:** If more than one role is required for notebook instances, training, and/or hosting, please replace the boto regexp with the appropriate full IAM role ARN string(s).
#
# Replace **`<LabBucketName>`** with the resource name that was provided with your lab account.

# In[1]:

#bucket = 'ml-pipeline-bucket'
bucket = 'ml-pipeline-bucket'
prefix = 'sagemaker/DEMO-xgboost-dm'

# Define IAM role
import boto3
import re
from sagemaker import get_execution_role

role = get_execution_role()

# Bring in the Python libraries that you'll use throughout the analysis.

# In[2]:

import numpy as np                                # For matrix operations and numerical processing
import pandas as pd                               # For munging tabular data
import matplotlib.pyplot as plt                   # For charts and visualizations
from IPython.display import Image                 # For displaying images in the notebook
from IPython.display import display               # For displaying outputs in the notebook
from time import gmtime, strftime                 # For labeling Amazon SageMaker models, endpoints, etc.
import sys                                        # For writing outputs to the notebook
import math                                       # For ceiling function
import json                                       # For parsing hosting outputs
import os                                         # For manipulating filepath names
import sagemaker                                  # Amazon SageMaker's Python SDK provides many helper functions
from sagemaker.predictor import CSVSerializer    # Converts strings for HTTP POST requests on inference

# ---
#
# ## Data
# Start by downloading the [Bank Marketing Data Set](https://archive.ics.uci.edu/ml/datasets/bank+marketing) from the University of California, Irvine (UCI) Machine Learning ML Repository.
#
# **Source information:**
#
# S. Moro, P. Cortez and P. Rita. A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems, Elsevier, 62:22-31, June 2014

# In[3]:

get_ipython().system('aws s3 cp s3://aws-tc-largeobjects/ILT-TF-200-MLDWTS/lab1/bank-additional.zip .')
#!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank-additional.zip
get_ipython().system('unzip -o bank-additional.zip')

# Now, read this into a Pandas dataframe and take a look.

# In[4]:

data = pd.read_csv('./bank-additional/bank-additional-full.csv', sep=';')
pd.set_option('display.max_columns', 500)     # Makes sure you can see all of the columns
pd.set_option('display.max_rows', 20)         # Keeps the output on one page

# In[5]:

data.head()

# Let's talk about the data.  At a high level, you can see:
#
# * There are just over 40K customer records and 20 features for each customer.
# * The features are mixed--some numeric, some categorical.
# * The data appears to be sorted, at least by `time` and `contact`.
#

# ### Exploration
# Let's explore the data. First, let's understand how the features are distributed.

# In[6]:

# Frequency tables for each categorical feature
for column in data.select_dtypes(include=['object']).columns:
    display(pd.crosstab(index=data[column], columns='% observations', normalize='columns')*100)

# In[7]:

# Histograms for each numeric feature
display(data.describe())
display(data.describe(include=np.object))
#%matplotlib inline
hist = data.hist(bins=30, sharey=True, figsize=(10, 10))

# Notice that:
#
# - Almost 90% of the values for our target variable y are "no", so most customers did not subscribe to a term deposit.
# - Many of the predictive features take on values of "unknown". Some are more common than others. We should think carefully as to what causes a value of "unknown" (are these customers non-representative in some way?) and how to handle that.
# - Even if "unknown" is included as its own distinct category, what does it mean, given that those observations likely fall within one of the other categories of that feature?
# - Many of the predictive features have categories with few observations in them. If we find a small category to be highly predictive of our target outcome, do we have enough evidence to make a generalization about that?
# - Contact timing is particularly skewed--almost a third in May and less than 1% in December. What does this mean for predicting our target variable next December?
# - There are no missing values in our numeric features, or missing values have already been imputed.
#   - `pdays` takes a value near 1,000 for almost all customers. This is likely a placeholder value signifying no previous contact.
# - Several numeric features have a long tail. Do we need to handle these few observations with extremely large values differently?
# - Several numeric features (particularly the macroeconomic ones) occur in distinct buckets. Should these be treated as categorical?
#
# Next, let's look at how our features relate to the target that we are attempting to predict.

# In[8]:

for column in data.select_dtypes(include=['object']).columns:
    if column != 'y':
        display(pd.crosstab(index=data[column], columns=data['y'], normalize='columns'))

for column in data.select_dtypes(exclude=['object']).columns:
    print(column)
    hist = data[[column, 'y']].hist(by='y', bins=30)
    plt.show()

# Notice that:
# - Customers who are "blue-collar", "married", "unknown" default status, contacted by "telephone", and/or in "May" are a substantially lower portion of "yes" than "no" for subscribing.
# - Distributions for numeric variables are different across "yes" and "no" subscribing groups, but the relationships may not be straightforward or obvious.
#
# Now let's look at how our features relate to one another.

# In[9]:

display(data.corr())
pd.plotting.scatter_matrix(data, figsize=(12, 12))
plt.show()

# Notice that:
#
# - Features vary widely in their relationships with one another. Some have highly negative correlation; others have highly positive correlation.
# - Relationships between features are non-linear and discrete in many cases.

# ### Transformation
#
# Cleaning up data is part of nearly every machine learning project.  It arguably presents the biggest risk if done incorrectly and is one of the more subjective aspects in the process.

# In[10]:

data['no_previous_contact'] = np.where(data['pdays'] == 999, 1, 0)                                 # Indicator variable to capture when pdays takes a value of 999
data['not_working'] = np.where(np.in1d(data['job'], ['student', 'retired', 'unemployed']), 1, 0)   # Indicator for individuals not actively employed
model_data = pd.get_dummies(data)                                                                  # Convert categorical variables to sets of indicators

# In[11]:

model_data = model_data.drop(['duration', 'emp.var.rate', 'cons.price.idx', 'cons.conf.idx', 'euribor3m', 'nr.employed'], axis=1)

# ### Train-test split

# In[12]:

train_data, validation_data, test_data = np.split(model_data.sample(frac=1, random_state=1729), [int(0.7 * len(model_data)), int(0.9 * len(model_data))])   # Randomly sort the data; then split out first 70%, second 20%, and last 10%

# Amazon SageMaker's XGBoost container expects data in the libSVM or CSV data format. For this example, you'll stick to CSV. Note that the first column must be the target variable, and the CSV should not include headers. Also, notice that although repetitive, it's easiest to do this after the train|validation|test split rather than before. This avoids any misalignment issues due to random reordering.

# In[13]:

pd.concat([train_data['y_yes'], train_data.drop(['y_no', 'y_yes'], axis=1)], axis=1).to_csv('train.csv', index=False, header=False)
pd.concat([validation_data['y_yes'], validation_data.drop(['y_no', 'y_yes'], axis=1)], axis=1).to_csv('validation.csv', index=False, header=False)

# Now, copy the file to Amazon S3 for Amazon SageMaker's managed training to pick up.

# In[14]:

boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train/train.csv')).upload_file('train.csv')
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'validation/validation.csv')).upload_file('validation.csv')

# ---
#
# ## Training
#
# First, you need to specify the Amazon Elastic Container Registry (Amazon ECR) container location for Amazon SageMaker's implementation of XGBoost.

# In[15]:

container = sagemaker.image_uris.retrieve('xgboost',boto3.Session().region_name, '1.0-1')

# Then, because you're training with the CSV file format, create `s3_input`s for the training function to use as a pointer to the files in Amazon S3. This also specifies that the content type is CSV.

# In[16]:

s3_input_train = sagemaker.inputs.TrainingInput(s3_data='s3://{}/{}/train'.format(bucket, prefix), content_type='csv')
s3_input_validation = sagemaker.inputs.TrainingInput(s3_data='s3://{}/{}/validation/'.format(bucket, prefix), content_type='csv')

# Now, you need to specify training parameters to the estimator. This includes:
# - The `xgboost` algorithm container
# - The IAM role to use
# - Training instance type and count
# - Amazon S3 location for output data
# - Algorithm hyperparameters
#
# And then a `.fit()` function, which specifies the Amazon S3 location for output data. In this case, both a training and validation set are passed in.

# In[17]:

sess = sagemaker.Session()

xgb = sagemaker.estimator.Estimator(container,
                                    role,
                                    instance_count=1,
                                    instance_type='ml.m4.xlarge',
                                    output_path='s3://{}/{}/output'.format(bucket, prefix),
                                    sagemaker_session=sess)
xgb.set_hyperparameters(max_depth=5,
                        eta=0.2,
                        gamma=4,
                        min_child_weight=6,
                        subsample=0.8,
                        silent=0,
                        objective='binary:logistic',
                        num_round=100)

xgb.fit({'train': s3_input_train, 'validation': s3_input_validation})

# ---
#
# ## Hosting
# Now that you've trained the `xgboost` algorithm on the data, deploy a model that's hosted behind a real-time endpoint.
#
# First, you need to determine how to pass data into and receive data from the endpoint. The data is currently stored as NumPy arrays in the memory of your notebook instance. To send it in an HTTP POST request, you'll serialize it as a CSV string and then decode the resulting CSV.
#
# **Note:** For inference with the CSV format, Amazon SageMaker XGBoost requires that the data does NOT include the target variable.

# In[18]:

xgb_predictor = xgb.deploy(initial_instance_count=1,
                           serializer=CSVSerializer(),
                           instance_type='ml.m4.xlarge')

# ---
#
# ## Evaluation
# There are many ways to compare the performance of a machine learning model, but start by simply comparing actual to predicted values. In this case, you're simply predicting whether the customer subscribed to a term deposit (`1`) or not (`0`), which produces a simple confusion matrix.
#

# Now, use a simple function to:
#
# 1. Loop over the test dataset
# 1. Split it into mini-batches of rows
# 1. Convert those mini-batches to CSV string payloads (notice that the target variable is dropped from the dataset first)
# 1. Retrieve mini-batch predictions by invoking the XGBoost endpoint
# 1. Collect predictions and convert from the CSV output that the model provides into a NumPy array

# In[19]:

def predict(data, rows=500):
    split_array = np.array_split(data, int(data.shape[0] / float(rows) + 1))
    predictions = ''
    for array in split_array:
        predictions = ','.join([predictions, xgb_predictor.predict(array).decode('utf-8')])

    return np.fromstring(predictions[1:], sep=',')

predictions = predict(test_data.drop(['y_no', 'y_yes'], axis=1).values)

# Now, check the confusion matrix to see how well the model predicted vs. actuals.

# In[20]:

pd.crosstab(index=test_data['y_yes'], columns=np.round(predictions), rownames=['actuals'], colnames=['predictions'])

# Of about 4,000 potential customers, the model predicted that 136 would subscribe, and 94 actually did. There were also 389 customers who subscribed that were not predicted to subscribe. This is less than desirable, but the model can (and should) be tuned to improve this. Most importantly, note that with minimal effort, the model produced accuracies similar to those published [here](http://media.salford-systems.com/video/tutorial/2015/targeted_marketing.pdf).
#
# **Note:** Because there is some element of randomness in the algorithm's subsample, your results may differ slightly from the text written above.

# ### Cleanup
#
# If you are finished with this notebook, run the following cell. This removes the hosted endpoint you created and avoids any charges from a stray instance being left on.

# In[21]:

sagemaker.Session().delete_endpoint(xgb_predictor.endpoint_name)

# In[ ]:


import json
import codecs
import os

def load_data(file_path):
    if os.path.exists(file_path):
        with codecs.open(file_path, 'r', 'utf-8') as file_handler:
            return json.load(file_handler)
    else:
        raise SystemExit('Wrong file path')

def get_biggest_bar(bars_list):
    return [max(bars_list, key=lambda x:
            x['SeatsCount'])[x] for x in ['Name', 'Address']]

def get_smallest_bar(bars_list):
    return [min(bars_list, key=lambda x:
            x['SeatsCount'])[x] for x in ['Name', 'Address']]

def get_closest_bar(bars_list, usr_longitude, usr_latitude):
    return [min(bars_list, key=lambda x:
            abs(float(x['Latitude_WGS84']) - usr_latitude) +
            abs(float(x['Longitude_WGS84']) - usr_longitude))[x]
            for x in ['Name', 'Address']]

def get_user_coordinate(coordinate_name):
    try:
        return float(input('Enter your'
                           ' current {}\n'.format(coordinate_name.lower())))
    except ValueError:
        raise SystemExit('{} must be a number'.format(coordinate_name))

if __name__ == '__main__':
    list_of_bars = load_data(input('Enter file path\n'))
    user_longitude = get_user_coordinate('Longitude')
    user_latitude = get_user_coordinate('Latitude')
    print('The biggest bar is {},'
          ' located at {}'.format(*get_biggest_bar(list_of_bars)))
    print('The smallest bar is {},'
          ' located at {}'.format(*get_smallest_bar(list_of_bars)))
    print('The closest bar is {},'
          ' located at {}'.format(*get_closest_bar(list_of_bars,
                                                   user_longitude,
                                                   user_latitude)))


import girder_worker
from girder_worker import logger
import traceback as tb
import celery
from celery import Celery, __version__
from distutils.version import LooseVersion
from celery.signals import (task_prerun, task_postrun,
                            task_failure, task_success,
                            worker_ready, before_task_publish, task_revoked)
from celery.result import AsyncResult

from celery.task.control import inspect

from six.moves import configparser
import sys
from .utils import JobStatus, StateTransitionException
from girder_client import GirderClient

class GirderAsyncResult(AsyncResult):
    def __init__(self, *args, **kwargs):
        self._job = None
        super(GirderAsyncResult, self).__init__(*args, **kwargs)

    @property
    def job(self):
        if self._job is None:
            try:
                # GirderAsyncResult() objects may be instantiated in either a girder REST
                # request,  or in some other context (e.g. from a running girder_worker
                # instance if there is a chain).  If we are in a REST request we should
                # have access to the girder package and can directly access the database
                # If we are in a girder_worker context (or even in python console or a
                # testing context) then we should get an ImportError and we can make a REST
                # request to get the information we need.
                from girder.utility.model_importer import ModelImporter
                job_model = ModelImporter.model('job', 'jobs')

                try:
                    return job_model.findOne({'celeryTaskId': self.task_id})
                except IndexError:
                    return None

            except ImportError:
                # Make a rest request to get the job info
                return None

        return self._job

class Task(celery.Task):
    """Girder Worker Task object"""

    _girder_job_title = '<unnamed job>'
    _girder_job_type = 'celery'
    _girder_job_public = False
    _girder_job_handler = 'celery_handler'
    _girder_job_other_fields = {}

    # These keys will be removed from apply_async's kwargs or options and
    # transfered into the headers of the message.
    reserved_headers = [
        'girder_client_token',
        'girder_api_url']

    # These keys will be available in the 'properties' dictionary inside
    # girder_before_task_publish() but will not be passed along in the message
    reserved_options = [
        'girder_user',
        'girder_job_title',
        'girder_job_type',
        'girder_job_public',
        'girder_job_handler',
        'girder_job_other_fields']

    def AsyncResult(self, task_id, **kwargs):
        return GirderAsyncResult(task_id, backend=self.backend,
                                 task_name=self.name, app=self.app, **kwargs)

    def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
                    link=None, link_error=None, shadow=None, **options):

        # Pass girder related job information through to
        # the signals by adding this information to options['headers']
        # This sets defaults for reserved_options based on the class defaults,
        # or values defined by the girder_job() dectorator
        headers = {
            'girder_job_title': self._girder_job_title,
            'girder_job_type': self._girder_job_type,
            'girder_job_public': self._girder_job_public,
            'girder_job_handler': self._girder_job_handler,
            'girder_job_other_fields': self._girder_job_other_fields,
        }

        # Certain keys may show up in either kwargs (e.g. via .delay(girder_token='foo')
        # or in options (e.g.  .apply_async(args=(), kwargs={}, girder_token='foo')
        # For those special headers,  pop them out of kwargs or options and put them
        # in headers so they can be picked up by the before_task_publish signal.
        for key in self.reserved_headers + self.reserved_options:
            if kwargs is not None and key in kwargs:
                headers[key] = kwargs.pop(key)
            if key in options:
                headers[key] = options.pop(key)

        if 'headers' in options:
            options['headers'].update(headers)
        else:
            options['headers'] = headers

        return super(Task, self).apply_async(
            args=args, kwargs=kwargs, task_id=task_id, producer=producer,
            link=link, link_error=link_error, shadow=shadow, **options)

    @property
    def canceled(self):
        """
        A property to indicate if a task has been canceled.

        :returns True is this task has been canceled, False otherwise.
        """
        return is_revoked(self)

    def describe(self):
        # The describe module indirectly depends on this module, so to
        # avoid circular imports, we import describe_function here.
        from describe import describe_function
        return describe_function(self.run)

    def call_item_task(self, inputs, outputs={}):
        return self.run.call_item_task(inputs, outputs)

@before_task_publish.connect
def girder_before_task_publish(sender=None, body=None, exchange=None,
                               routing_key=None, headers=None, properties=None,
                               declare=None, retry_policy=None, **kwargs):
    if 'jobInfoSpec' not in headers:
        try:
            # Note: If we can import these objects from the girder packages we
            # assume our producer is in a girder REST request. This allows
            # us to create the job model's directly. Otherwise there will be an
            # ImportError and we can create the job via a REST request using
            # the jobInfoSpec in headers.
            from girder.utility.model_importer import ModelImporter
            from girder.plugins.worker import utils
            from girder.api.rest import getCurrentUser

            job_model = ModelImporter.model('job', 'jobs')

            user = headers.pop('girder_user', getCurrentUser())
            task_args, task_kwargs = body[0], body[1]

            job = job_model.createJob(
                **{'title': headers.pop('girder_job_title', Task._girder_job_title),
                   'type': headers.pop('girder_job_type', Task._girder_job_type),
                   'handler': headers.pop('girder_job_handler', Task._girder_job_handler),
                   'public': headers.pop('girder_job_public', Task._girder_job_public),
                   'user': user,
                   'args': task_args,
                   'kwargs': task_kwargs,
                   'otherFields': dict(celeryTaskId=headers['id'],
                                       **headers.pop('girder_job_other_fields',
                                                     Task._girder_job_other_fields))})

            headers['jobInfoSpec'] = utils.jobInfoSpec(job)

        except ImportError:
            # TODO: Check for self.job_manager to see if we have
            #       tokens etc to contact girder and create a job model
            #       we may be in a chain or a chord or some-such
            pass

    if 'girder_api_url' not in headers:
        try:
            from girder.plugins.worker import utils
            headers['girder_api_url'] = utils.getWorkerApiUrl()
        except ImportError:
            # TODO: handle situation where girder_worker is producing the message
            #       Note - this may not come up at all depending on how we pass
            #       girder_api_url through to the next task (e.g. in the context
            #       of chaining events)
            pass

    if 'girder_client_token' not in headers:
        try:
            from girder.utility.model_importer import ModelImporter
            headers['girder_client_token'] = ModelImporter.model('token').createToken()
        except ImportError:
            # TODO: handle situation where girder_worker is producing the message
            #       Note - this may not come up at all depending on how we pass
            #       girder_token through to the next task (e.g. in the context
            #       of chaining events)
            pass

    # Finally,  remove all reserved_options from headers
    for key in Task.reserved_options:
        headers.pop(key, None)

@worker_ready.connect
def check_celery_version(*args, **kwargs):
    if LooseVersion(__version__) < LooseVersion('4.0.0'):
        sys.exit("""You are running Celery {}.

girder-worker requires celery>=4.0.0""".format(__version__))

def deserialize_job_info_spec(**kwargs):
    return girder_worker.utils.JobManager(**kwargs)

class JobSpecNotFound(Exception):
    pass

def _job_manager(request=None, headers=None, kwargs=None):
    if hasattr(request, 'jobInfoSpec'):
        jobSpec = request.jobInfoSpec

    # We are being called from revoked signal
    elif headers is not None and \
            'jobInfoSpec' in headers:
        jobSpec = headers['jobInfoSpec']

    # Deprecated: This method of passing job information
    # to girder_worker is deprecated. Newer versions of girder
    # pass this information automatically as apart of the
    # header metadata in the worker scheduler.
    elif 'jobInfo' in kwargs:
        jobSpec = kwargs.pop('jobInfo', {})

    else:
        raise JobSpecNotFound

    return deserialize_job_info_spec(**jobSpec)

def _update_status(task, status):
    # For now,  only automatically update status if this is
    # not a child task. Otherwise child tasks completion will
    # update the parent task's jobModel in girder.
    if task.request.parent_id is None:
        task.job_manager.updateStatus(status)

@task_prerun.connect
def gw_task_prerun(task=None, sender=None, task_id=None,
                   args=None, kwargs=None, **rest):
    """Deserialize the jobInfoSpec passed in through the headers.

    This provides the a JobManager class as an attribute of the
    task before task execution.  decorated functions may bind to
    their task and have access to the job_manager for logging and
    updating their status in girder.
    """
    try:
        task.job_manager = _job_manager(task.request, task.request.headers)
        _update_status(task, JobStatus.RUNNING)

    except JobSpecNotFound:
        task.job_manager = None
        logger.warn('No jobInfoSpec. Setting job_manager to None.')
    except StateTransitionException:
        # Fetch the current status of the job
        status = task.job_manager.refreshStatus()
        # If we are canceling we want to stay in that state
        if status != JobStatus.CANCELING:
            raise

    try:
        task.girder_client = GirderClient(apiUrl=task.request.girder_api_url)
        task.girder_client.token = task.request.girder_client_token
    except AttributeError:
        task.girder_client = None

@task_success.connect
def gw_task_success(sender=None, **rest):
    try:

        if not is_revoked(sender):
            _update_status(sender, JobStatus.SUCCESS)

        # For tasks revoked directly
        else:
            _update_status(sender, JobStatus.CANCELED)
    except AttributeError:
        pass
    except StateTransitionException:
        # Fetch the current status of the job
        status = sender.job_manager.refreshStatus()
        # If we are in CANCELING move to CANCELED
        if status == JobStatus.CANCELING or is_revoked(sender):
            _update_status(sender, JobStatus.CANCELED)
        else:
            raise

@task_failure.connect
def gw_task_failure(sender=None, exception=None,
                    traceback=None, **rest):
    try:

        msg = '%s: %s\n%s' % (
            exception.__class__.__name__, exception,
            ''.join(tb.format_tb(traceback)))

        sender.job_manager.write(msg)

        _update_status(sender, JobStatus.ERROR)
    except AttributeError:
        pass

@task_postrun.connect
def gw_task_postrun(task=None, sender=None, task_id=None,
                    args=None, kwargs=None,
                    retval=None, state=None, **rest):
    try:
        task.job_manager._flush()
        task.job_manager._redirectPipes(False)
    except AttributeError:
        pass

@task_revoked.connect
def gw_task_revoked(sender=None, request=None, **rest):
    try:
        sender.job_manager = _job_manager(headers=request.message.headers,
                                          kwargs=request.kwargsrepr)
        _update_status(sender, JobStatus.CANCELED)
    except AttributeError:
        pass
    except JobSpecNotFound:
        logger.warn('No jobInfoSpec. Unable to move \'%s\' into CANCELED state.')

# Access to the correct "Inspect" instance for this worker
_inspector = None

def _worker_inspector(task):
    global _inspector
    if _inspector is None:
        _inspector = inspect([task.request.hostname])

    return _inspector

# Get this list of currently revoked tasks for this worker
def _revoked_tasks(task):
    return _worker_inspector(task).revoked().get(task.request.hostname, [])

def is_revoked(task):
    """
    Utility function to check is a task has been revoked.

    :param task: The task.
    :type task: celery.app.task.Task
    :return True, if this task is in the revoked list for this worker, False
            otherwise.
    """
    return task.request.id in _revoked_tasks(task)

class _CeleryConfig:
    CELERY_ACCEPT_CONTENT = ['json', 'pickle', 'yaml']

broker_uri = girder_worker.config.get('celery', 'broker')
try:
    backend_uri = girder_worker.config.get('celery', 'backend')
except configparser.NoOptionError:
    backend_uri = broker_uri

app = Celery(
    main=girder_worker.config.get('celery', 'app_main'),
    backend=backend_uri, broker=broker_uri,
    task_cls='girder_worker.app:Task')

app.config_from_object(_CeleryConfig)


import synapse.axon as s_axon
import synapse.daemon as s_daemon
import synapse.telepath as s_telepath

import synapse.lib.service as s_service

from synapse.tests.common import *

craphash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
asdfhash = '6a204bd89f3c8348afd5c77c717a097a'

class AxonTest(SynTest):

    def test_axon_basics(self):
        with self.getTestDir() as axondir:

            axon = s_axon.Axon(axondir)

            self.false(axon.has('md5', craphash))
            self.false(axon.has('md5', asdfhash))

            iden0 = axon.alloc(8)

            self.nn(axon.chunk(iden0, b'asdfasdf'))

            self.true(axon.has('md5', asdfhash))
            self.false(axon.has('md5', craphash))

            byts = b''.join(axon.bytes('md5', asdfhash))

            self.eq(byts, b'asdfasdf')

            axon.fini()

            axon = s_axon.Axon(axondir)

            self.true(axon.has('md5', asdfhash))
            self.false(axon.has('md5', craphash))

            byts = b''.join(axon.bytes('md5', asdfhash))

            self.eq(byts, b'asdfasdf')

            self.none(axon.wants('md5', asdfhash, 8))
            self.nn(axon.wants('md5', craphash, 8))

            axon.fini()

    def test_axon_sync(self):

        with self.getTestDir() as axondir:

            byts = os.urandom(128)
            bytsmd5 = hashlib.md5(byts).hexdigest()

            axon = s_axon.Axon(axondir, syncsize=64)

            iden = axon.alloc(128)
            for chnk in chunks(byts, 10):
                blob = axon.chunk(iden, chnk)

            self.nn(blob)

            self.true(axon.has('md5', bytsmd5))

            axon.fini()

    def test_axon_host(self):

        self.thisHostMustNot(platform='windows')

        with self.getTestDir() as datadir:

            with open(os.path.join(datadir, 'foo'), 'w') as fd:
                fd.write('useless file to skip')

            host = s_axon.AxonHost(datadir)
            usage = host.usage()

            props = {
                'syncmax': s_axon.megabyte * 10,
                'bytemax': s_axon.megabyte * 10,
            }

            axfo = host.add(**props)

            self.nn(usage.get('total'))

            axon = host.axons.get(axfo[0])

            iden = axon.alloc(100)
            blob = axon.chunk(iden, b'V' * 100)

            self.nn(blob)

            self.true(axon.has('md5', blob[1].get('hash:md5')))
            self.true(axon.has('sha1', blob[1].get('hash:sha1')))
            self.true(axon.has('sha256', blob[1].get('hash:sha256')))

            host.fini()

            host = s_axon.AxonHost(datadir)
            axon = host.axons.get(axfo[0])

            self.true(axon.has('md5', blob[1].get('hash:md5')))
            self.true(axon.has('sha1', blob[1].get('hash:sha1')))
            self.true(axon.has('sha256', blob[1].get('hash:sha256')))

            props = {
                'syncmax': s_axon.megabyte * 10,
            }
            self.raises(NotEnoughFree, host.add, **props)

            host.fini()

    def test_axon_host_clone(self):

        self.thisHostMustNot(platform='windows')

        busurl = 'local://%s/axons' % guid()

        dmon = s_daemon.Daemon()
        dmon.listen(busurl)

        dmon.share('axons', s_service.SvcBus(), fini=True)

        with self.getTestDir() as datadir:

            dir0 = gendir(datadir, 'host0')
            dir1 = gendir(datadir, 'host1')
            dir2 = gendir(datadir, 'host2')

            opts = {
                'axonbus': busurl,
            }

            host0 = s_axon.AxonHost(dir0, hostname='host0', **opts)
            host1 = s_axon.AxonHost(dir1, hostname='host1', **opts)
            host2 = s_axon.AxonHost(dir2, hostname='host2', **opts)

            props = {
                'syncmax': s_axon.megabyte,
                'bytemax': s_axon.megabyte,
            }

            axfo0 = host0.add(**props)

            axon0 = s_telepath.openlink(axfo0[1].get('link'))
            self.true(axon0._waitClonesReady(timeout=8))

            iden = axon0.alloc(100)
            blob = axon0.chunk(iden, b'V' * 100)

            self.nn(blob)

            self.true(axon0.has('md5', blob[1].get('hash:md5')))
            self.true(axon0.has('sha1', blob[1].get('hash:sha1')))
            self.true(axon0.has('sha256', blob[1].get('hash:sha256')))

            axon0.fini()

            host0.fini()
            host1.fini()
            host2.fini()

        dmon.fini()

    def test_axon_clustered(self):

        self.thisHostMustNot(platform='windows')

        busurl = 'local://%s/axons' % guid()

        dmon = s_daemon.Daemon()
        dmon.listen(busurl)

        dmon.share('axons', s_service.SvcBus(), fini=True)

        with self.getTestDir() as datadir:

            dir0 = gendir(datadir, 'host0')
            dir1 = gendir(datadir, 'host1')
            dir2 = gendir(datadir, 'host2')

            opts = {
                'axonbus': busurl,
            }

            host0 = s_axon.AxonHost(dir0, hostname='host0', **opts)
            host1 = s_axon.AxonHost(dir1, hostname='host1', **opts)
            host2 = s_axon.AxonHost(dir2, hostname='host2', **opts)

            props = {
                'clones': 2,
                'syncmax': s_axon.megabyte,
                'bytemax': s_axon.megabyte,
            }

            axfo0 = host0.add(**props)

            axon0 = s_telepath.openlink(axfo0[1].get('link'))

            # wait for clones to come online
            self.true(axon0._waitClonesReady(timeout=8))

            #self.nn( usage.get('total') )
            #axon = host.axons.get(iden)

            iden = axon0.alloc(100)
            blob = axon0.chunk(iden, b'V' * 100)

            self.nn(blob)

            self.true(axon0.has('md5', blob[1].get('hash:md5')))
            self.true(axon0.has('sha1', blob[1].get('hash:sha1')))
            self.true(axon0.has('sha256', blob[1].get('hash:sha256')))

            axon0.fini()

            host0.fini()
            host1.fini()
            host2.fini()

        dmon.fini()

    def test_axon_cluster(self):

        self.thisHostMustNot(platform='windows')

        busurl = 'local://%s/axons' % guid()

        dmon = s_daemon.Daemon()
        dmon.listen(busurl)

        dmon.share('axons', s_service.SvcBus(), fini=True)
        svcprox = s_service.openurl(busurl)

        axcluster = s_axon.AxonCluster(svcprox)

        with self.getTestDir() as datadir:

            dir0 = gendir(datadir, 'host0')
            dir1 = gendir(datadir, 'host1')
            dir2 = gendir(datadir, 'host2')

            opts = {
                'axonbus': busurl,
            }

            host0 = s_axon.AxonHost(dir0, hostname='host0', **opts)
            host1 = s_axon.AxonHost(dir1, hostname='host1', **opts)
            host2 = s_axon.AxonHost(dir2, hostname='host2', **opts)

            props = {
                'clones': 1,
                'syncmax': s_axon.megabyte,
                'bytemax': s_axon.megabyte,
            }

            axfo0 = host0.add(**props)

            axcluster._waitWrAxons(1, 4)

            self.false(axcluster.has('md5', craphash))
            self.false(axcluster.has('md5', asdfhash))

            buf = b'asdfasdf'
            iden = axcluster.alloc(len(buf))
            self.nn(axcluster.chunk(iden, buf))

            self.false(axcluster.has('md5', craphash))
            self.true(axcluster.has('md5', asdfhash))

            blobs = axcluster.find('md5', craphash)
            self.eq(len(blobs), 0)

            blobs = axcluster.find('md5', asdfhash)
            self.eq(len(blobs), 1)

            blob = blobs[0]
            byts = b''.join(axcluster.iterblob(blob))
            self.eq(byts, buf)

            blob[1].pop('.axon')
            byts = b''.join(axcluster.iterblob(blob))
            self.eq(byts, buf)

            self.nn(axcluster.wants('md5', craphash, len(buf)))
            self.none(axcluster.wants('md5', asdfhash, len(buf)))

            host0.fini()
            host1.fini()
            host2.fini()

        dmon.fini()

    def test_axon_autorun(self):

        self.thisHostMustNot(platform='windows')

        with self.getTestDir() as dirname:

            opts = {
                'autorun': 2,
                'syncmax': s_axon.megabyte,
                'bytemax': s_axon.megabyte,
            }
            host = s_axon.AxonHost(dirname, **opts)
            self.eq(len(host.axons), 2)
            host.fini()

    def test_axon_eatbytes(self):
        self.thisHostMustNot(platform='windows')

        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                blob0 = axon.eatbytes(b'visi')
                with io.BytesIO(b'vertex') as fd:
                    blob1 = axon.eatfd(fd)

                port = axon.getAxonInfo()[1].get('link')[1].get('port')

                with s_telepath.openurl('tcp://127.0.0.1/axon', port=port) as prox:
                    blob2 = prox.eatbytes(b'hurr')
                    with io.BytesIO(b'durr') as fd:
                        blob3 = prox.eatfd(fd)

        self.eq(blob0[1].get('axon:blob'), '442f602ecf8230b2a59a44b4f845be27')
        self.eq(blob1[1].get('axon:blob'), 'd4552906c1f6966b96d27e6fc79441b5')
        self.eq(blob2[1].get('axon:blob'), '0d60960570ef6da0a15f68c24b420334')
        self.eq(blob3[1].get('axon:blob'), '97c11d1057f75c9c0b79090131709f62')

    #def test_axon_proxy(self):

    # Axon File System Tests
    def test_axon_fs_create(self):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_create('/foofile', 33204)
                actual = axon.fs_getattr('/foofile')

                self.eq(actual['st_nlink'], 1)
                self.eq(actual['st_mode'], 33204)
                self.gt(actual['st_atime'], 1000000000)
                self.gt(actual['st_ctime'], 1000000000)
                self.gt(actual['st_mtime'], 1000000000)

                self.raises(NoSuchDir, axon.fs_create, '/foodir/foo2', 33204)
                #self.raises(NoSuchDir, axon.fs_create, 'foo2nope', 33204)  # No parent

    def test_axon_fs_getattr(self):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                actual = axon.fs_getattr('/')

                self.eq(actual['st_nlink'], 2)
                self.eq(actual['st_mode'], 16893)
                self.gt(actual['st_atime'], 1000000000)
                self.gt(actual['st_ctime'], 1000000000)
                self.gt(actual['st_mtime'], 1000000000)

    def test_axon_fs_getxattr(self):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                actual = axon.fs_getattr('/')
                self.eq(actual['st_nlink'], 2)
                self.eq(actual['st_mode'], 16893)
                self.gt(actual['st_atime'], 1000000000)
                self.gt(actual['st_ctime'], 1000000000)
                self.gt(actual['st_mtime'], 1000000000)

                nlink = axon.fs_getxattr('/', 'st_nlink')
                self.eq(nlink, actual['st_nlink'])
                self.raises(NoSuchData, axon.fs_getxattr, '/', 'zzz')
                self.raises(NoSuchData, axon.fs_getxattr, '/haha', 'st_nlink')

    def test_axon_fs_mkdir(self):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                actual = axon.fs_mkdir('/foo', 0x1FD)
                self.eq(actual, None)

                actual = axon.fs_getattr('/foo')
                self.eq(actual['st_nlink'], 2)
                self.eq(actual['st_mode'], 16893)
                self.gt(actual['st_atime'], 1000000000)
                self.gt(actual['st_ctime'], 1000000000)
                self.gt(actual['st_mtime'], 1000000000)

                self.raises(NoSuchDir, axon.fs_mkdir, '/foodir/foo2', 16893)
                #self.raises(NoSuchDir, axon.fs_mkdir, 'noparent', 16893)

                self.raises(FileExists, axon.fs_mkdir, '/foo', 0x1FD)

    def test_axon_fs_read(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                data = b'haha\n'

                fd = tempfile.SpooledTemporaryFile()
                fd.write(data)

                axon.fs_create('/haha', 33204)

                _, blobprops = axon.eatfd(fd)
                blobsize = blobprops.get('axon:blob:size')
                blob = blobprops.get('axon:blob')

                axon._fs_update_blob('/haha', blobsize, blob)
                fd.close()

                sz = axon.fs_getattr('/haha').get('st_size')
                actual = axon.fs_read('/haha', sz, 0)
                self.eq(actual, data)

                self.raises(NoSuchEntity, axon.fs_read, '/hoho2', 100, 0)

                axon.fs_create('/haha2', 33204)
                actual = axon.fs_read('/haha2', 1000000000, 0)
                self.eq(actual, b'')

    def test_axon_fs_readdir(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_create('/foofile', 33204)
                self.eq(sorted(axon.fs_readdir('/')), sorted(['.', '..', 'foofile']))

                self.raises(NoSuchEntity, axon.fs_readdir, '/haha')
                self.raises(NotSupported, axon.fs_readdir, '/foofile')

    def test_axon_fs_rmdir(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_mkdir('/foo', 0x1FD)
                axon.fs_create('/foo/haha', 33204)

                self.raises(NotEmpty, axon.fs_rmdir, '/foo')

                axon.fs_unlink('/foo/haha')
                self.nn(axon.fs_getattr('/foo'))
                axon.fs_rmdir('/foo')
                self.raises(NoSuchEntity, axon.fs_getattr, '/foo')
                self.raises(NoSuchEntity, axon.fs_rmdir, '/foo')

    def test_axon_fs_rename(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_mkdir('/dir', 0x1FD)
                axon.fs_create('/dir/a', 33204)
                axon.fs_rename('/dir/a', '/dir/b')

                actual = axon.fs_readdir('/dir')
                self.eq(sorted(actual), ['.', '..', 'b'])

                # source doesnt exist
                self.raises(NoSuchEntity, axon.fs_rename, 'fake', 'other')

                # source doesnt have parent
                tufo = axon.core.getTufoByProp('axon:path', '/dir')
                axon.core.delTufo(tufo)
                self.raises(NoSuchDir, axon.fs_rename, '/dir/b', '/dir/c')

                # dst doesnt have parent
                axon.fs_mkdir('/dira', 0x1FD)
                axon.fs_mkdir('/dirb', 0x1FD)
                axon.fs_create('/dira/a', 33204)
                tufo = axon.core.getTufoByProp('axon:path', '/dirb')
                axon.core.delTufo(tufo)
                self.raises(NoSuchDir, axon.fs_rename, '/dira/a', '/dirb/b')

                # dst not empty
                axon.fs_mkdir('/flda', 0x1FD)
                axon.fs_mkdir('/fldb', 0x1FD)
                axon.fs_create('/flda/a', 33204)
                axon.fs_create('/fldb/a', 33204)
                self.raises(NotEmpty, axon.fs_rename, '/flda', '/fldb')

                # overwrite a regular file with a directory
                axon.fs_mkdir('/adir', 0x1FD)
                axon.fs_create('/reg', 33204)
                axon.fs_rename('/adir', '/reg')
                self.none(axon.fs_getattr('/reg').get('st_size'))

                # update all the kids
                axon.fs_mkdir('/cool', 0x1FD)
                axon.fs_create('/cool/a', 33204)
                axon.fs_create('/cool/b', 33204)
                axon.fs_create('/cool/c', 33204)
                axon.fs_rename('/cool', '/cooler')

                self.raises(NoSuchEntity, axon.fs_getattr, '/cool')
                self.raises(NoSuchEntity, axon.fs_getattr, '/cool/a')
                self.raises(NoSuchEntity, axon.fs_getattr, '/cool/b')
                self.raises(NoSuchEntity, axon.fs_getattr, '/cool/c')

                self.nn(axon.fs_getattr('/cooler'))
                self.nn(axon.fs_getattr('/cooler/a'))
                self.nn(axon.fs_getattr('/cooler/b'))
                self.nn(axon.fs_getattr('/cooler/c'))

                # nested dirs
                axon.fs_mkdir('/nest1', 0x1FD)
                axon.fs_mkdir('/nest1/nest2', 0x1FD)
                axon.fs_mkdir('/nest1/nest2/nest3', 0x1FD)
                axon.fs_create('/nest1/nest2/nest3/reg', 33204)

                axon.fs_rename('/nest1', '/nest')

                self.nn(axon.fs_getattr('/nest'))
                self.nn(axon.fs_getattr('/nest/nest2'))
                self.nn(axon.fs_getattr('/nest/nest2/nest3'))
                self.nn(axon.fs_getattr('/nest/nest2/nest3/reg'))

    def test_axon_fs_truncate(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_create('/foofile', 33204)
                tufo = axon.core.getTufoByProp('axon:path', '/foofile')
                axon.core.setTufoProps(tufo, st_size=100, blob=32 * 'a')
                self.eq(tufo[1].get('axon:path:st_size'), 100)
                self.eq(tufo[1].get('axon:path:blob'), 32 * 'a')

                axon.fs_truncate('/foofile')
                tufo = axon.core.getTufoByProp('axon:path', '/foofile')
                self.eq(tufo[1].get('axon:path:st_size'), 0)
                self.eq(tufo[1].get('axon:path:blob'), None)

                axon.fs_truncate('/notthere')

    def test_axon_fs_unlink(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_create('/foofile', 33204)

                actual = axon.fs_getattr('/foofile')
                self.eq(actual['st_nlink'], 1)
                self.eq(actual['st_mode'], 33204)
                self.gt(actual['st_atime'], 1000000000)
                self.gt(actual['st_ctime'], 1000000000)
                self.gt(actual['st_mtime'], 1000000000)

                axon.fs_unlink('/foofile')
                self.raises(NoSuchEntity, axon.fs_getattr, '/foofile')
                self.raises(NoSuchFile, axon.fs_unlink, '/foofile')

    def test_axon_fs_utimens(self, *args, **kwargs):
        with self.getTestDir() as dirname:
            with s_axon.Axon(dirname) as axon:

                axon.fs_create('/foofile', 33204)
                actual = axon.fs_getattr('/foofile')
                ctime = actual['st_ctime']

                self.gt(actual['st_atime'], 1000000000)
                self.gt(actual['st_ctime'], 1000000000)
                self.gt(actual['st_mtime'], 1000000000)

                axon.fs_utimens('/foofile', (0, 0))
                actual = axon.fs_getattr('/foofile')

                self.eq(actual['st_mtime'], 0)
                self.eq(actual['st_ctime'], ctime)
                self.eq(actual['st_atime'], 0)

                axon.fs_utimens('/foofile')
                self.eq(actual['st_mtime'], 0)
                self.eq(actual['st_ctime'], ctime)
                self.eq(actual['st_atime'], 0)

    # test_axon_fs_write  - do not implement this function

    def test_axon__fs_isdir(self, *args, **kwargs):
        self.eq(s_axon.Axon._fs_isdir(None), False)
        self.eq(s_axon.Axon._fs_isdir(0), False)
        self.eq(s_axon.Axon._fs_isdir(33204), False)
        self.eq(s_axon.Axon._fs_isdir(16893), True)

    def test_axon__fs_isfile(self, *args, **kwargs):
        self.eq(s_axon.Axon._fs_isfile(None), False)
        self.eq(s_axon.Axon._fs_isfile(0), False)
        self.eq(s_axon.Axon._fs_isfile(33204), True)
        self.eq(s_axon.Axon._fs_isfile(16893), False)

    def test_axon_get_renameprops(self, *args, **kwargs):
        tufo = ('99ac9490ad2e1d4669de1c005a4ec666',
            {'tufo:form': 'axon:path', 'axon:path:st_ctime': 1491191818, 'axon:path:st_mode': 16893,
            'axon:path:st_atime': 1491191818, 'axon:path': '/dir', 'axon:path:base': 'dir', 'axon:path:dir': '/',
            'axon:path:st_nlink': 3, 'axon:path:st_mtime': 1491191818, 'axon:path:blob': 32 * 'a'})
        actual = s_axon.Axon._get_renameprops(tufo)

        self.eq(actual['st_nlink'], 3)
        self.eq(actual['st_mode'], 16893)
        self.eq(actual['blob'], 32 * 'a')
        self.gt(actual['st_atime'], 1000000000)
        self.gt(actual['st_ctime'], 1000000000)
        self.gt(actual['st_mtime'], 1000000000)

    def test_axon_telepath(self):
        with self.getTestDir() as dirname:

            with s_daemon.Daemon() as dmon:

                link = dmon.listen('tcp://127.0.0.1:0/')
                port = link[1].get('port')

                with s_axon.Axon(dirname) as axon:
                    dmon.share('axon', axon)

                    prox = s_telepath.openurl('tcp://127.0.0.1/axon', port=port)

                    with io.BytesIO(b'vertex') as fd:
                        blob = prox.eatfd(fd)
                        self.eq(blob[1]['axon:blob:sha256'], 'e1b683e26a3aad218df6aa63afe9cf57fdb5dfaf5eb20cddac14305d67f48a02')