#Rahman ve Rahim Olan Allah'ın Adıyla, Hamd Alemlerin Rabb'i Allah'a Mahsustur!!!

import math
import numpy as np
#FOR TRAJECTORY IMPORT
from numpy import linalg as LA
from scipy import optimize
from copy import deepcopy

from State import DesiredTrajectoryState
from utils import Hessian, polyder, normalize, position_to_numpy
import time
import matplotlib.pyplot as plt

from mpl_toolkits.mplot3d import Axes3D
from scipy.optimize import linear_sum_assignment as hungarian

from Parameters import *
from enums import *

class TrajGenerator:
    def __init__(self , order = 10):
        #Order can not change, for this implementation
        self.waypoints = None
        self.order = order
        self.yaw = 0
        self.heading = np.zeros(2)
        self.trajectory_generated = False

    def generate_trajectory_for_path(self,initial_position,waypoints,max_vel,gamma,path_manipulate_coef):
        waypoints.insert(0,initial_position)
        waypoints = position_to_numpy(waypoints)
        self.manipulate_coef = path_manipulate_coef
        self.waypoints = waypoints/self.manipulate_coef #waypoints numpy array
        len,dim = waypoints.shape
        self.dim = dim
        self.len = len
        self.max_vel = max_vel
        self.gamma = gamma
        self.TS = np.zeros(self.len)  #Segment time'lar initialize edildi
        self.optimize() # UPDATE THE COEFFICIENT OF TRAJECTORY, Her cagirilista yeni trajectory olusturur
    #RAHMAN VE RAHIM OLAN ALLAH'ın ADIYLA!

    def generate_trajectory_for_rotate(self,initial_position,waypoints,step_size,max_vel,gamma):
        np.insert(waypoints,0,np.array([initial_position.x,initial_position.y,initial_position.z]))
        #waypoints = position_to_numpy(waypoints)
        self.waypoints = waypoints #waypoints numpy array
        len,dim = waypoints.shape
        self.dim = dim
        self.len = len
        self.max_vel = max_vel
        self.gamma = gamma
        self.TS = np.zeros(self.len)  #Segment time'lar initialize edildi
        self.optimize_for_fixed_time(step_size) # UPDATE THE COEFFICIENT OF TRAJECTORY, Her cagirilista yeni trajectory olusturur

    def generate_trajectory(self,initial_position,waypoints,max_vel,gamma):
        waypoints.insert(0,initial_position)
        waypoints = position_to_numpy(waypoints)
        self.waypoints = waypoints #waypoints numpy array
        len,dim = waypoints.shape
        self.dim = dim
        self.len = len
        self.max_vel = max_vel
        self.gamma = gamma
        self.TS = np.zeros(self.len)  #Segment time'lar initialize edildi
        self.optimize() # UPDATE THE COEFFICIENT OF TRAJECTORY, Her cagirilista yeni trajectory olusturur

    def reset(self):
        self.dim = None
        self.len = None
        self.TS = None
        self.waypoints = None
        self.coeffs = None
        self.trajectory_generated = False

    def get_cost(self,T):
        coeffs,cost = self.MinimizeSnap(T)
        cost = cost + self.gamma*np.sum(T)
        return cost

    def optimize_for_fixed_time(self,step_size):
        T = np.ones(self.len-1)*(1/step_size)

        self.TS[1:] = np.cumsum(T)
        self.coeffs, self.cost = self.MinimizeSnap(T)

    def optimize(self):
        diff = self.waypoints[0:-1] - self.waypoints[1:]
        Tmin = LA.norm(diff,axis = -1)/self.max_vel #Olabilecek en dusuk hız, bizim verdigimiz max_vel ile belirlenir
        t1 = time.time()
        T = optimize.minimize(self.get_cost,Tmin, method="COBYLA",constraints= ({'type': 'ineq', 'fun': lambda T: T-Tmin}))['x']
        #print("NE KADAR SURDU",time.time()-t1)

        #T'yi optimize etmek icin GAMMA'lı cost isin icine katılarak cost hesaplandı, evet okey

        self.TS[1:] = np.cumsum(T)
        self.coeffs, self.cost = self.MinimizeSnap(T)

    def MinimizeSnap(self,T):
        unkns = 4*(self.len - 2)

        Q = Hessian(T)
        A,B = self.get_constraints(T)

        invA = LA.inv(A)

        if unkns != 0:
            R = invA.T@Q@invA

            Rfp = R[:-unkns,-unkns:]
            Rpp = R[-unkns:,-unkns:]

            B[-unkns:,] = -LA.inv(Rpp)@Rfp.T@B[:-unkns,]

        P = invA@B
        cost = np.trace(P.T@Q@P)

        return P, cost

    def get_constraints(self,T):
        n = self.len - 1
        o = self.order

        A = np.zeros((self.order*n, self.order*n))
        B = np.zeros((self.order*n, self.dim))

        B[:n,:] = self.waypoints[ :-1, : ]
        B[n:2*n,:] = self.waypoints[1: , : ]

        #waypoints contraints
        for i in range(n):
            A[i, o*i : o*(i+1)] = polyder(0, order = self.order)
            A[i + n, o*i : o*(i+1)] = polyder(T[i], order = self.order)

        #continuity contraints
        for i in range(n-1):
            A[2*n + 4*i: 2*n + 4*(i+1), o*i : o*(i+1)] = -polyder(T[i],'all',order = self.order)
            A[2*n + 4*i: 2*n + 4*(i+1), o*(i+1) : o*(i+2)] = polyder(0,'all',order = self.order)

        #start and end at rest
        A[6*n - 4 : 6*n, : o] = polyder(0,'all',order = self.order)
        A[6*n : 6*n + 4, -o : ] = polyder(T[-1],'all',order = self.order)

        #free variables
        for i in range(1,n):
            A[6*n + 4*i : 6*n + 4*(i+1), o*i : o*(i+1)] = polyder(0,'all',order=self.order)

        return A,B

    def get_yaw(self,vel):
        prev_heading = self.heading
        curr_heading = vel/LA.norm(vel)
        cosine = max(-1,min(np.dot(prev_heading, curr_heading),1))
        dyaw = np.arccos(cosine)
        norm_v = np.cross(prev_heading,curr_heading)
        self.yaw += np.sign(norm_v)*dyaw

        if self.yaw > np.pi: self.yaw -= 2*np.pi
        if self.yaw < -np.pi: self.yaw += 2*np.pi

        self.heading = curr_heading
        return self.yaw,dyaw

    def get_des_state(self,t):

        if t > self.TS[-1]: t = self.TS[-1] - 0.001

        i = np.where(t >= self.TS)[0][-1] #Segment selection, assume we have 1 segment!

        t = t - self.TS[i] #For each segment at start t = 0, bu satır bunu saglıyor.
        coeff = (self.coeffs.T)[:,self.order*i:self.order*(i+1)] #Give coefficient
        pos  = coeff@polyder(t,order = self.order)   #Gives pos value at segment time t! not t+dt!
        vel  = coeff@polyder(t,1,order = self.order) #Gives vel value at segment time t! not t+dt!
        accl = coeff@polyder(t,2,order = self.order) #Gives accl value at segment time t! not t+dt!
        jerk = coeff@polyder(t,3,order = self.order) #Gives Jerk value at segment time t! not t+dt! ---> Result: Trajectory'i yanlıs isliyoruz!

        thrust = accl + np.array([0, 0, 9.81]) #Thrust calculation from many paper
        z_body = normalize(thrust) #Drone Body z_body unit vector!
        jerk_orth_zbody = jerk - (np.dot(jerk, z_body) * z_body) #This calculation from paper of Mellinger and Kumar
        h_w = jerk_orth_zbody / np.linalg.norm(thrust) #This calculation from paper of Mellinger and Kumar, pls refer paper and ask question if you want!

        #set yaw in the direction of velocity
        yaw,dyaw = self.get_yaw(vel[:2]) #Get yaw value at segment time t! We can add the yaw controller, yaw---> yaw_rate!

        x_world = np.array([np.cos(yaw), np.sin(yaw), 0])
        y_body = normalize(np.cross(z_body, x_world))
        x_body = np.cross(y_body, z_body)
        omega = np.array([-np.dot(h_w, y_body), np.dot(h_w, x_body), z_body[2] * dyaw]) #In the WORLD FRAME, not body frame. This equation comes from, Mellinger Kumar Paper. Model....
        #Bu omega'nın z'sini body'e cevirebiliriz! Bunun body-z'si + yaw--->yawrate PID olmalı! duzgun kontrol olmazsa! Ama calısması gerek. Position icin de aynı

        return DesiredTrajectoryState(pos, vel, accl, yaw, omega)

class TrajectoryExecuter:
    def __init__(self,des_state,Tmax,
                 pos = None,
                 control_frequency = 62):

        self.initial_step_time = 0 #Used for trajectory time calculation elhamdulillah
        self.t = 0
        self.Tmax = Tmax
        self.dt = 1/control_frequency #YANLIS her bir .step() fonksiyonu cagırıldıgında 1/56sn sonraki traj state donuyor. Buraya da bir kontrol mekanizması konulabilir! nerede olduğuna gore!
                                      # Ve bu boyle her step bir sonraki hedef degeri verir ? Belki de initial veriyo.
        self.des_state = des_state
        if pos is None: pos = self.des_state(0).desired_position #initial Position #NOT USED
        end_state = self.des_state(self.Tmax-0.000001) ### ????? Tmax'ta çalışmıyor
        self.end_position = end_state.desired_position
        self.end_yaw = end_state.desired_yaw
        self.traj_completed = False

    #Rahman ve Rahim Olan Allah'ın Adıyla!
    def step(self):
        if self.t == 0:
            self.initial_step_time = time.time() #Inıtıalize time

        self.t = time.time()- self.initial_step_time
        print("Passed Time T: ",self.t)
        desired_state = self.des_state(self.t)
        #desired_state.desired_velocity[2] += 0.125 #BUNU MELLINGER KONTROLCU KENDISI EKLIYOR GEREK YOK
        if self.t >= self.Tmax:
            self.traj_completed = True
            return DesiredTrajectoryState(self.end_position,np.array([0,0,0]),np.zeros(3),self.end_yaw,np.zeros(3)),self.traj_completed ### En son gitmesi gereken pozisyonda Hover
        return desired_state,self.traj_completed


import logging
import discord
from discord import embeds
import os
import toml
config = toml.load('config.toml')
from src.modules.commands import BaseCommand
from src.modules.commands.utils import TraceCommand

class InfoCommand(BaseCommand):

    @TraceCommand.traceback_print
    async def apply(self):
        import src.texts.help_texts as texts
        from src.texts.const import const
        logging.info("Enviando mensaje de información")
        bot_info_channel_id = config['INFO_BOT_CHANNEL_ID']
        channel = self.ctx.guild.get_channel(bot_info_channel_id)

        embed = discord.Embed(
            colour=discord.Colour.blue(),
            title=f"{const.EVENT_NAME} {const.EVENT_YEAR}" ,
            description=texts.START_DESCRIPTION_MESSAGE
        )
        embed.add_field(name=f'{const.COMMAND_PREFIX}help',value=texts.HELP_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}login',value=texts.LOGIN_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}create <team_name>',value=texts.CREATE_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}invite <discord_user>',value=texts.INVITE_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}join <team_name>',value=texts.JOIN_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}leave',value=texts.LEAVE_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}ask <question>',value=texts.ASK_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}joke',value=texts.JOKE_COMMAND, inline=False)
        embed.add_field(name=f'{const.COMMAND_PREFIX}rpsls',value=texts.RPSLS_COMMAND, inline=False)

        await channel.send(embed=embed)


from itertools import combinations
from collections import deque, defaultdict

def get_data(filename: str):
    with open(filename, 'r') as f:
        return f.read()

lines = get_data('input.txt')
lines = lines.strip().split('\n')
lines = [x.split(',') for x in lines]
lines = [tuple(int(x) for x in y) for y in lines]

def adjacent(t1, t2):
    if abs(t1[0]-t2[0]) > 1 or abs(t1[1]-t2[1]) > 1 or abs(t1[2]-t2[2]) > 1:
        return False
    c = abs(t1[0]-t2[0]) + abs(t1[1]-t2[1]) + abs(t1[2]-t2[2])
    return c == 1

total = 6 * len(lines)

# not terribly helpful for part two :(
for c in combinations(lines, 2):
    if adjacent(c[0], c[1]):
        total -= 2

print("Part one:", total)

mx = max(lines, key=lambda x: x[0])[0]+1
my = max(lines, key=lambda x: x[1])[1]+1
mz = max(lines, key=lambda x: x[2])[2]+1
upper = (mx, my, mz)
mnx = min(lines, key=lambda x: x[0])[0]-1
mny = min(lines, key=lambda x: x[1])[1]-1
mnz = min(lines, key=lambda x: x[2])[2]-1
lower = (mnx, mny, mnz)
lava = set(lines)

ds = ((1, 0, 0), (-1, 0, 0), (0, 1, 0), (0, -1, 0), (0, 0, 1), (0, 0, -1))

ext = defaultdict(int)
seen = set()

Q = deque()
Q.append((mx, my, mz))

# could've been recursive? wasn't sure about python's depth limit
while Q:
    pt = Q.popleft()
    if pt in seen:
        continue
    seen.add(pt)
    for d in ds:
        nxt = (pt[0]+d[0], pt[1]+d[1], pt[2]+d[2])
        # one direction is lava == one exterior face
        if (nxt) in lava:
            ext[pt] += 1
        else:
            if all([x[0] <= x[1] for x in zip(nxt, upper)]) and all([x[0] >= x[1] for x in zip(nxt, lower)]):
                Q.append(nxt)

print("Part two:", sum(ext.values()))


import numpy as np
import pandas as pd
from sklearn.neighbors import NearestNeighbors
from sklearn.preprocessing import StandardScaler
from core.recommender_systems.RecommenderSystem import RecommenderSystem

class CoolStartRS(RecommenderSystem):
    """
        Implement the RecommenderSystem class, this class just need
        information on the songs and, through a "songs space" is able to
        give recommendation, considering the last song listened
        by the user.

        The recommendation is performed by means of K-NN with the cosine
        similarity as metric. N closest songs-points, to the last listened
        song-point, are recommended.

        Parameters
        ----------

    """

    def __init__(self):
        super().__init__(table_name='SONGS')
        self.rec_songs = {}
        self.columns_of_number_type = self.sql_handler.select_columns(table_name=self.table_name,
                                                                      data_types=['integer', 'double'])
        x = self.sql_handler.search(table_name=self.table_name, conditions=(self.columns_of_number_type, []))
        self.x = pd.DataFrame(data=x, columns=self.columns_of_number_type)

    def k_neighbors(self, current_user_id: str, current_song_id: str, n_songs: int, index: list, get_listened: bool):
        """
            Select n_songs from index row, under the condition if the already listened songs
            have to be considered or not.

            Parameters
            ----------
            current_user_id : ID of the logged user

            current_song_id : ID of the last listened song

            n_songs : number of songs to be recommended

            index : list of row index of the songs, ordered according the distances from current_song_id

            get_listened : if already listened songs have to be considered or not

            Returns
            -------
            rec_songs : list of recommended songs by ID
        """
        if get_listened:
            rec_songs = [self.get_vector(current_song_id)]
        else:
            rec_songs = {}
        counter, list_position = (0, 0)

        while list_position < len(index[1]) and counter < n_songs:
            idx = index[1][list_position]
            rec_song = self.sql_handler.search(self.table_name, (('song_id',), [('ROWID', '=', int(idx))]))
            if len(rec_song) != 0:
                rec_song_listened = self.sql_handler.search('HISTORY', (('song_id',),
                                                                        [('user_id', '=', current_user_id),
                                                                         'AND',
                                                                         ('song_id', '=', rec_song)]))

                if len(rec_song_listened) != 0:
                    rec_song_listened = rec_song == rec_song_listened

                if get_listened:
                    if rec_song_listened:
                        rec_songs.append(self.get_vector(rec_song))
                    counter += 1
                else:
                    if not rec_song_listened:
                        rec_songs[rec_song] = 1 - index[0][list_position]  # cosine similarity = 1 - cosine distance
                        counter += 1

            list_position += 1
        return rec_songs

    def get_vector(self, song_id: str):
        """
            Given a songID, this method provide the array of number features.

            Parameters
            ----------
            song_id : ID of the given song

            Returns
            -------
            array of features song
        """
        columns = self.sql_handler.columns_of(table_name=self.table_name)
        idx_columns_of_number_type = []
        for column in self.columns_of_number_type:
            idx_columns_of_number_type.append(columns.index(column))

        song_data = self.sql_handler.search(table_name=self.table_name,
                                            conditions=(('*',), [('song_id', '=', song_id)]))
        song_vector = []
        for idx in idx_columns_of_number_type:
            song_vector.append(song_data[idx])

        return np.array(song_vector)

    def recommend_songs(self, current_user_id: str, current_song_id: str, n_songs: int = 10):
        """
            Create a song space by their features, and apply KNN on the current song.
            The Algorithm follow this procedure:
            1. Select n_songs items closest to current_song_id
            2. Select from (1) already listened songs, current_song_id is part of this set
                2.1. If no how was already listened, then suggest them to the user
            3. Compute the 'artificial middle song' among songs from (2)
            4. Select the n_songs never listened items closest to the artificial song point
            5. Recommend items from (4)

            Parameters
            ----------
            current_user_id : ID of the logged user

            current_song_id : ID of the last listened song

            n_songs : number of songs to be recommended

            Returns
            -------
            rec_songs : list of recommended songs by ID
        """
        song_vector = self.get_vector(current_song_id)
        scaler = StandardScaler()
        scaler.fit(self.x)
        scaled_data = scaler.transform(self.x)
        scaled_song_vector = scaler.transform(song_vector.reshape(1, -1))
        neigh = NearestNeighbors(n_neighbors=10, metric='cosine', n_jobs=-1)
        neigh.fit(scaled_data)

        index = neigh.kneighbors(X=scaled_song_vector, n_neighbors=10, return_distance=True)
        index = [index[0][0].tolist(), index[1][0].tolist()]  # just for a better readability

        already_listened_songs_vectors = self.k_neighbors(current_user_id, current_song_id, n_songs, index, True)

        if len(already_listened_songs_vectors) > 1:
            listened_songs_matrix = np.array(already_listened_songs_vectors)
            listened_songs_mean_vector = np.mean(listened_songs_matrix, axis=0)

            array_comparison_bool_matrix = np.equal(self.x, listened_songs_mean_vector)  # matrix of bool

            # check if listened_songs_mean_vector is present in self.x as a vector row
            if True in np.apply_along_axis(lambda x: True if np.alltrue(x) else False, 1, array_comparison_bool_matrix):
                listened_mean_row = [(self.columns_of_number_type[idx], listened_songs_mean_vector[idx])
                                     for idx in range(0, len(listened_songs_mean_vector))]
                column_conditions = []
                for column_name, column_value in listened_mean_row:
                    condition = (column_name, '=', column_value)
                    column_conditions.append(condition)
                    column_conditions.append('AND')

                conditions = (('song_id',), column_conditions[:-1])
                mean_song_id = self.sql_handler.search(self.table_name, conditions)

                if mean_song_id and mean_song_id not in self.sql_handler.search('HISTORY',
                                                                                (('song_id',),
                                                                                [('user_id', '=', current_user_id),
                                                                                'AND',
                                                                                ('song_id', '=', mean_song_id)])):
                    self.rec_songs[mean_song_id] = 1.0  # mean_song_id first recommendation
                    n_songs = n_songs - 1  # update number of items to find

            scaled_listened_songs_mean_vector = scaler.transform(listened_songs_mean_vector.reshape(1, -1))

            index = neigh.kneighbors(X=scaled_listened_songs_mean_vector, n_neighbors=scaled_data.shape[0],
                                     return_distance=True)
            index = [index[0][0].tolist(), index[1][0].tolist()]  # just for a better readability
            self.rec_songs.update(self.k_neighbors(current_user_id, current_song_id, n_songs, index, False))
        else:
            self.rec_songs = self.k_neighbors(current_user_id, current_song_id, n_songs, index, False)

        return self.rec_songs

    def __str__(self):
        return self.print('CoolStartRS', self.rec_songs) if self.rec_songs else ''


import importlib
from sys import exit

from BasicPyLib import simpleLogger
# noinspection PyUnresolvedReferences
from Config import base_settings
from BasicPyLib.BasicTools import CONSTANTS, isAllLettersCapital
from BasicPyLib.Printable import PrintableII, Printable
import time

from Config.configTools import get_user_input_and_set_default_values
from IBridgePy.MarketManagerBase import setup_services
from IBridgePy.constants import DataProviderName, BrokerName, LiveBacktest, TraderRunMode, TimeGeneratorType, MarketName

"""
To create a new config
step 1: create a class like MARKET_MANAGER_Config; the format must be XXX_Config or XXX_XXX_Config
step 2: in base_setting.py add setting values as dictionary, naming must be XXX or XXX_XXX. It must to add all possible values and give default values
step 3: If it is user input setting, put default value for user in setting.py
step 4: If it needs special config other than base, add something like Config/backtest_settings.py, naming must be xxxx_settings.py
"""

class Config(PrintableII):
    def __init__(self, dict_settings):
        for key in dict_settings:
            setattr(self, key, dict_settings[key])

    def override(self, settings, settingFileName):
        for key in settings:
            # TODO: The whole part is due to IB_Client is a dict, not a XXX_config class
            if isinstance(settings[key], dict):
                originalValues = None
                try:
                    originalValues = getattr(self, key)
                except AttributeError as e:
                    print(__name__ + '::override: EXIT, %s. Hint: Check Config::base_settings if it has the attribute.' % (e,))
                    exit()
                newValueDict = settings[key]
                for k in newValueDict:
                    if k in originalValues:
                        originalValues[k] = newValueDict[k]
                    else:
                        print(__name__ + '::override: EXIT, key=%s in the file of <%s.py> does not exist in Config::base_settings.py' % (k, settingFileName))
                        exit()
            else:
                self.set_value(key, settings[key], settingFileName)
        return self

    def set_value(self, fieldName, value, settingName):
        """

        :param fieldName:
        :param value:
        :param settingName: string, used to display the fileName/source of the settings
        :return:
        """
        if hasattr(self, fieldName):
            setattr(self, fieldName, value)
        else:
            print(__name__ + '::set_value: class=%s key=%s settingFileName=%s does not exist.' % (self.__class__, fieldName, settingName))
            print('base_config: %s' % (self,))
            exit()

# noinspection PyPep8Naming
class MARKET_MANAGER_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class REPEATER_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class TIME_GENERATOR_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class BROKER_SERVICE_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class PROJECT_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class TRADER_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class BROKER_CLIENT_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class EMAIL_CLIENT_Config(Config):
    def __init__(self, dict_settings):
        Config.__init__(self, dict_settings)

# noinspection PyPep8Naming
class UserConfigBase(Printable):  # should not be used directly
    def __init__(self):
        """
        For example, there is a file Config::hft_settings.py Then, a possible value of addOnSettings is "hft_settings"
        """
        # NEW config system
        self.projectConfig = None
        self.marketManagerConfig = None
        self.repeaterConfig = None
        self.timeGeneratorConfig = None
        self.traderConfig = None
        self.brokerClientConfig = None
        self.emailClientConfig = None

        self.initialize_quantopian = None
        self.handle_data_quantopian = None
        self.before_trading_start_quantopian = None

        self.timeGenerator = None
        self.dataFromServer = None
        self.singleTrader = None
        self.trader = None
        self.dataProvider = None
        self.dataProviderClient = None
        self.log = None
        self.userLog = None
        self.brokerService = None
        self.brokerClient = None
        self.emailClient = None  # to send email out

    def load_settings(self, str_settings):
        module_settings = importlib.import_module(str_settings)
        for item in dir(module_settings):
            if isAllLettersCapital(item):
                if item == 'PROJECT':
                    self.projectConfig = PROJECT_Config(getattr(module_settings, item))
                elif item == 'MARKET_MANAGER':
                    self.marketManagerConfig = MARKET_MANAGER_Config(getattr(module_settings, item))
                elif item == 'REPEATER':
                    self.repeaterConfig = REPEATER_Config(getattr(module_settings, item))
                elif item == 'TIME_GENERATOR':
                    self.timeGeneratorConfig = TIME_GENERATOR_Config(getattr(module_settings, item))
                elif item == 'TRADER':
                    self.traderConfig = TRADER_Config(getattr(module_settings, item))
                elif item == 'BROKER_CLIENT':
                    self.brokerClientConfig = BROKER_CLIENT_Config(getattr(module_settings, item))
                elif item == 'EMAIL_CLIENT':
                    self.emailClientConfig = EMAIL_CLIENT_Config(getattr(module_settings, item))
                else:
                    print(__name__ + '::load_settings: EXIT, cannot handle item=%s in %s.py' % (item, str_settings))
                    exit()
        return self

    def overrideBy(self, str_settings):
        module_settings = importlib.import_module(str_settings)
        for item in dir(module_settings):
            if isAllLettersCapital(item):
                if item == 'PROJECT':
                    self.projectConfig.override(getattr(module_settings, item), str_settings)
                elif item == 'MARKET_MANAGER':
                    self.marketManagerConfig.override(getattr(module_settings, item), str_settings)
                elif item == 'REPEATER':
                    self.repeaterConfig.override(getattr(module_settings, item), str_settings)
                elif item == 'TIME_GENERATOR':
                    self.timeGeneratorConfig.override(getattr(module_settings, item), str_settings)
                elif item == 'TRADER':
                    self.traderConfig.override(getattr(module_settings, item), str_settings)
                elif item == 'BROKER_CLIENT':
                    self.brokerClientConfig.override(getattr(module_settings, item), str_settings)
                elif item == 'EMAIL_CLIENT':
                    self.emailClientConfig.override(getattr(module_settings, item), str_settings)
                else:
                    print(__name__ + '::overrideBy: EXIT, cannot handle item=%s in %s' % (item, str_settings))
                    exit()
        return self

    def build(self):
        # userLog is for the function of record (). User will use it for any reason.
        dateTimeStr = time.strftime("%Y_%m_%d_%H_%M_%S")
        self.userLog = simpleLogger.SimpleLoggerClass(filename='userLog_' + dateTimeStr + '.txt', logLevel='NOTSET', addTime=False)
        sysLogFileName = 'TraderLog_' + time.strftime("%Y-%m-%d") + '.txt'
        self.log = simpleLogger.SimpleLoggerClass(sysLogFileName, self.projectConfig.logLevel)
        return self

    def validate(self):
        if self.projectConfig.dataProviderName in [DataProviderName.RANDOM, DataProviderName.LOCAL_FILE]:
            assert(self.projectConfig.brokerName == BrokerName.LOCAL_BROKER)

        if self.projectConfig.histIngestionPlan is None:
            if self.projectConfig.liveOrBacktest == LiveBacktest.BACKTEST:
                assert(self.projectConfig.dataProviderName == DataProviderName.RANDOM)

        if self.traderConfig.runMode in [TraderRunMode.RUN_LIKE_QUANTOPIAN, TraderRunMode.SUDO_RUN_LIKE_QUANTOPIAN]:
            if self.projectConfig.repBarFreq != 60:
                print(__name__ + '::validate: EXIT, projectConfig.repBarFreq=%s It should be 60 when traderConfig.runMode is either RUN_LIKE_QUANTOPIAN or SUDO_RUN_LIKE_QUANTOPIAN' % (self.projectConfig.repBarFreq,))
                exit()

        # if self.traderConfig.runMode == TraderRunMode.REGULAR:
        #     if self.marketManagerConfig.marketName != MarketName.NONSTOP:
        #         print(__name__ + '::validate: EXIT, marketManagerConfig.marketName=%s, it should be NONSTOP' % (self.marketManagerConfig.marketName,))
        #         exit()

    def load_userInput_to_userConfig(self, user_input):
        # No action to map from user_input to user_config
        for item in ['accountCode', 'repBarFreq', 'fileName', 'logLevel', 'dataProviderName', 'histIngestionPlan']:
            if hasattr(user_input, item):
                self.projectConfig.set_value(item, getattr(user_input, item), __name__ + '::load_userInput_to_userConfig')
        if hasattr(user_input, 'marketName'):
            self.marketManagerConfig.set_value('marketName', user_input.marketName, __name__ + '::load_userInput_to_userConfig')

        if hasattr(user_input, 'timeGeneratorType'):
            self.timeGeneratorConfig.set_value('timeGeneratorType', user_input.timeGeneratorType, __name__ + '::load_userInput_to_userConfig')
            if user_input.timeGeneratorType == TimeGeneratorType.AUTO:
                if hasattr(user_input, 'startTime'):
                    self.timeGeneratorConfig.set_value('startingTime', user_input.startTime, __name__ + '::load_userInput_to_userConfig')
                else:
                    print(__name__ + '::load_input: EXIT, startTime is None.')
                    exit()

                if hasattr(user_input, 'endTime'):
                    self.timeGeneratorConfig.set_value('endingTime', user_input.endTime, __name__ + '::load_userInput_to_userConfig')
                else:
                    print(__name__ + '::load_input: EXIT, endTime is None.')
                    exit()

                if hasattr(user_input, 'freq'):
                    self.timeGeneratorConfig.set_value('freq', user_input.freq, __name__ + '::load_userInput_to_userConfig')
                else:
                    print(__name__ + '::load_input: EXIT, freq is None.')
                    exit()
            elif user_input.timeGeneratorType == TimeGeneratorType.CUSTOM:
                if hasattr(user_input, 'customSpotTimeList'):
                    self.timeGeneratorConfig.set_value('custom', user_input.customSpotTimeList, __name__ + '::load_userInput_to_userConfig')
                else:
                    print(__name__ + '::load_input: EXIT, customSpotTimeList is empty.')
                    exit()

        if hasattr(user_input, 'runMode'):
            if user_input.runMode in [TraderRunMode.RUN_LIKE_QUANTOPIAN, TraderRunMode.SUDO_RUN_LIKE_QUANTOPIAN]:
                self.projectConfig.set_value('repBarFreq', 60, __name__ + '::load_userInput_to_userConfig')
                self.traderConfig.set_value('runMode', user_input.runMode, __name__ + '::load_userInput_to_userConfig')
                if user_input.runMode == TraderRunMode.SUDO_RUN_LIKE_QUANTOPIAN:
                    self.marketManagerConfig.set_value('marketName', MarketName.NONSTOP, __name__ + '::load_userInput_to_userConfig')

        self.initialize_quantopian = user_input.initialize
        self.handle_data_quantopian = user_input.handle_data
        self.before_trading_start_quantopian = user_input.before_trading_start

    def prepare_userConfig_with_trader(self, trader, globalsV):
        userInput = get_user_input_and_set_default_values(globalsV)

        # base userConfig is passed-in.
        # This part only merge userInput into userConfig and then build userConfig.
        self.load_userInput_to_userConfig(userInput)  # set many values in userConfig
        self.build()  # set many values in userConfig

        # cannot instantiate trader here because defined_functions user's input need to go after trader.
        setup_services(self, trader)  # set many values in userConfig
        trader.update_from_userConfig(self)  # should not set any values in userConfig
        self.validate()

class UserConfig(CONSTANTS):
    # Override sequence MATTERS!!!
    REGULAR = UserConfigBase().load_settings('Config.base_settings').overrideBy('settings')
    BACKTEST = UserConfigBase().load_settings('Config.base_settings').overrideBy('Config.backtest_settings').overrideBy('settings')
    HFT = UserConfigBase().load_settings('Config.base_settings').overrideBy('Config.hft_settings').overrideBy('settings')
    ROBINHOOD = UserConfigBase().load_settings('Config.base_settings').overrideBy('Config.robinhood_settings').overrideBy('settings')
    TD = UserConfigBase().load_settings('Config.base_settings').overrideBy('Config.td_settings').overrideBy('settings')

    @staticmethod
    def choose(settingNames):
        if isinstance(settingNames, str):
            list_settingName = [settingNames]
        else:
            list_settingName = settingNames
        ans = UserConfigBase().load_settings('Config.base_settings')
        for settingName in list_settingName:
            # print(__name__ + '::choose: loading %s' % (settingFileName,))
            ans = ans.overrideBy('Config.%s' % (settingName,))
        ans = ans.overrideBy('settings')
        # print(ans)
        return ans

def test__isAllLettersCapital():
    assert(isAllLettersCapital('asdfEdasdf') is False)
    assert(isAllLettersCapital('BROKER_CLIENT') is True)
    assert(isAllLettersCapital('BROKER_CLIENt') is False)


import telebot
import textwrap
import time

API_KEY = "5345795883:AAGD2PIlg7Bqq_IEt1CIuGzmmgExOmOomDQ"
bot = telebot.TeleBot(API_KEY)

def reportNewFoll(username, subs):
    try:
        followers = ""
        for i in subs:
            followers +=  "https://twitter.com/"+i+"\n"

        message = username+' (https://twitter.com/'+username+') '+" followed "+str(len(subs))+" accounts today.\n"+followers
        #bot.send_message("@twittergemsmonitor",message)
        sendMsgFormated(message)
    except Exception as e:
        print(e)

def writeTotal(Matches):
    try:
        message = 'Matches Between Accounts:\n\n'
        if len(Matches)>0:
            for val in Matches:
                message += 'https://twitter.com/'+val[0]+" - "+str(val[1])+"\n"
        else:
            message+="There weren't any matches today :("

        message+="\n\n#total"

        #bot.send_message("@twittergemsmonitor",message)
        sendMsgFormated(message)
    except Exception as e:
        print(e)

def writeSingle(Matches):
    try:
        message = 'Single Mentions:\n\n'
        if len(Matches)>0:
            for val in Matches:
                message += 'https://twitter.com/'+val[0]+" - "+str(val[1])+"\n"
        else:
            message+="There weren't any single matches today :("

        message+="\n\n#single"

        sendMsgFormated(message)

    except Exception as e:
        print(e)

def sendMsgFormated(string):
    chunks = textwrap.wrap(string, width=4050,replace_whitespace=False)
    for i in chunks:
        bot.send_message("@twittergemsmonitor",i)
        time.sleep(5)

# n = ["a","a","1","2","3","4","5","11","11"]
# writeSingle(n)
# subs = ''
# followers = ["1","2","3","4"]
# for i in followers:
#     subs +=  "https://twitter.com/"+i+"\n"

# print(subs)
#reportNewFoll("test_name", ["1","2","3","4","4"])
#writeTotal([('dung_beetle4013', 3), ('tarumi_kei', 2)])


import paramiko
import io
import sys
import boto3
import json
from botocore.exceptions import ClientError

def get_private_key():
    secret_name = "/dev/ssh"
    client = boto3.client('secretsmanager', region_name='us-east-1')

    try:
        get_secret_value_response = client.get_secret_value(
            SecretId=secret_name
        )
    except ClientError as e:
        raise e

    else:
        secret = get_secret_value_response['SecretString']

        secret_dict = json.loads(secret)
        private_key = secret_dict['PrivateKey']

        return private_key

def copy_file(private_key, user, ip, local_path, remote_path):
    private_key_str = io.StringIO()
    private_key_str.write(private_key)
    private_key_str.seek(0)

    key = paramiko.RSAKey.from_private_key(private_key_str)

    private_key_str.close()
    del private_key_str

    trans = paramiko.Transport(ip, 22)
    trans.start_client()
    trans.auth_publickey(user, key)

    del key

    print('Opening transport')
    conn = trans.open_session()

    print('Opening SFTP session')
    sftp = paramiko.SFTPClient.from_transport(trans)

    print('Copying local path {} to remote path {}'.format(local_path, remote_path))
    sftp.put(local_path, remote_path)

    print('Closing SFTP session')
    sftp.close()

    print('Closing transport')
    trans.close()

private_key = get_private_key()
copy_file(private_key, sys.argv[1], sys.argv[2], 'testfile.txt', 'testfile.txt')

# Overwrite and delete the private_key variable, just for safety
private_key = '##############################################'
del private_key


import re
from datetime import datetime
from nfce.models import (Company,
                         EletronicInvoice,
                         PaymentTotals,
                         Product)

UNWANTED_WORDS = [r"UN: *", r"Vl. Unit.:", r"Qtde.:", r"Código:", r"CNPJ:", r"\(", r"\)", r"\n", r"\r", r"\t"]
UNWANTED_WORDS_REGEX = re.compile('|'.join(UNWANTED_WORDS))

def get_totals_info(totals: list, payment_type: str) -> dict:

    values = {}
    for total in totals:
        if total.find('label') is None:
            continue

        key = sanitize_text(total.find('label').string)

        if key == payment_type:
            continue

        key = 'tax' if 'Tributos Totais' in key else key.replace('R$:', '').strip()

        value = to_float(total.find('span').text)
        values[key] = value

    return values

def sanitize_text(text: str) -> str:
    """Sanitize text to remove unwanted characters

    Args:
        text (str): text to be sanitized

    Returns:
        str: sanitized text
    """

    try:
        text = UNWANTED_WORDS_REGEX.sub('', text)
        return text.strip()
    except:
        return ""

def to_float(number: str, radix: str = ',') -> float:
    """ Convert a string to float point number

    Args:
        number (str): string to be converted
        radix (str, optional): radix point separator. Defaults to ",".

    Returns:
        float: converted string
    """
    number = sanitize_text(number)

    return float(number.replace(radix, "."))

class NFCeParser():

    def __init__(self, content_selector='conteudo', info_selector='infos'):
        self.content_selector = content_selector
        self.info_selector = info_selector
        self.page = None
        self.content = None

    def parse(self, page) -> EletronicInvoice:
        self.page = page
        self.content = self.page.find("div", id=self.content_selector)
        self.info = self.page.find("div", id=self.info_selector)

        company = self._get_company()
        totals = self._get_totals()
        items = self._get_items()
        general_info = self._get_general_info()

        invoice = EletronicInvoice(company, items, totals, **general_info)

        return invoice

    def _get_company(self) -> Company:
        """ Get company data from an invoice content

        Returns:
            Company: company data
        """
        info = self.content.find_all("div", class_="text")
        company = Company(sanitize_text(self.content.find_all("div", class_="txtTopo")[0].text),
                          sanitize_text(info[0].text),
                          sanitize_text(info[1].text))
        return company

    def _get_totals(self) -> PaymentTotals:
        """Get all totals from an invoice

        Returns:
            PaymentTotals: totals information
        """

        totals = self.content.find_all("div", id="linhaTotal")
        payment_type = sanitize_text(self.content.find_all("label", class_="tx")[0].text)

        values = get_totals_info(totals, payment_type)

        total_items = int(values.get('Qtd. total de itens:', 0))
        total_after_discount = values.get('Valor a pagar', 0)
        discounts = values.get('Descontos', 0)
        total_before_discount = values.get('Valor total', total_after_discount-discounts)
        payment_type = sanitize_text(payment_type)
        exchange = values.get('Troco', 0)
        tax = values.get('tax', 0)

        return PaymentTotals(exchange, tax, payment_type, total_before_discount, total_after_discount, total_items, discounts)

    def _get_items(self) -> list:
        """Get items from eletronic invoice

        Args:
            content (BeautifullSoup): an html div where table content is located

        Returns:
            [list]: list contaning all produts information
        """
        table = self.content.find("table", id="tabResult")
        products = table.find_all("span", class_="txtTit")
        codes = table.find_all("span", class_="RCod")
        quantities = table.find_all("span", class_="Rqtd")
        uoms = table.find_all("span", class_="RUN")
        unitary_prices = table.find_all("span", class_="RvlUnit")
        total_prices = table.find_all("span", class_="valor")

        qty = len(products)

        items = {}
        not_processed_items = 0
        for i in range(0, qty):
            try:
                code = sanitize_text(codes[i].text)
                if code in items.keys():
                    items[code].quantity += to_float(quantities[i].text)
                    items[code].total_price = items[code].quantity * items[code].price
                else:
                    product = Product(code,
                            sanitize_text(products[i].text),
                            to_float(unitary_prices[i].text),
                            to_float(quantities[i].text),
                            unity_of_measure=sanitize_text(uoms[i].text),
                            total_price=to_float(total_prices[i].text))
                    items[code] = product
            except Exception as ex:
                not_processed_items += 1

        if not_processed_items > 0:
            raise Exception(f"{not_processed_items} items could not be processed")

        return list(items.values())

    def _get_general_info(self) -> dict:
        """Get general information from an invoice

        Returns:
            dict: general information
        """

        try:
            access_key = sanitize_text(self.info.find("span", class_="chave").text)
            access_key = ''.join(access_key.split(' '))
            contents = self.info.find('ul', class_='ui-listview').find('li').contents
            contents = list(filter(lambda el: el != '\n' and el.name != 'br' and el.name != 'strong', contents))

            date = contents[2].split('\n')[0]
            date = ''.join(date.rsplit(':', 1)) # remove last occurrency of : and join to form the timezone
            general_info = {
                'access_key': access_key,
                'number': contents[0],
                'serie': contents[1],
                'issue_date': datetime.strptime(date, '%d/%m/%Y %H:%M:%S%z')
            }
        except:
            general_info = {
                'access_key': '',
                'number': '',
                'serie': '',
                'issue_date': ''
            }

        return general_info


# https://stackoverflow.com/questions/19475955/using-django-models-in-external-python-script
from django.core.management.base import BaseCommand, CommandError

from provision.models import IPzone, NfsExport, Restriction
import sys

class Command(BaseCommand):
    help = "ipdates an ipzmarks to repair duplicates injected by a bug in qrba"

    def add_arguments(self, parser):
        parser.add_argument('zonename', type=str)

    def handle(self, *args, **options):
        zonename = options['zonename']
        zone = IPzone.objects.filter(name=zonename)

        if zone.count() != 1:
            print( "cannot find ipzone " + str(zonename))
            sys.exit(-1)
        unset = set()
        niipa = set()

        ipzmarker = zone.get_ipzone_marker()
        if 'unset' in str(ipzmarker):
            print("ipzmarker unset for zone " + str(zone))
            # z.set_ipaddrs(ipzmarker)
            unset.add(str(zone))

        ipaddrs = zone.get_ipaddrs()
        if str(ipzmarker) not in str(ipaddrs):
            print("ipzmarker " + str(ipzmarker) + " not found in ipaddrs for z " + str(zone))
            niipa.add(str(zone))
            #ipaddrs.append(ipzmarker)
            #zone.set_ipaddrs(ipaddrs)

            #rpset = set()
            #for r in Restriction.objects.all():
            #    for ipz in r.get_ipzones():
            #        if ipz.__eq__(z):
            #            rpset.add(r)

            #nsfxparents = set()
            #for x in NfsExport.objects.all():
            #    xrqs = x.restrictions.get_queryset()
            #    for xr in xrqs:
            #        for r in rpset:
            #            if r.__eq__(xr):
            #                nsfxparents.add(x)

            #if len(nsfxparents) > 0:
            #    print("   nsfparents:")
            #    for x in nsfxparents:
            #        msg = "       " + str(x)
            #        print(msg)
            #    print("\n")

            # else:
            #    print( "ipzmarker " + str(ipzmarker) + " found for z " + str(z) )

            # ipzmarker = z.get_ipzone_marker()
            # ipaddrs = z.get_ipaddrs()
            # if str(ipzmarker) not in str(ipaddrs):
            #    print( "   unset ipzmarker for z " + str(z))

        unlist = []
        for x in unset:
            unlist.append(x)
        unlist.sort()
        nilist = []
        for x in niipa:
            nilist.append(x)
        nilist.sort()

        print("num unset: " + str(len(unlist)))
        print("num niipa: " + str(len(nilist)))
        print("unset: " + str(unset))
        print("niipa: " + str(niipa))


from flask import Flask, render_template, request, jsonify
import garage
app = Flask(__name__)

@app.route('/trigger', methods=['POST'])
def triggerDoor():

	status = {
		'doorState' : "Closing..." if garage.isDoorOpen() else "Opening..."
	}
	garage.sendDoorSignal()

	return jsonify(status)

@app.route('/status')
def getStatus():
	status = {
		'isDoorOpen': garage.isDoorOpen()
      	}

	return jsonify(status)

if __name__ == "__main__":
	app.run(host='0.0.0.0', port=80, debug=True)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
from webassets import Environment, Bundle

dir_static = os.path.join(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
    'static')

static_env = Environment(dir_static)

static_env.register('js_all', Bundle(
    'js/script.js',
    output='gen/all.js'))
static_env['js_all'].build()

# font-awesome is broken when included
static_env.register('css_all', Bundle(
    'css/style.css',
    filters='cssmin', output='gen/all.css'))
static_env['css_all'].build()


import torch
import numpy as np
from scipy.sparse import csr_matrix

def adj_format(adj_mat):
	# adj_mat is V*V tensor, which V is number of image labels.
	E = torch.count_nonzero(adj_mat)
	V = adj_mat.shape[0]
    # Creating a E * V sparse matrix
	adj_mat_S = csr_matrix(adj_mat)
	EV_mat = torch.from_numpy(csr_matrix((E, V),dtype = np.int32).toarray())
	indices = adj_mat_S.nonzero()
	rows = indices[0]
	cols = indices[1]
	data = adj_mat_S.data
	for i in range(E):
		EV_mat[i][rows[i]]=data[i]
		EV_mat[i][cols[i]]=-data[i]
	return EV_mat

def Violation_num(adj_mat, labels):
	# adj_mat is V*V, and labels is a 1*V vector , which V is number of image labels.
	V_vec = torch.matmul(adj_mat,labels) # positive or zero=no_violaton, negative = violation
	V_num=(V_vec<0).sum()
	V_avg = V_num/len(V_vec)

	print(f'Violation scores, and Violation average: {V_vec},  {V_avg}')
	return V_avg

##test sample
row = np.array([0, 1])
col = np.array([1, 2])
data = np.array([1, 1])
B = csr_matrix((data, (row, col)), shape=(3, 3)).toarray()
B = torch.from_numpy(B)

C = adj_format(B)
P = torch.tensor([2,1,0],dtype=torch.int32)
Violation_num(C, P)

P = torch.tensor([0,1,2],dtype=torch.int32)
Violation_num(C, P)


import cv2
import numpy as np
import os

class LoadYOLO():
    def __init__(self, labelPath, cfgPath, weightPath, input_w, input_h, threshold=0.5):
        self.labels = open(labelPath).read().strip().split("\n")

        # initialize a list of colors to represent each possible class label
        np.random.seed(69)
        #self.colors = np.random.randint(0, 255, size=(len(self.labels), 3), dtype="uint8")
        self.net = cv2.dnn.readNetFromDarknet(
            cfgPath, weightPath
        )
        self.output_layers_names = self.net.getUnconnectedOutLayersNames()
        self.threshold = threshold
        self.input_w = input_w
        self.input_h = input_h

    def detect(self, imPlate):

        #self.image = cv2.imread(imPlate)
        self.image = imPlate
        (self.H, self.W) = self.image.shape[:2]

        self.blob = cv2.dnn.blobFromImage(
            self.image, 1/255., (self.input_w,self.input_h),
            swapRB=True, crop=False
        )
		# construct a blob from the input image and then perform a forward
		# pass of the YOLO object detector, giving us our bounding boxes and
		# associated probabilities
        self.net.setInput(self.blob)
        self.layerOutputs = self.net.forward(self.output_layers_names)

        return self.process_output(self.layerOutputs)

    def process_output(self, layerOutputs):
		# initialize our lists of detected bounding boxes, confidences, and
		# class IDs, respectively
        boxes = []
        confidences = []
        classIDs = []

        #loop over layer outputs
        for output in layerOutputs:
            #lopp over each detection
            for detection in output:
                # extract the class ID and confidence (i.e., probability) of
                # the current object detection
                scores = detection[5:]
                classID = np.argmax(scores)
                confidence = scores[classID]

                # filter out weak predictions by ensuring the detected
                # probability is greater than the minimum probability
                if confidence>self.threshold:
                    # scale the bounding box coordinates back relative to the
                    # size of the image, keeping in mind that YOLO actually
                    # returns the center (x, y)-coordinates of the bounding
                    # box followed by the boxes' width and height
                    box = detection[:4] * np.array([self.W, self.H, self.W, self.H])
                    (centerX, centerY, width, height) = box.astype("int")

                    # use the center (x, y)-coordinates to derive the top and
                    # and left corner of the bounding box
                    x = int(centerX - (width/2))
                    y = int(centerY - (height/2))

                    # update our list of bounding box coordinates, confidences,
                    # and class IDs
                    boxes.append([x, y, int(width), int(height)])
                    confidences.append(float(confidence))
                    classIDs.append(classID)

        # apply non-maxima suppression to suppress weak, overlapping bounding boxes
        indexes = cv2.dnn.NMSBoxes(boxes, confidences, self.threshold, 0.3)
        bboxes = []
        outlabels = []

        # ensure at least one detection exists
        if len(indexes)>0:
            for i in indexes.flatten():
                # extract the bounding box coordinates

                for k in range(len(boxes[i])):
                    if boxes[i][k]<0:
                        boxes[i][k]=0

                x,y,w,h = boxes[i]

                H,W = self.image.shape[:2]
                if x+w>W and x>0 and w>0:
                    wid = W-(x+w)
                    if x>w:
                        x += wid
                    else:
                        w +=wid

                if y+h>H and y>0 and h>0:
                    wid = H-(y+h)
                    if y>h:
                        y += wid
                    else:
                        h +=wid

                label = f"{self.labels[classIDs[i]]}"
                confidence = str(round(confidences[i], 3))

                bboxes.append([x,y,w,h])
                outlabels.append(label)

            return list(zip(bboxes, outlabels))
        else:
            return 'None Detected'


"""
采集美团-北京店铺数据

采集的店铺数据包括:
  - id 店铺id
  - title 店铺名
  - avgscore 平均评分
  - backCateName 店铺类型
  - areaname 商圈
  - avgprice 平均价格
  - comments 评论数量

同目录其他文件:
  - cookie - 保存你的 cookie 值
  - citys.json - 城市列表
@Author kvimsg@live.com
@Since 2023-02
"""

import requests
import logging
import json
from typing import List, Dict
import csv
import os

LOGGER_FORMAT = '%(asctime)s %(levelname)s - %(message)s'
logging.basicConfig(format=LOGGER_FORMAT, level='INFO')

class MeituanDataCollector:
    """
    美团店铺数据采集器
    """

    def __init__(self, city_id, cookie):
        self.search_api = "https://apimobile.meituan.com/group/v4/poi/pcsearch/" + str(city_id)
        self.headers = {
            "Host": "apimobile.meituan.com",
            "Origin": "https://sh.meituan.com",
            "Referer": "https://sh.meituan.com/",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
            "Cookie": cookie
        }

    def get_data(self, q, offset, limit) -> List[Dict]:
        """
        获取指定数量的店铺列表
        """
        params = self.__build_params(q, offset, limit)
        res = requests.get(self.search_api, params, headers=self.headers)
        try:
            res.raise_for_status()
        except requests.HTTPError as e:
            logging.warn("Get %s failed", res.url, exc_info=e)
            return []
        search_result = res.json()['data']['searchResult']
        result = []
        for item in search_result:
            result.append({
                "id": item["id"], # id
                "title": item["title"], # 店铺名
                "avgscore": item["avgscore"], # 平均评分
                "backCateName": item["backCateName"], # 店铺类型
                "areaname": item["areaname"], # 商圈
                "avgprice": item["avgprice"], # 平均价格
                "comments": item["comments"], # 评论数量
            })
        return result

    def get_all_data(self, q) -> List[Dict]:
        """
        获取所有店铺列表
        """
        data = []
        offset = 0
        limit = 34
        while True:
            _data = self.get_data(q, offset, limit)
            if not _data: break
            data.extend(_data)
            logging.info("%s门店 %s - %s 采集完成, 成功 %s", q, offset, offset + limit, len(_data))
            offset += limit
        logging.info("%s门店采集完成, 总数 %s", q, len(data))
        return data

    def chagne_city(self, city_id):
        self.search_api = "https://apimobile.meituan.com/group/v4/poi/pcsearch/" + str(city_id)

    def __build_params(self, q, offset=0, limit=32) -> Dict:
        """
        构建请求参数
        """
        return {
            "offset": offset,
            "limit": limit,
            "q": q,
            "cateId": -1,
            "sort": "rating"
        }

def save_to_json(data, filename):
    """
    存储到 json 文件
    """
    json_filename = os.path.join(os.path.dirname(__file__), filename + ".json")
    with open(json_filename, 'w', encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4)
        logging.info("数据已写入 %s", json_filename)

def save_to_csv(data, filename):
    """
    存储到 csv 文件
    """
    if not data: return
    csv_filename = os.path.join(os.path.dirname(__file__), filename + ".csv")
    with open(csv_filename, "w", encoding="utf-8", newline="") as f:
        header = data[0].keys()
        writer = csv.DictWriter(f, fieldnames=header, delimiter=",")
        writer.writeheader()
        for _ in data:
            writer.writerow(_)
        logging.info("数据已写入 %s", csv_filename)

def load_cookie() -> str:
    cookie_file = os.path.join(os.path.dirname(__file__), "cookie")
    with open(cookie_file, "r") as f:
        return f.read()

def find_city_by_name(name: str) -> Dict:
    cookie_file = os.path.join(os.path.dirname(__file__), "citys.json")
    citys = []
    with open(cookie_file, "r", encoding="utf-8") as f:
        citys = json.load(f)
    for city in citys:
        if city["name"] == name:
            return city
    return {}

def load_city_id(cityname):
    city_info = find_city_by_name(cityname)
    if not city_info:
        logging.error("城市 %s 无效", cityname)
        quit()
    return city_info["id"]

if __name__ == "__main__":

    """
    更改 q 和 city 参数即可执行
    """

    q = "川菜"    # 查询店铺名称
    city = "北京" # 城市

    filename = f"{city}-{q}"
    city_id = load_city_id(city)
    cookie = load_cookie()

    collector = MeituanDataCollector(city_id, cookie)
    data = collector.get_all_data(q)

    save_to_csv(data, filename)


#!/usr/bin/python3
import os
import glob
import argparse
import sys

parser = argparse.ArgumentParser(description="Finds python programs with the name given. Only works on Linux and if python_programs directory already exists.")
parser.add_argument("-f", "--file", help='python find_python_program.py [-f|--file] <filename>')
parser.add_argument('-s', '--search', help='python find_python_program.py -f <filename> [-s|--search] <relatedtext>')
args = parser.parse_args()

def main(filename):
	file_list = [found_file for found_file in glob.glob(f'./**/*{file_name}', recursive=True)] # replaced '/home/jkligel/python_programs/**/*{filename} with current directory to look through.'
	for i in file_list:
		print(i)
	return file_list

def search(file_list, relatedtext):
	for file in file_list:
		if os.path.isfile(file):
			with open(file) as fr:
				for line in fr:
					if relatedtext in line:
						print(line)

if len(sys.argv) < 2:
	file_name = str(input('What file are you looking for? '))
	file_list = main(file_name)
	print()
	boolean = str(input('Do you want to search a file?(y/n)' ))
	if boolean == 'y':
		relatedtext = str(input('What is the text you are looking for? '))
		search(file_list[0], relatedtext)
else:
	file_name = args.file
	file_list = main(file_name)
	print()
	if args.search:
		search(file_list, args.search)


import os
from twilio.rest import Client
from dotenv import load_dotenv

load_dotenv()

account_sid = os.environ.get('ACCOUNT_SID')
auth_token = os.environ.get('AUTH_TOKEN')
client = Client(account_sid, auth_token)

message = client.messages.create(
  from_=os.environ.get('FROM_NUMBER'),
  body='Don\'t forget, you have an meeting in 2 days',
  to=os.environ.get('TO_NUMBER')
)

print(message.sid)


import sendgrid,json
from datetime import datetime,timedelta

API_KEY = 'Registrate your own key'

sg = sendgrid.SendGridAPIClient(apikey=API_KEY)

def how_spammed(str_date):
	end_time = datetime.strptime(str_date, '%Y-%m-%d')
	start_time = end_time + timedelta(days=1)
	response = sg.client.suppression.spam_reports.get(query_params={
		'end_time':int(end_time.timestamp()),
		'start_time': int(start_time.timestamp())
	})
	return len(json.loads(response.body))

if __name__ == '__main__':
	print('You had {} spam reports in 2016-01-01'.format(how_spammed('2016-01-01')))
	print('Check your results')


import random
import unittest
from django.db.utils import IntegrityError
from django.test import TestCase
from django.utils import timezone
from datetime import timedelta, date
from data_aggregator.models import Assignment, Job, Participation, Term, \
    Week, Course, JobType, AnalyticTypes, User, TaskTypes
from data_aggregator.utilities import datestring_to_datetime
from mock import MagicMock, patch

class TestTermManager(TestCase):

    def setUp(self):
        self.mock_get_term_by_date = \
            self.create_patch('data_aggregator.models.get_term_by_date')
        self.mock_get_term_by_year_and_quarter = self.create_patch(
            'data_aggregator.models.get_term_by_year_and_quarter')
        self.mock_get_or_create_from_sws_term = self.create_patch(
            'data_aggregator.models.TermManager.get_or_create_from_sws_term')
        self.mock_get_term_for_sis_term_id = self.create_patch(
            'data_aggregator.models.TermManager.get_term_for_sis_term_id')
        self.mock_get_term_for_date = self.create_patch(
            'data_aggregator.models.TermManager.get_term_for_date')

    def restart_all(self):
        self.mock_get_term_by_date.reset_mock()
        self.mock_get_or_create_from_sws_term.reset_mock()
        self.mock_get_term_by_year_and_quarter.reset_mock()
        self.mock_get_term_for_sis_term_id.reset_mock()
        self.mock_get_term_for_date.reset_mock()

    def create_patch(self, name):
        patcher = patch(name)
        self.addCleanup(patcher.stop)
        return patcher.start()

    def test_get_or_create_term_from_sis_term_id(self):

        # assert call sequence when NO sis_term_id is supplied and there is
        # no existing term for the current date does NOT EXIST
        self.mock_get_term_for_date.return_value = None
        mock_sws_term = MagicMock()
        self.mock_get_term_by_date.return_value = mock_sws_term
        Term.objects.get_or_create_term_from_sis_term_id()
        self.assertFalse(self.mock_get_term_for_sis_term_id.called)
        self.assertFalse(self.mock_get_term_by_year_and_quarter.called)
        self.mock_get_term_for_date.assert_called_once()
        self.mock_get_term_by_date.assert_called_once()
        self.mock_get_or_create_from_sws_term.assert_called_once()

        self.restart_all()

        # assert call sequence when NO sis_term_id is supplied and
        # an existing term for the current date EXISTS
        mock_sws_term = MagicMock()
        self.mock_get_term_for_date.return_value = mock_sws_term
        term, created = Term.objects.get_or_create_term_from_sis_term_id()
        self.assertFalse(self.mock_get_term_for_sis_term_id.called)
        self.assertFalse(self.mock_get_term_by_year_and_quarter.called)
        self.mock_get_term_for_date.assert_called_once()
        self.assertFalse(self.mock_get_term_by_date.called)
        self.assertFalse(self.mock_get_term_by_year_and_quarter.called)
        self.assertFalse(self.mock_get_or_create_from_sws_term.called)
        self.assertEqual((term, created), (mock_sws_term, False))

        self.restart_all()

        # assert call sequence when sis_term_id IS SUPPLIED but NOT present
        # in the database
        self.mock_get_term_for_sis_term_id.return_value = None  # not in db
        self.mock_get_term_for_date.return_value = None
        Term.objects.get_or_create_term_from_sis_term_id(
                                                    sis_term_id="2021-spring")
        self.mock_get_term_for_sis_term_id.assert_called_once()
        self.mock_get_term_by_year_and_quarter.assert_called_once()
        self.mock_get_or_create_from_sws_term.assert_called_once()
        self.assertFalse(self.mock_get_term_for_date.called)
        self.assertFalse(self.mock_get_term_by_date.called)

        self.restart_all()

        # assert call sequence when sis_term_id IS SUPPLIED and IS present
        # in the database
        self.mock_get_term_for_date.return_value = None
        self.mock_get_term_for_sis_term_id.return_value = MagicMock()  # in db
        Term.objects.get_or_create_term_from_sis_term_id(
                                                    sis_term_id="2021-spring")
        self.mock_get_term_for_sis_term_id.assert_called_once()
        self.assertFalse(self.mock_get_term_by_year_and_quarter.called)
        self.assertFalse(self.mock_get_term_for_date.called)
        self.assertFalse(self.mock_get_term_by_date.called)
        self.assertFalse(self.mock_get_or_create_from_sws_term.called)

        self.restart_all()

        # assert call sequence when sis_term_id IS NOT SUPPLIED and NOT present
        # in the database and current date is NOT IN A TERM
        self.mock_get_term_for_date.return_value = None
        mock_sws_term = MagicMock()
        self.mock_get_term_by_date.return_value = mock_sws_term
        self.mock_get_or_create_from_sws_term.return_value = \
            (mock_sws_term, False)
        term, created = Term.objects.get_or_create_term_from_sis_term_id()
        self.assertFalse(self.mock_get_term_for_sis_term_id.called)
        self.assertFalse(self.mock_get_term_by_year_and_quarter.called)
        self.mock_get_term_for_date.assert_called_once()
        self.mock_get_term_by_date.assert_called_once()
        self.mock_get_or_create_from_sws_term.assert_called_once()
        self.assertEqual((term, created), (mock_sws_term, False))

class TestWeekManager(TestCase):

    def setUp(self):
        self.mock_week = MagicMock()
        self.mock_week_get_or_create = self.create_patch(
            'data_aggregator.models.WeekManager.get_or_create')
        self.mock_week_get_or_create.return_value = \
            self.mock_week, None
        self.mock_get_relative_week = self.create_patch(
            'data_aggregator.utilities.get_relative_week')
        self.mock_term = MagicMock()
        self.mock_get_or_create_term_from_sis_term_id = self.create_patch(
            'data_aggregator.models.TermManager.'
            'get_or_create_term_from_sis_term_id')
        self.mock_get_or_create_term_from_sis_term_id.return_value = \
            self.mock_term, None

    def restart_all(self):
        self.mock_week_get_or_create.reset_mock()
        self.mock_get_relative_week.reset_mock()
        self.mock_get_or_create_term_from_sis_term_id.reset_mock()

    def create_patch(self, name):
        patcher = patch(name)
        self.addCleanup(patcher.stop)
        return patcher.start()

    def test_get_or_create_week(self):
        # assert call sequence when no parameters are passed
        Week.objects.get_or_create_week()
        self.assertTrue(self.mock_get_or_create_term_from_sis_term_id.called)
        self.assertTrue(self.mock_get_relative_week.called)
        self.assertTrue(self.mock_week_get_or_create.called)
        self.mock_week_get_or_create.assert_called_with(
            term=self.mock_term,
            week=self.mock_get_relative_week.return_value)

        self.restart_all()

        # assert call sequence when term is passed
        Week.objects.get_or_create_week(sis_term_id="2021-summer")
        self.assertTrue(self.mock_get_or_create_term_from_sis_term_id.called)
        self.mock_get_or_create_term_from_sis_term_id.assert_called_with(
            sis_term_id="2021-summer")
        self.assertTrue(self.mock_get_relative_week.called)
        self.assertTrue(self.mock_week_get_or_create.called)
        self.mock_week_get_or_create.assert_called_with(
            term=self.mock_term,
            week=self.mock_get_relative_week.return_value)

        self.restart_all()

        # assert call sequence when week is passed
        Week.objects.get_or_create_week(week_num=2)
        self.assertTrue(self.mock_get_or_create_term_from_sis_term_id.called)
        self.assertFalse(self.mock_get_relative_week.called)
        self.assertTrue(self.mock_week_get_or_create.called)
        self.mock_week_get_or_create.assert_called_with(
            term=self.mock_term,
            week=2)

        self.restart_all()

        # assert call sequence when term and week is passed
        Week.objects.get_or_create_week(sis_term_id="2021-summer", week_num=2)
        self.assertTrue(self.mock_get_or_create_term_from_sis_term_id.called)
        self.assertFalse(self.mock_get_relative_week.called)
        self.assertTrue(self.mock_week_get_or_create.called)
        self.mock_week_get_or_create.assert_called_with(
            term=self.mock_term,
            week=2)

class TestWeek(TestCase):

    def test_get_week_end_date(self):
        # 2021-summer
        term = Term()
        term.first_day_quarter = date(2021, 6, 20)
        week = Week()
        week.term = term
        week.week = 1
        self.assertEqual(week.end_date, date(2021, 6, 26))
        week.week = 2
        self.assertEqual(week.end_date, date(2021, 7, 3))
        week.week = 3
        self.assertEqual(week.end_date, date(2021, 7, 10))
        week.week = 4
        self.assertEqual(week.end_date, date(2021, 7, 17))
        week.week = 5
        self.assertEqual(week.end_date, date(2021, 7, 24))
        week.week = 6
        self.assertEqual(week.end_date, date(2021, 7, 31))
        week.week = 7
        self.assertEqual(week.end_date, date(2021, 8, 7))
        week.week = 8
        self.assertEqual(week.end_date, date(2021, 8, 14))
        week.week = 9
        self.assertEqual(week.end_date, date(2021, 8, 21))
        week.week = 10
        self.assertEqual(week.end_date, date(2021, 8, 28))
        week.week = 11
        self.assertEqual(week.end_date, date(2021, 9, 4))
        week.week = 12
        self.assertEqual(week.end_date, date(2021, 9, 11))

        # 2021-autumn
        term = Term()
        term.first_day_quarter = date(2021, 9, 28)
        week = Week()
        week.term = term
        week.week = 1
        self.assertEqual(week.end_date, date(2021, 10, 2))
        week.week = 2
        self.assertEqual(week.end_date, date(2021, 10, 9))
        week.week = 3
        self.assertEqual(week.end_date, date(2021, 10, 16))
        week.week = 4
        self.assertEqual(week.end_date, date(2021, 10, 23))
        week.week = 5
        self.assertEqual(week.end_date, date(2021, 10, 30))
        week.week = 6
        self.assertEqual(week.end_date, date(2021, 11, 6))
        week.week = 7
        self.assertEqual(week.end_date, date(2021, 11, 13))
        week.week = 8
        self.assertEqual(week.end_date, date(2021, 11, 20))
        week.week = 9
        self.assertEqual(week.end_date, date(2021, 11, 27))
        week.week = 10
        self.assertEqual(week.end_date, date(2021, 12, 4))
        week.week = 11
        self.assertEqual(week.end_date, date(2021, 12, 11))
        week.week = 12
        self.assertEqual(week.end_date, date(2021, 12, 18))

class TestJob(TestCase):

    type = JobType()
    type.type = AnalyticTypes.assignment
    target_date_start = timezone.now()
    target_date_end = timezone.now() + timedelta(days=1)
    start = timezone.now()
    end = timezone.now() + timedelta(minutes=5)
    created = timezone.now()

    def get_test_job_full(self):
        # initialize job full of values
        job = Job()
        job.type = TestJob.type
        job.target_date_start = TestJob.target_date_start
        job.target_date_end = TestJob.target_date_end
        job.pid = 12345
        job.start = TestJob.start
        job.end = TestJob.end
        job.message = "mock_message"
        job.created = TestJob.created
        return job

    def test_status(self):
        job = Job()
        job.target_date_start = timezone.now()
        job.target_date_end = timezone.now() + timedelta(days=1)
        job.pid = None
        job.start = None
        job.end = None
        job.message = None
        self.assertEqual(job.status, "pending")
        job.pid = 12345
        self.assertEqual(job.status, "claimed")
        job.target_date_end = timezone.now() - timedelta(minutes=1)
        self.assertEqual(job.status, "expired")
        job.start = timezone.now()
        self.assertEqual(job.status, "running")
        job.end = timezone.now()
        self.assertEqual(job.status, "completed")
        job.message = "error"
        self.assertEqual(job.status, "failed")

    def test_restart_job(self):
        job = self.get_test_job_full()
        orig_target_date_start = job.target_date_start
        orig_target_date_end = job.target_date_end
        # retart job
        job.restart_job(save=False)
        # assert values after restart
        self.assertEqual(job.type, TestJob.type)
        self.assertNotEqual(job.target_date_start, orig_target_date_start)
        self.assertNotEqual(job.target_date_end, orig_target_date_end)
        self.assertEqual(job.pid, None)
        self.assertEqual(job.start, None)
        self.assertEqual(job.end, None)
        self.assertEqual(job.message, "")
        self.assertEqual(job.created, TestJob.created)

    def test_claim_job(self):
        job = self.get_test_job_full()
        job.pid = None  # set pid to None
        # claim job
        job.claim_job(save=False)
        # assert values after claim
        self.assertEqual(job.type, TestJob.type)
        self.assertNotEqual(job.target_date_start, None)
        self.assertNotEqual(job.target_date_end, None)
        self.assertNotEqual(job.pid, None)  # pid is set in job claim
        self.assertEqual(job.start, None)  # start is set to None in job claim
        self.assertEqual(job.end, None)  # end is set to None in job claim
        self.assertEqual(job.message, '')  # message is set to '' in job claim
        self.assertEqual(job.created, TestJob.created)

    def test_start_job(self):
        job = self.get_test_job_full()
        # claim job
        job.start_job(save=False)
        # assert values after claim
        self.assertEqual(job.type, TestJob.type)
        self.assertNotEqual(job.target_date_start, None)
        self.assertNotEqual(job.target_date_end, None)
        self.assertNotEqual(job.pid, None)
        self.assertNotEqual(job.start, None)  # start is set in job start
        self.assertEqual(job.end, None)  # end is set to None job start
        self.assertEqual(job.message, '')  # message is set to '' job start
        self.assertEqual(job.created, TestJob.created)

        job = self.get_test_job_full()
        with self.assertRaises(RuntimeError):
            # test that runtime error is raised when trying to start an
            # unclaimed job
            job.pid = None
            job.start_job(save=False)

    def test_end_job(self):
        job = self.get_test_job_full()
        # claim job
        job.end_job(save=False)
        # assert values after claim
        self.assertEqual(job.type, TestJob.type)
        self.assertNotEqual(job.target_date_start, None)
        self.assertNotEqual(job.target_date_end, None)
        self.assertNotEqual(job.pid, None)
        self.assertNotEqual(job.start, None)
        self.assertNotEqual(job.end, None)  # end is set in job end
        self.assertEqual(job.message, '')  # message is set to '' job end
        self.assertEqual(job.created, TestJob.created)

        job = self.get_test_job_full()
        with self.assertRaises(RuntimeError):
            # test that runtime error is raised when trying to start an
            # unclaimed job
            job.pid = None
            job.start_job(save=False)

        job = self.get_test_job_full()
        with self.assertRaises(RuntimeError):
            # test that runtime error is raised when trying to start an
            # unstarted job
            job.pid = None
            job.start = None
            job.start_job(save=False)

    def test_to_dict(self):
        job = self.get_test_job_full()
        job_dict = job.to_dict()
        self.assertEqual(job_dict["type"], job.type.type)
        self.assertEqual(job_dict["target_date_start"], job.target_date_start)
        self.assertEqual(job_dict["target_date_end"], job.target_date_end)
        self.assertEqual(job_dict["pid"], job.pid)
        self.assertEqual(job_dict["start"], job.start)
        self.assertEqual(job_dict["end"], job.end)
        self.assertEqual(job_dict["message"], job.message)
        self.assertEqual(job_dict["created"], job.created)

class TestJobManager(TestCase):

    fixtures = ['data_aggregator/fixtures/mock_data/da_job.json',
                'data_aggregator/fixtures/mock_data/da_jobtype.json']

    def get_mock_job_manager(self):
        Job.objects.get_pending_jobs = \
            MagicMock(side_effect=Job.objects.get_pending_jobs)
        Job.objects.get_pending_or_running_jobs = \
            MagicMock(side_effect=Job.objects.get_pending_or_running_jobs)
        return Job.objects

    def test_restart_jobs(self):
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-2T12:00:00.0Z")):
            mock_jm = self.get_mock_job_manager()
            # claim all active jobs in the target range
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.assignment)
            for job in active_jobs:
                job.claim_job()
                self.assertEqual(job.status, "claimed")
            job_ids = [job.id for job in active_jobs]
            # restart all active jobs in the target range
            mock_jm.restart_jobs(job_ids)
            # query for all active jobs again and ensure that they were
            # restarted
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.assignment)
            for job in active_jobs:
                self.assertEqual(job.status, "pending")

    def test_get_active_jobs(self):
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-2T12:00:00.0Z")):
            mock_jm = self.get_mock_job_manager()
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.assignment)
            self.assertEqual(len(active_jobs), 2)
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.participation)
            self.assertEqual(len(active_jobs), 1)
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-1T12:00:00.0Z")):
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.assignment)
            self.assertEqual(len(active_jobs), 1)
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.participation)
            self.assertEqual(len(active_jobs), 3)
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-03-2T12:00:00.0Z")):
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.assignment)
            self.assertEqual(len(active_jobs), 0)
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.participation)
            self.assertEqual(len(active_jobs), 0)

    def test_get_running_jobs_for_term_week(self):
        # test get running assignment jobs
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-2T12:00:00.0Z")):
            mock_jm = self.get_mock_job_manager()
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.assignment)
            self.assertEqual(len(active_jobs), 2)
            # fake starting the active jobs
            for job in active_jobs:
                job.pid = random.randint(1, 1000)  # fake pid
                job.start = timezone.now()
                job.save()
            # assert the correct context
            for job in active_jobs:
                self.assertEqual(job.context["sis_term_id"], "2013-spring")
                self.assertEqual(job.context["week"], 1)
            running_jobs = mock_jm.get_running_jobs_for_term_week(
                AnalyticTypes.assignment, "2013-spring", 1)
            self.assertEqual(len(running_jobs), 2)
            wrong_term_jobs = mock_jm.get_running_jobs_for_term_week(
                AnalyticTypes.assignment, "2013-autumn", 1)
            self.assertEqual(len(wrong_term_jobs), 0)
            wrong_week_jobs = mock_jm.get_running_jobs_for_term_week(
                AnalyticTypes.assignment, "2013-spring", 2)
            self.assertEqual(len(wrong_week_jobs), 0)

        # test get running participation jobs
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-2T12:00:00.0Z")):
            mock_jm = self.get_mock_job_manager()
            active_jobs = mock_jm.get_active_jobs(AnalyticTypes.participation)
            self.assertEqual(len(active_jobs), 1)
            # fake starting the active jobs
            for job in active_jobs:
                job.pid = random.randint(1, 1000)  # fake pid
                job.start = timezone.now()
                job.save()
            # assert the correct context
            for job in active_jobs:
                self.assertEqual(job.context["sis_term_id"], "2013-spring")
                self.assertEqual(job.context["week"], 1)
            running_jobs = mock_jm.get_running_jobs_for_term_week(
                AnalyticTypes.participation, "2013-spring", 1)
            self.assertEqual(len(running_jobs), 1)
            wrong_term_jobs = mock_jm.get_running_jobs_for_term_week(
                AnalyticTypes.participation, "2013-autumn", 1)
            self.assertEqual(len(wrong_term_jobs), 0)
            wrong_week_jobs = mock_jm.get_running_jobs_for_term_week(
                AnalyticTypes.participation, "2013-spring", 2)
            self.assertEqual(len(wrong_week_jobs), 0)

        # test for invalid analytics type
        with self.assertRaises(ValueError):
            mock_jm.get_running_jobs_for_term_week(
                TaskTypes.create_terms, "2013-spring", 2)
            mock_jm.get_running_jobs_for_term_week(
                TaskTypes.create_or_update_courses, "2013-spring", 2)
            mock_jm.get_running_jobs_for_term_week(
                TaskTypes.create_or_update_users, "2013-spring", 2)

    def test_claim_batch_of_assignment_jobs(self):
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-02T12:00:00.0Z")):

            mock_jm = self.get_mock_job_manager()
            # assert that all assignment jobs are initially unclaimed
            for job in mock_jm.get_jobs(AnalyticTypes.assignment):
                self.assertNotEqual(job.status, "claimed")
            # assert that participation jobs are correctly claimed
            claimed_assignment_jobs = \
                mock_jm.claim_batch_of_jobs(AnalyticTypes.assignment)
            self.assertEqual(len(claimed_assignment_jobs), 2)
            for job in claimed_assignment_jobs:
                self.assertEqual(job.status, "claimed")
                self.assertEqual(job.type.type, AnalyticTypes.assignment)
            self.assertEqual(mock_jm.get_pending_jobs.called, True)
            self.assertEqual(mock_jm.get_pending_or_running_jobs.called,
                             False)

            # assert that all assignment jobs are now claimed
            for job in mock_jm.get_active_jobs(AnalyticTypes.assignment):
                self.assertEqual(job.status, "claimed")
            # assert reclaiming assignment jobs
            claimed_assignment_jobs = \
                mock_jm.claim_batch_of_jobs(AnalyticTypes.assignment)
            self.assertEqual(len(claimed_assignment_jobs), 2)
            for job in claimed_assignment_jobs:
                self.assertEqual(job.status, "claimed")
                self.assertEqual(job.type.type, AnalyticTypes.assignment)
            self.assertEqual(mock_jm.get_pending_jobs.called, True)
            self.assertEqual(mock_jm.get_pending_or_running_jobs.called,
                             True)

    def test_claim_batch_of_participation_jobs(self):
        with patch.object(
                timezone, "now",
                return_value=datestring_to_datetime("2021-04-2T12:00:00.0Z")):

            mock_jm = self.get_mock_job_manager()
            # assert that all participation jobs are initially unclaimed
            for job in mock_jm.get_active_jobs(AnalyticTypes.participation):
                self.assertNotEqual(job.status, "claimed")
            # assert that participation jobs are correctly claimed
            claimed_participation_jobs = \
                mock_jm.claim_batch_of_jobs(AnalyticTypes.participation)
            self.assertEqual(len(claimed_participation_jobs), 1)
            for job in claimed_participation_jobs:
                self.assertEqual(job.status, "claimed")
                self.assertEqual(job.type.type, AnalyticTypes.participation)
            self.assertEqual(mock_jm.get_pending_jobs.called, True)
            self.assertEqual(mock_jm.get_pending_or_running_jobs.called,
                             False)

            # assert that all participation jobs are initially claimed
            for job in mock_jm.get_active_jobs(AnalyticTypes.participation):
                self.assertEqual(job.status, "claimed")
            # assert reclaiming participation jobs
            claimed_participation_jobs = \
                mock_jm.claim_batch_of_jobs(AnalyticTypes.participation)
            self.assertEqual(len(claimed_participation_jobs), 1)
            for job in claimed_participation_jobs:
                self.assertEqual(job.status, "claimed")
                self.assertEqual(job.type.type, AnalyticTypes.participation)
            self.assertEqual(mock_jm.get_pending_jobs.called, True)
            self.assertEqual(mock_jm.get_pending_or_running_jobs.called,
                             True)

class TestAssignmentManager(TestCase):

    fixtures = ['data_aggregator/fixtures/mock_data/da_assignment.json',
                'data_aggregator/fixtures/mock_data/da_course.json',
                'data_aggregator/fixtures/mock_data/da_job.json',
                'data_aggregator/fixtures/mock_data/da_jobtype.json',
                'data_aggregator/fixtures/mock_data/da_term.json',
                'data_aggregator/fixtures/mock_data/da_user.json',
                'data_aggregator/fixtures/mock_data/da_week.json']

    @patch("data_aggregator.models.Assignment")
    @patch("data_aggregator.models.User.objects.get")
    def test_create_or_update_assignment(self, mock_user_get,
                                         mock_assignment):
        job = Job()
        user = User()
        term = Term()
        week = Week()
        week.week = 1
        week.term = term

        course = Course()
        course.term = term

        raw_assign_dict = {
            'assignment_id': 6418106,
            'title': 'Basic python, submission',
            'unlock_at': None,
            'points_possible': 2.0,
            'non_digital_submission': False,
            'multiple_due_dates': False,
            'due_at': '2021-06-26T06:05:00Z',
            'status': 'on_time',
            'muted': False,
            'max_score': 2.0,
            'min_score': 0.0,
            'first_quartile': 2.0,
            'median': 2.0,
            'third_quartile': 2.0,
            'module_ids': [],
            'excused': False,
            'submission': {
                'posted_at': '2021-07-01T16:27:44Z',
                'score': 2.0,
                'submitted_at': '2021-06-25T05:47:51Z'},
            'canvas_user_id': 3933194,
            'canvas_course_id': 1458152}

        mock_user_get.return_value = user

        # check for create
        assign, created = Assignment.objects.create_or_update_assignment(
                                            job, week, course, raw_assign_dict)

        mock_user_get.assert_called_once()
        mock_assignment_inst = mock_assignment()
        mock_assignment_inst.save.assert_called_once()

        self.assertEqual(created, True)

        self.assertEqual(assign.job, job)
        self.assertEqual(assign.week, week)
        self.assertEqual(assign.course, course)
        self.assertEqual(assign.user, user)
        self.assertEqual(assign.assignment_id,
                         raw_assign_dict["assignment_id"])
        self.assertEqual(assign.title, raw_assign_dict["title"])
        self.assertEqual(assign.unlock_at, raw_assign_dict["unlock_at"])
        self.assertEqual(assign.points_possible,
                         raw_assign_dict["points_possible"])
        self.assertEqual(assign.non_digital_submission,
                         raw_assign_dict["non_digital_submission"])
        self.assertEqual(assign.due_at, raw_assign_dict["due_at"])
        self.assertEqual(assign.status, raw_assign_dict["status"])
        self.assertEqual(assign.muted, raw_assign_dict["muted"])
        self.assertEqual(assign.max_score, raw_assign_dict["max_score"])
        self.assertEqual(assign.first_quartile,
                         raw_assign_dict["first_quartile"])
        self.assertEqual(assign.median, raw_assign_dict["median"])
        self.assertEqual(assign.third_quartile,
                         raw_assign_dict["third_quartile"])
        self.assertEqual(assign.excused, raw_assign_dict["excused"])
        submission = raw_assign_dict["submission"]
        self.assertEqual(assign.score, submission["score"])
        self.assertEqual(assign.posted_at, submission["posted_at"])
        self.assertEqual(assign.submitted_at, submission["submitted_at"])

        # check update
        mock_assignment_inst.save.side_effect = IntegrityError
        mock_existing_assignment = mock_assignment.objects.get.return_value
        _, created = Assignment.objects.create_or_update_assignment(
                                        job, week, course, raw_assign_dict)
        self.assertEqual(created, False)
        mock_existing_assignment.save.assert_called_once()

    def test_integrity_error(self):
        # assert that saving a non unique participation raises an integrity
        # error
        assign = Assignment.objects.all().first()
        duplicate_assign = Assignment()
        duplicate_assign.course = assign.course
        duplicate_assign.job = assign.job
        duplicate_assign.week = assign.week
        duplicate_assign.user = assign.user
        duplicate_assign.assignment_id = assign.assignment_id
        duplicate_assign.title = assign.title
        duplicate_assign.unlock_at = assign.unlock_at
        duplicate_assign.points_possible = assign.points_possible
        duplicate_assign.non_digital_submission = assign.non_digital_submission
        duplicate_assign.due_at = assign.due_at
        duplicate_assign.status = assign.status
        duplicate_assign.muted = assign.muted
        duplicate_assign.min_score = assign.min_score
        duplicate_assign.max_score = assign.max_score
        duplicate_assign.first_quartile = assign.first_quartile
        duplicate_assign.median = assign.median
        duplicate_assign.third_quartile = assign.third_quartile
        duplicate_assign.excused = assign.excused
        duplicate_assign.score = assign.score
        duplicate_assign.posted_at = assign.posted_at
        duplicate_assign.submitted_at = assign.submitted_at
        with self.assertRaises(IntegrityError):
            duplicate_assign.save()

class TestParticipationManager(TestCase):

    fixtures = ['data_aggregator/fixtures/mock_data/da_participation.json',
                'data_aggregator/fixtures/mock_data/da_course.json',
                'data_aggregator/fixtures/mock_data/da_job.json',
                'data_aggregator/fixtures/mock_data/da_jobtype.json',
                'data_aggregator/fixtures/mock_data/da_term.json',
                'data_aggregator/fixtures/mock_data/da_user.json',
                'data_aggregator/fixtures/mock_data/da_week.json']

    @patch("data_aggregator.models.Participation")
    @patch("data_aggregator.models.User.objects.get")
    def test_create_or_update_participation(self, mock_user_get,
                                            mock_participation):
        job = Job()
        user = User()
        term = Term()
        week = Week()
        week.week = 1
        week.term = term

        course = Course()
        course.term = term

        raw_partic_dict = {
            'page_views': 9,
            'max_page_views': 9,
            'page_views_level': 3,
            'participations': 0,
            'max_participations': 0,
            'participations_level': 0,
            'tardiness_breakdown': {
                'missing': 0,
                'late': 0,
                'on_time': 0,
                'floating': 0,
                'total': 0},
            'canvas_user_id': 3193994,
            'canvas_course_id': 835454}

        mock_user_get.return_value = user

        # check for create
        partic, created = Participation.objects.create_or_update_participation(
                                            job, week, course, raw_partic_dict)

        mock_user_get.assert_called_once()
        mock_participation_inst = mock_participation()
        mock_participation_inst.save.assert_called_once()

        self.assertEqual(created, True)

        self.assertEqual(partic.job, job)
        self.assertEqual(partic.week, week)
        self.assertEqual(partic.course, course)
        self.assertEqual(partic.user, user)
        self.assertEqual(partic.page_views, raw_partic_dict["page_views"])
        self.assertEqual(partic.max_page_views,
                         raw_partic_dict["max_page_views"])
        self.assertEqual(partic.page_views_level,
                         raw_partic_dict["page_views_level"])
        self.assertEqual(partic.participations,
                         raw_partic_dict["participations"])
        self.assertEqual(partic.max_participations,
                         raw_partic_dict["max_participations"])
        self.assertEqual(partic.participations_level,
                         raw_partic_dict["participations_level"])
        tardiness_breakdown = raw_partic_dict["tardiness_breakdown"]
        self.assertEqual(partic.time_missing, tardiness_breakdown["missing"])
        self.assertEqual(partic.time_late, tardiness_breakdown["late"])
        self.assertEqual(partic.time_on_time, tardiness_breakdown["on_time"])
        self.assertEqual(partic.time_floating, tardiness_breakdown["floating"])
        self.assertEqual(partic.time_total, tardiness_breakdown["total"])

        # check update
        mock_participation_inst.save.side_effect = IntegrityError
        mock_existing_participation = \
            mock_participation.objects.get.return_value
        _, created = Participation.objects.create_or_update_participation(
                                        job, week, course, raw_partic_dict)
        self.assertEqual(created, False)
        mock_existing_participation.save.assert_called_once()

    def test_integrity_error(self):
        # assert that saving a non unique participation raises an integrity
        # error
        partic = Participation.objects.all().first()
        duplicate_partic = Participation()
        duplicate_partic.course = partic.course
        duplicate_partic.job = partic.job
        duplicate_partic.week = partic.week
        duplicate_partic.user = partic.user
        duplicate_partic.page_views = partic.page_views
        duplicate_partic.max_page_views = partic.max_page_views
        duplicate_partic.page_views_level = partic.page_views_level
        duplicate_partic.participations = partic.participations
        duplicate_partic.max_participations = partic.max_participations
        duplicate_partic.participations_level = partic.participations_level
        duplicate_partic.time_total = partic.time_total
        duplicate_partic.time_on_time = partic.time_on_time
        duplicate_partic.time_late = partic.time_late
        duplicate_partic.time_missing = partic.time_missing
        duplicate_partic.time_floating = partic.time_floating
        with self.assertRaises(IntegrityError):
            duplicate_partic.save()

if __name__ == "__main__":
    unittest.main()