from functools import partial
import numpy as np
import cv2
import os
import sys
import time
import pickle
from scipy.optimize import least_squares, minimize, Bounds, curve_fit
from lmfit import Parameters, fit_report, minimize

from common.meshes import Model
from common.face_model import ICTFaceModel68

import jax
from jax.config import config
#config.update("jax_debug_nans", True)
config.update("jax_enable_x64", True)
#os.environ['JAX_PLATFORMS']='cpu'

import jax.dlpack as jdlp

from jax import jit
import jax.experimental.host_callback as hcb
import jax.numpy as jnp
from jaxopt.linear_solve import solve_cg

from jaxopt import LevenbergMarquardt, LBFGS, ScipyBoundedMinimize, ScipyBoundedLeastSquares, ScipyLeastSquares

#@partial(jax.jit, static_argnames=['neutral', 'bs_arr'])

def unpack(params):
    measured_points = params[0:68,0:2]
    rmat = params[68:68+3,:]
    tvec = params[68+3]
    return measured_points,rmat,tvec

def project_points(points, rmat, tvec, cmat):
    transformed = (points@rmat.T + tvec)@cmat.T
    projected = jnp.divide(transformed[:,:2], transformed[:,2:])
    return projected

def ortho_proj(points, rmat, tvec, cmat):
    scale = jnp.divide(cmat[0][0], tvec[2])
    c_x = cmat[0][2]
    c_y = cmat[1][2]
    t_x = c_x + tvec[0]*scale
    t_y = c_y + tvec[1]*scale
    translation = jnp.array([t_x, t_y, 0])
    projected = ((points*scale).dot(rmat.T) + translation)[:,2]
    return projected

#def residuals(w, measured_points, rmat, tvec, cmat, neutral, bs_arr):
@jax.jit
def residuals(w, non_fit_params, cmat, former_w, neutral, bs_arr, b_fit, b_prior, b_sparse):
    measured_points, rmat, tvec = unpack(non_fit_params)

    tvec_flat = tvec.flatten()

    #scale = jnp.divide(cmat[0][0], tvec_flat[2])

    sparse_weighted = neutral+(bs_arr.T@w.reshape(-1)).T

    #e_fit =(project_points(sparse_weighted, rmat, tvec_flat, cmat)/scale) - (measured_points/scale) # face it
    e_fit = project_points(sparse_weighted, rmat, tvec_flat, cmat) - measured_points # face it

    #hcb.id_print(e_fit)

    a=1.002
    b=2e-5
    #c=2.47
    c=2.47
    d=0.0002

    e_prior = (c/jnp.pi)*( jnp.arctan((w-a)/b)-jnp.arctan((w-a+1)/b) )+c+d # face it

    #e_prior = jnp.linalg.norm( ( jnp.arctan((w-a)/b)-jnp.arctan((w-a+1)/b) )+4*jnp.pi+c, ord=1)
    #prior = 4*( jnp.arctan((w.astype(jnp.float64).at[0]-a)/b)-jnp.arctan((w.astype(jnp.float64).at[0]-a+1)/b) )+4*jnp.pi+c
    #hcb.id_print(prior)
    #hcb.id_print(w.at[0])

    e_sparse =  jnp.array(jnp.linalg.norm(w, ord = 1)).reshape(-1,) # a154blancoiribera
    #e_sparse = jnp.array(jnp.linalg.norm(w, ord = 2)**2).reshape(-1,)
    #e_sparse = reg*reg

    #e_temp = jnp.array(jnp.linalg.norm(former_w-w, ord = 2)**2).reshape(-1,)

    #hcb.id_print(e_sparse)
    #residuals_arr = jnp.append(jnp.concatenate([e_fit.ravel(), prior.ravel()]), e_sparse)

    residuals_arr = jnp.concatenate((b_fit*e_fit.ravel(), b_prior*e_prior, b_sparse*e_sparse))#,0.7*e_reg])

    return residuals_arr

@jax.jit
def energy(w, non_fit_params, cmat, former_w, neutral, bs_arr, b_fit, b_prior, b_sparse):
    measured_points, rmat, tvec = unpack(non_fit_params)

    sparse_weighted = neutral+(bs_arr.T@w.reshape(-1)).T
    e_fit = jnp.linalg.norm(project_points(sparse_weighted, rmat, tvec, cmat) - measured_points, ord=2)**2

    #a=1.002
    #b=2e-5
    #c=2.47
    #e_prior = jnp.linalg.norm( (jnp.pi/4)*( jnp.arctan((w-a)/b)-jnp.arctan((w-a-1)/b) )+c, ord=2)**2

    e_sparse = jnp.linalg.norm(w, ord = 1)
    e_prior = jnp.linalg.norm(former_w-w, ord = 2)**2
    #nans = jnp.any(jnp.isnan(residuals))
    #if(nans):
    #jax.debug.print(residuals)
    return b_fit*e_fit+b_sparse*e_sparse+b_prior*e_prior

@partial(jax.jit, static_argnames=['n_full'])
def map_to_original(w, n_full, idxs):
    n_fitted = w.size
    w_out = jnp.zeros(n_full, dtype=jnp.float64)
    w_out = w_out.at[idxs].set(w)
    #for i in range(n_fitted):
    #    w_out = w_out.at[idxs[i]].set(w[i])
    return w_out

class ExpressionFitting:
    def __init__(self, camera_matrix, bs_mapper=None,
                 bs_to_ignore = ['cheekPuff_L', 'cheekPuff_R', 'eyeLookDown_L', 'eyeLookDown_R',
                                 'eyeLookIn_L', 'eyeLookIn_R', 'eyeLookOut_L', 'eyeLookOut_R',
                                 'eyeLookUp_L', 'eyeLookUp_R']) -> None:
                 #bs_to_ignore = ['cheekPuff_L', 'cheekPuff_R']) -> None:
        self.cam_mat = jax.device_put(jnp.array(camera_matrix, dtype=jnp.float64))

        self.bs_mapper = bs_mapper

        ICT_Model = ICTFaceModel68.from_pkl("./common/ICTFaceModel.pkl", load_blendshapes=True)

        self.right_contour_idx = list(range(0,8))
        self.chin_idx = 8
        self.left_contour_idx = list(range(9,17))

        self.n_bs = ICT_Model.n_blendshapes
        self.neutral_sparse_verts = jax.device_put(jnp.array(ICT_Model.neutral_vertices, dtype=jnp.float64))
        self.bs_names = list(ICT_Model.bs_names)

        self.name_to_idx = {name: idx for idx, name in enumerate(self.bs_names)}

        sparse_bs_arr_full = jax.device_put(jnp.array(ICT_Model.get_blendshape_arr(), dtype=jnp.float64))

        fitted_bs_idxs_list = []
        for i, name in enumerate(self.bs_names):
            if name not in bs_to_ignore:
                fitted_bs_idxs_list.append(i)

        self.fitted_bs_idxs = jnp.array(fitted_bs_idxs_list)
        self.n_fitted_bs = self.fitted_bs_idxs.size
        self.sparse_bs_arr = sparse_bs_arr_full[self.fitted_bs_idxs,:,:]
        #removed_idxs = []
        #for bs_remove in bs_to_ignore:
        #    bs_idx = self.bs_names.index(bs_remove)
        #    self.bs_names.remove(bs_remove)
        #    del self.name_to_idx[bs_remove]
        #    self.sparse_bs_arr = np.delete(self.sparse_bs_arr, bs_idx, 0)
        #    removed_idxs.append(bs_idx)
        #    self.n_bs -= 1

        #self.removed_idxs = np.sort(np.array(removed_idxs))

        #idxs_to_insert = []
        #for i, idx in enumerate(self.removed_idxs):
        #    idxs_to_insert.append(idx-i)

        #self.idxs_to_insert = np.array(idxs_to_insert)

        self._jaxopt_lm = LevenbergMarquardt(residuals, damping_parameter=1e-03, stop_criterion='grad-l2-norm',
                                     tol=0.0001, xtol=0.0001, gtol=0.0001, solver='cholesky',
                                     #geodesic=True, verbose=False,
                                     jit=True)#, maxiter=500)#, implicit_diff=True, unroll='auto')
        self.lm = jax.jit(self._jaxopt_lm.run)

        self.scipy_lm = ScipyLeastSquares(fun=residuals, loss='linear',
                                          options={"ftol": 0.001, "xtol": 0.001, "gtol": 0.001, "max_nfev":100},
                                          method='lm', dtype=jnp.float64, jit=True).run

        self.scipy_bounded_lbfgs = ScipyBoundedMinimize(method='L-BFGS-B', dtype=jnp.float64,
                                                        jit=True, fun=energy, tol=0.0001, maxiter=500).run
        self.former_w = jnp.zeros(self.n_fitted_bs)
        self.lower_bounds = jnp.zeros(self.n_fitted_bs)
        self.upper_bounds = jnp.ones(self.n_fitted_bs)

        non_fitting_params = jnp.array(self.pack_params(np.ones((68,2), dtype=np.float64), cv2.Rodrigues(np.array([0,0,0], dtype=np.float64))[0], np.ones(3,dtype=np.float64)), dtype=jnp.float64)
        self.lm(jnp.ones(self.n_fitted_bs, dtype=jnp.float64)*0.0001, non_fitting_params, self.cam_mat, self.former_w, self.neutral_sparse_verts, self.sparse_bs_arr, 1., 1., 1.)

    def fit(self, measured_points, rvec, tvec, b_fit, b_prior, b_sparse, method: str, debug = False):
        assert method in ['jaxopt_lm', 'scipy_lm', 'l-bfgs-b']

        rmat = cv2.Rodrigues(rvec)[0]
        #
        w_in = jnp.ones(self.n_fitted_bs, dtype=jnp.float64)*0.0001

        non_fitting_params = jnp.array(self.pack_params(measured_points, rmat, tvec), dtype=jnp.float64)
        #st = time.perf_counter()
        if method == 'jaxopt_lm':
            out = self.lm(w_in, non_fitting_params, self.cam_mat, self.former_w, self.neutral_sparse_verts, self.sparse_bs_arr, b_fit, b_prior, b_sparse)
        elif method == 'scipy_lm':
            out = self.scipy_lm(w_in, non_fitting_params, self.cam_mat, self.former_w, self.neutral_sparse_verts, self.sparse_bs_arr, b_fit, b_prior, b_sparse)
        elif method == 'l-bfgs-b':
            bounds = (self.lower_bounds, self.upper_bounds)
            out = self.scipy_bounded_lbfgs(w_in, bounds, non_fitting_params, self.cam_mat, self.former_w, self.neutral_sparse_verts, self.sparse_bs_arr, b_fit, b_prior, b_sparse)
        #print("=========",time.perf_counter() - st)
        #w_out = jax.device_get(out.params)
        w_out = jnp.clip(out.params, 0, 1)
        #w_out = out.params

        has_nan = jnp.any(jnp.isnan(w_out))
        if has_nan:
            w_out = w_in
            print("Fitting failed")
        else:
            self.former_w = w_out.copy()

        #n_bs_removed = self.idxs_to_insert.size
        #if n_bs_removed > 0:
        #    w_out = jnp.insert(w_out, self.idxs_to_insert, 0.)
        if self.n_fitted_bs < self.n_bs:
            #w_full = jnp.zeros(self.n_bs, dtype=jnp.float32)
            #w_full.at[self.fitted_bs_idxs].set(w_out)
            #print(self.fitted_bs_idxs)
            #w_out = w_full
            w_out = map_to_original(w_out, self.n_bs, self.fitted_bs_idxs)

        if debug:
            return self.post_process(w_in, w_out), jax.device_get(w_out)
        else:
            return self.post_process(w_in, w_out)

    def post_process(self, w_in, w_out):

        if self.bs_mapper is not None:
            return jdlp.to_dlpack(self.bs_mapper(w_out), take_ownership=True)
        else:
            return jdlp.to_dlpack(w_out.copy(), take_ownership=True)

    def pack_params(self, measured_points, rmat, tvec):
        fitting = np.zeros((68+3+1, 3), dtype=np.float64)

        fitting[0:68, 0:2]  = measured_points[:,:]
        fitting[68:68+3, :] = rmat[:,:]
        fitting[68+3]       = tvec.reshape(-1)[:]

        return fitting


from ..utils import Object

class Notification(Object):
    """
    Contains information about a notification

    Attributes:
        ID (:obj:`str`): ``Notification``

    Args:
        id (:obj:`int`):
            Unique persistent identifier of this notification
        date (:obj:`int`):
            Notification date
        is_silent (:obj:`bool`):
            True, if the notification was explicitly sent without sound
        type (:class:`telegram.api.types.NotificationType`):
            Notification type

    Returns:
        Notification

    Raises:
        :class:`telegram.Error`
    """
    ID = "notification"

    def __init__(self, id, date, is_silent, type, **kwargs):

        self.id = id  # int
        self.date = date  # int
        self.is_silent = is_silent  # bool
        self.type = type  # NotificationType

    @staticmethod
    def read(q: dict, *args) -> "Notification":
        id = q.get('id')
        date = q.get('date')
        is_silent = q.get('is_silent')
        type = Object.read(q.get('type'))
        return Notification(id, date, is_silent, type)


import numpy as np
from typing import *
import scipy
from loompy import timestamp

class MemoryLoomLayer():
	"""
	A layer residing in memory (without a corresponding layer on disk), typically
	as part of a :class:`loompy.LoomView`. MemoryLoomLayer supports a subset of
	the operations suported for regular layers.
	"""
	def __init__(self, name: str, matrix: np.ndarray) -> None:
		self.name = name  #: Name of the layer
		self.shape = matrix.shape  #: Shape of the layer
		self.values = matrix

	def __getitem__(self, slice: Tuple[Union[int, slice], Union[int, slice]]) -> np.ndarray:
		return self.values[slice]

	def __setitem__(self, slice: Tuple[Union[int, slice], Union[int, slice]], data: np.ndarray) -> None:
		self.values[slice] = data

	def sparse(self, rows: np.ndarray, cols: np.ndarray) -> scipy.sparse.coo_matrix:
		"""
		Return the layer as :class:`scipy.sparse.coo_matrix`
		"""
		return scipy.sparse.coo_matrix(self.values[rows, :][:, cols])

	def permute(self, ordering: np.ndarray, *, axis: int) -> None:
		"""
		Permute the layer along an axis

		Args:
			axis: The axis to permute (0, permute the rows; 1, permute the columns)
			ordering: The permutation vector
		"""
		if axis == 0:
			self.values = self.values[ordering, :]
		elif axis == 1:
			self.values = self.values[:, ordering]
		else:
			raise ValueError("axis must be 0 or 1")

class LoomLayer():
	"""
	Represents a layer (matrix) of values in the loom file, which can be accessed by slicing.
	"""

	def __init__(self, name: str, ds: Any) -> None:
		self.ds = ds  #: The :class:`.LoomConnection` object this layer belongs to
		self.name = name  #: Name of the layer (str)
		self.shape = ds.shape  #: Shape of the layer, tuple of (n_rows, n_cols)
		self.dtype = ""  #: Datatype of the layer (str)
		if name == "":
			self.dtype = self.ds._file["/matrix"].dtype
		else:
			self.dtype = self.ds._file["/layers/" + self.name].dtype

	def last_modified(self) -> str:
		"""
		Return a compact ISO8601 timestamp (UTC timezone) indicating when the file was last modified

		Note: if the layer does not contain a timestamp, and the mode is 'r+', a new timestamp will be set and returned.
		Otherwise, the current time in UTC will be returned.
		"""
		if self.name == "":
			if "last_modified" in self.ds._file["/matrix"].attrs:
				return self.ds._file["/matrix"].attrs["last_modified"]
			elif self.ds._file.mode == 'r+':
				self.ds._file["/matrix"].attrs["last_modified"] = timestamp()
				self.ds._file.flush()
				return self.ds._file["/matrix"].attrs["last_modified"]

		if self.name != "":
			if "last_modified" in self.ds._file["/layers/" + self.name].attrs:
				return self.ds._file["/layers/" + self.name].attrs["last_modified"]
			elif self.ds._file.mode == 'r+':
				self.ds._file["/layers/" + self.name].attrs["last_modified"] = timestamp()
				self.ds._file.flush()
				return self.ds._file["/layers/" + self.name].attrs["last_modified"]

		return timestamp()

	def __getitem__(self, slice: Tuple[Union[int, slice], Union[int, slice]]) -> np.ndarray:
		if self.name == "":
			return self.ds._file['/matrix'].__getitem__(slice)
		return self.ds._file['/layers/' + self.name].__getitem__(slice)

	def __setitem__(self, slice: Tuple[Union[int, slice], Union[int, slice]], data: np.ndarray) -> None:
		if self.name == "":
			self.ds._file['/matrix'][slice] = data
			self.ds._file["/matrix"].attrs["last_modified"] = timestamp()
			self.ds._file.attrs["last_modified"] = timestamp()
			self.ds._file.flush()
		else:
			self.ds._file['/layers/' + self.name][slice] = data
			self.ds._file["/layers/" + self.name].attrs["last_modified"] = timestamp()
			self.ds._file.attrs["last_modified"] = timestamp()
			self.ds._file.flush()

	def sparse(self, rows: np.ndarray = None, cols: np.ndarray = None, dtype = None) -> scipy.sparse.coo_matrix:
		if rows is not None:
			if np.issubdtype(rows.dtype, np.bool_):
				rows = np.where(rows)[0]
		if cols is not None:
			if np.issubdtype(cols.dtype, np.bool_):
				cols = np.where(cols)[0]

		n_genes = self.ds.shape[0] if rows is None else rows.shape[0]
		n_cells = self.ds.shape[1] if cols is None else cols.shape[0]
		# Calculate sparse data length to be able to reserve proper sized arrays beforehand
		nnonzero = 0
		for (ix, selection, view) in self.ds.scan(items=cols, axis=1, layers=[self.name], what=["layers"], batch_size=4096):
			if rows is not None:
				vals = view.layers[self.name][rows, :]
			else:
				vals = view.layers[self.name][:, :]
			nnonzero += np.count_nonzero(vals)
		data = np.empty((nnonzero,), dtype=dtype) #(data: List[np.ndarray] = []
		row = np.empty((nnonzero,), dtype=('uint16' if self.ds.shape[0] < 2**16 else 'uint32')) #row : List[np.ndarray] = []
		col = np.empty((nnonzero,), dtype=('uint32' if self.ds.shape[1] < 2**32 else 'uint64')) #col: List[np.ndarray] = []
		i = 0
		ci = 0
		for (ix, selection, view) in self.ds.scan(items=cols, axis=1, layers=[self.name], what=["layers"], batch_size=4096):
			if rows is not None:
				vals = view.layers[self.name][rows, :]
			else:
				vals = view.layers[self.name][:, :]
			if dtype:
				vals = vals.astype(dtype)
			nonzeros = np.where(vals != 0)
			n = len(nonzeros[0])
			data[ci:ci+n] = vals[nonzeros] #data.append(vals[nonzeros])
			row[ci:ci+n] = nonzeros[0] #row.append(nonzeros[0])
			col[ci:ci+n] = (nonzeros[1]+i) #col.append(nonzeros[1] + i)
			ci += n
			i += selection.shape[0]
		return scipy.sparse.coo_matrix((data, (row, col)), shape=(n_genes, n_cells), dtype=dtype)
		#return scipy.sparse.coo_matrix((np.concatenate(data, dtype=dtype), (np.concatenate(row), np.concatenate(col))), shape=(n_genes, n_cells), dtype=dtype)

	def _resize(self, size: Tuple[int, int], axis: int = None) -> None:
		"""Resize the dataset, or the specified axis.

		The dataset must be stored in chunked format; it can be resized up to the "maximum shape" (keyword maxshape) specified at creation time.
		The rank of the dataset cannot be changed.
		"Size" should be a shape tuple, or if an axis is specified, an integer.

		BEWARE: This functions differently than the NumPy resize() method!
		The data is not "reshuffled" to fit in the new shape; each axis is grown or shrunk independently.
		The coordinates of existing data are fixed.
		"""
		if self.name == "":
			self.ds._file['/matrix'].resize(size, axis)
		else:
			self.ds._file['/layers/' + self.name].resize(size, axis)

	def map(self, f_list: List[Callable[[np.ndarray], int]], axis: int = 0, chunksize: int = 1000, selection: np.ndarray = None) -> List[np.ndarray]:
		"""
		Apply a function along an axis without loading the entire dataset in memory.

		Args:
			f_list (list of func):		Function(s) that takes a numpy ndarray as argument

			axis (int):		Axis along which to apply the function (0 = rows, 1 = columns)

			chunksize (int): Number of rows (columns) to load per chunk

			selection (array of bool): Columns (rows) to include

		Returns:
			numpy.ndarray result of function application

			If you supply a list of functions, the result will be a list of numpy arrays. This is more
			efficient than repeatedly calling map() one function at a time.
		"""
		if hasattr(f_list, '__call__'):
			raise ValueError("f_list must be a list of functions, not a function itself")

		result = []
		if axis == 0:
			rows_per_chunk = chunksize
			for i in range(len(f_list)):
				result.append(np.zeros(self.shape[0]))
			ix = 0
			while ix < self.shape[0]:
				rows_per_chunk = min(self.shape[0] - ix, rows_per_chunk)
				if selection is not None:
					chunk = self[ix:ix + rows_per_chunk, :][:, selection]
				else:
					chunk = self[ix:ix + rows_per_chunk, :]
				for i in range(len(f_list)):
					result[i][ix:ix + rows_per_chunk] = np.apply_along_axis(f_list[i], 1, chunk)
				ix = ix + rows_per_chunk
		elif axis == 1:
			cols_per_chunk = chunksize
			for i in range(len(f_list)):
				result.append(np.zeros(self.shape[1]))
			ix = 0
			while ix < self.shape[1]:
				cols_per_chunk = min(self.shape[1] - ix, cols_per_chunk)
				if selection is not None:
					chunk = self[:, ix:ix + cols_per_chunk][selection, :]
				else:
					chunk = self[:, ix:ix + cols_per_chunk]
				for i in range(len(f_list)):
					result[i][ix:ix + cols_per_chunk] = np.apply_along_axis(f_list[i], 0, chunk)
				ix = ix + cols_per_chunk
		return result

	def _permute(self, ordering: np.ndarray, *, axis: int) -> None:
		if self.name == "":
			obj = self.ds._file['/matrix']
		else:
			obj = self.ds._file['/layers/' + self.name]
		if axis == 0:
			chunksize = 5000
			start = 0
			while start < self.shape[1]:
				submatrix = obj[:, start:start + chunksize]
				obj[:, start:start + chunksize] = submatrix[ordering, :]
				start = start + chunksize
		elif axis == 1:
			chunksize = 100000000 // self.shape[1]
			start = 0
			while start < self.shape[0]:
				submatrix = obj[start:start + chunksize, :]
				obj[start:start + chunksize, :] = submatrix[:, ordering]
				start = start + chunksize
		else:
			raise ValueError("axis must be 0 or 1")


from django import forms
from django.template.defaultfilters import force_escape
from baruwa.utils.regex import DOM_RE
import datetime, time
try:
    from django.forms.fields import email_re
    from django.forms.fields import ipv4_re
except ImportError:
    from django.core.validators import email_re
    from django.core.validators import ipv4_re

FILTER_ITEMS = (
    ('id','Message ID'),
    ('size','Size'),
    ('from_address','From Address'),
    ('from_domain', 'From Domain'),
    ('to_address','To Address'),
    ('to_domain','To Domain'),
    ('subject','Subject'),
    ('clientip','Received from'),
#    ('archive','Is Archived'),
    ('scaned', 'Was scanned'),
    ('spam','Is spam'),
    ('highspam','Is high spam'),
    ('saspam','Is SA spam'),
    ('rblspam','Is RBL listed'),
    ('whitelisted','Is whitelisted'),
    ('blacklisted','Is blacklisted'),
    ('sascore','SA score'),
    ('spamreport','Spam report'),
    ('virusinfected','Is virus infected'),
    ('nameinfected','Is name infected'),
    ('otherinfected','Is other infected'),
    ('date','Date'),
    ('time','Time'),
    ('headers','Headers'),
    ('isquarantined','Is quarantined'),
)

FILTER_BY = (
    (1,'is equal to'),
    (2,'is not equal to'),
    (3,'is greater than'),
    (4,'is less than'),
    (5,'contains'),
    (6,'does not contain'),
    (7,'matches regex'),
    (8,'does not match regex'),
    (9,'is null'),
    (10,'is not null'),
    (11,'is true'),
    (12,'is false'),
)

EMPTY_VALUES = (None, '')

BOOL_FIELDS = ["scaned", "spam", "highspam", "saspam", "rblspam",
    "whitelisted", "blacklisted", "virusinfected", "nameinfected",
    "otherinfected", "isquarantined"]
TEXT_FIELDS = ["id", "from_address", "from_domain", "to_address",
    "to_domain", "subject", "clientip", "spamreport", "headers"]
TIME_FIELDS = ["date","time"]
NUM_FIELDS = ["size", "sascore"]

BOOL_FILTER = [11, 12]
NUM_FILTER = [1, 2, 3, 4]
TEXT_FILTER = [1, 2, 5, 6, 7, 8, 9, 10]
TIME_FILTER = [1, 2, 3, 4]

def isnumeric(value):
    "Validate numeric values"
    return str(value).replace(".", "").replace("-", "").isdigit()

def to_dict(tuple_list):
    "Convert tuple to dictionary"
    dic = {}
    for val in tuple_list:
        dic[val[0]] = val[1]
    return dic

class FilterForm(forms.Form):
    "Filters form"
    filtered_field = forms.ChoiceField(choices=FILTER_ITEMS)
    filtered_by = forms.ChoiceField(choices=FILTER_BY)
    filtered_value = forms.CharField(required=False)

    def clean(self):
        "validate the form"
        cleaned_data = self.cleaned_data
        submited_field = cleaned_data.get('filtered_field')
        submited_by = int(cleaned_data.get('filtered_by'))
        submited_value = cleaned_data.get('filtered_value')
        if submited_by != 0:
            sbi = (submited_by - 1)
        else:
            sbi = submited_by

        if submited_field in BOOL_FIELDS:
            if not submited_by in BOOL_FILTER:
                filter_items = to_dict(list(FILTER_ITEMS))
                error_msg = "%s does not support the %s filter" % (
                    filter_items[submited_field],FILTER_BY[sbi][1])
                raise forms.ValidationError(error_msg)
        if submited_field in NUM_FIELDS:
            if not submited_by in NUM_FILTER:
                filter_items = to_dict(list(FILTER_ITEMS))
                error_msg = "%s does not support the %s filter" % (
                    filter_items[submited_field],FILTER_BY[sbi][1])
                raise forms.ValidationError(error_msg)
            if submited_value in EMPTY_VALUES:
                raise forms.ValidationError("Please supply a value to query")
            if not isnumeric(submited_value):
                raise forms.ValidationError("The value has to be numeric")
        if submited_field in TEXT_FIELDS:
            if not submited_by in TEXT_FILTER:
                filter_items = to_dict(list(FILTER_ITEMS))
                error_msg = "%s does not support the %s filter" % (
                    filter_items[submited_field],FILTER_BY[sbi][1])
                raise forms.ValidationError(error_msg)
            if submited_value in EMPTY_VALUES:
                raise forms.ValidationError("Please supply a value to query")
            if ( FILTER_BY[sbi][1] == 'is equal to' or FILTER_BY[sbi][1] == 'is not equal to' ):
                if (submited_field == 'from_address') or (
                    submited_field == 'to_address'):
                    if not email_re.match(submited_value.strip()):
                        raise forms.ValidationError(
                            '%s is not a valid e-mail address.'
                            % force_escape(submited_value))
                if (submited_field == 'from_domain') or (
                    submited_field == 'to_domain'):
                    if not DOM_RE.match(submited_value.strip()):
                        raise forms.ValidationError(
                            'Please provide a valid domain name')
                if submited_field == 'clientip':
                    if not ipv4_re.match(submited_value.strip()):
                        raise forms.ValidationError(
                            'Please provide a valid ipv4 address')
        if submited_field in TIME_FIELDS:
            if not submited_by in TIME_FILTER:
                filter_items = to_dict(list(FILTER_ITEMS))
                error_msg = "%s does not support the %s filter" % (
                    filter_items[submited_field],FILTER_BY[sbi][1])
                raise forms.ValidationError(error_msg)
            if submited_value in EMPTY_VALUES:
                raise forms.ValidationError("Please supply a value to query")
            if submited_field == 'date':
                try:
                    datetime.date(
                        *time.strptime(submited_value, '%Y-%m-%d')[:3])
                except ValueError:
                    raise forms.ValidationError(
                        'Please provide a valid date in YYYY-MM-DD format')
            if submited_field == 'time':
                try:
                    datetime.time(*time.strptime(submited_value, '%H:%M')[3:6])
                except ValueError:
                    raise forms.ValidationError(
                        'Please provide valid time in HH:MM format')

        return cleaned_data


import pandas as pd
import tweepy
from datetime import datetime

# twitter api credentials
ACCESS_TOKEN = 'XXXXXX'
ACCESS_SECRET = 'XXXXX'
CONSUMER_KEY = 'XXXXX'
CONSUMER_SECRET = 'XXXXX'

# Setup access to API
def connect_to_twitter_OAuth():
    auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
    auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)

    api = tweepy.API(auth, wait_on_rate_limit = True)
    return api

limit = 500
# Create API object
api = connect_to_twitter_OAuth()

print("try again")
tweets = []
# scraping 2000 tweets
for i in range(100):
    tweets.extend(api.user_timeline(
        screen_name = '@WilliamsRuto',
        include_rts = True,
        # keep full text
        tweet_mode = 'extended'

    ))

tweet_list = []
for tweet in tweets:
    text = tweet._json['full_text']

    refined_tweet = {
        'user': tweet.user.screen_name,
        'text' : text,
        'favorite_count': tweet.favorite_count,
        'retweet_count' : tweet.retweet_count,
        'create_at': tweet.created_at,
    }

    tweet_list.append(refined_tweet)

df = pd.DataFrame(tweet_list)

df.to_csv('william_ruto_tweets.csv')
print("done")

william = pd.read_csv("william_ruto_tweets.csv")
print(william.shape)


from ecsController import ecsController
import unittest
import logging

REGION = 'eu-west-2'
SEARCHTAG = 'DEVDAY'

class test_deleteState(unittest.TestCase):

    def test_delete(self):
        logger.info("-----Testing delete State method------")

        ecs = ecsController(REGION, SEARCHTAG)
        clustermap  =ecs.loadState()
        orig = len(clustermap)
        self.assertTrue(orig>0, msg="database should not have been empty")

        logger.info(f"******LOADED MAP > {clustermap} ")
        ecs._deleteState(clustermap)
        nomap = ecs.loadState()
        items = len(nomap)
        self.assertEqual(items,0,msg="there should be no items in the database")

        logger.info("restoring the database state")
        ecs.storeState(clustermap)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    logger =logging.getLogger()
    unittest.main()


"""Create yaml files from bib files."""
import bibtexparser
from bibtexparser.bparser import BibTexParser
from bibtexparser.customization import author, convert_to_unicode
import yaml
import argparse
import os

def format_authors(list_of_authors, special_author):
    authors = ""
    num_authors = len(list_of_authors)
    for idx, author in enumerate(list_of_authors):
        last, first = author.split(", ")
        initials = "".join([x[0] + ". " for x in first.split(" ")])
        if idx <= (num_authors - 3):
            joining_string = ", "
        elif idx == (num_authors - 2):
            joining_string = " \& "
        else:
            joining_string = ""
        author_name = initials + last
        if author_name == special_author:
            author_name = r"\underline{" + author_name + "}"
        authors += author_name + joining_string
    return authors

def customizations(record):
    record = author(record)
    record = convert_to_unicode(record)
    return record

def format_entries(entries_dict, special_author):
    for k in entries_dict.keys():
        if "collaboration" not in entries_dict[k]:
            entries_dict[k]["author"] = format_authors(
                entries_dict[k]["author"], special_author)

def get_publication_dict_from_bib(bibfile, special_author):
    bib_file = open(bibfile, "r")
    parser = BibTexParser()
    parser.customization = customizations
    bib_database = bibtexparser.load(bib_file, parser=parser)
    bib_file.close()
    entries_dict = bib_database.entries_dict
    format_entries(entries_dict, special_author)
    return entries_dict

def get_publication_list_from_dict(pub_dict):
    publist = "\\begin{enumerate}\n"
    for k in pub_dict:
        d = data[k]
        if "collaboration" in d:
            d["author"] = d["collaboration"]
        p = "\\item "
        p += d["author"] + ", " + "``" + d["title"] + "\"" + ", "
        if "journal" in d:
            p += "\href{" + "https://doi.org/" + d["doi"] + "}{" + d["journal"] + "}" + ", "
            p += "{\\bfseries " + d["volume"] + "}" + ", " + d["pages"] + ", "
        p += "(" + d["year"] + "), "
        p += "\href{" + "https://arxiv.org/abs/" + d["eprint"] + "}{arXiv:" + d["eprint"] + " [" + d["primaryclass"] +"]}" + ", "
        publist += "\\end{enumerate}\n"


from numba import njit
import numpy as np
from vectorbt.portfolio import nb as portfolio_nb
from vectorbt.portfolio.enums import SizeType

@njit
def order_func_nb(c, size, price, commperc, slippage):
    """Place an order (= element within group and row)."""
    # Get column index within group (if group starts at column 58 and current column is 59,
    # the column within group is 1, which can be used to get size)
    group_col = c.col - c.from_col
    return portfolio_nb.order_nb(
        size=size[group_col],
        price=price[c.i, c.col],
        size_type=SizeType.TargetAmount,
        fees=commperc,
        slippage=slippage,
        lock_cash=False,
        log=True,
    )


from bs4 import BeautifulSoup
import cfscrape
import shutil
import sys
import time
from random import randrange

def randomurlnums():
    urlnums = randrange(1,9999)
    if urlnums < 1000:
        while len(str(urlnums)) < 4:
            urlnums = '0'+str(urlnums)
    else:
        urlnums= str(urlnums)
    return urlnums
def randomurlletters():
    alphabet = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']
    first = alphabet[randrange(25)]
    second = alphabet[randrange(25)]
    letters = first+second
    return letters

# I sleep inside here because I got an ip banned by cloud flare )=

#this function just takes a url that links to a .png file and saves it in the same directory with given filename
#we have to do this through an http request because of cloudflare
def urlImg2File(url,filename):
    time.sleep(0.1)
    response = scraper.get(url,stream=True)
    #this is where the writing to file happens idk the specifics stolen from stack overflow
    with open(f'{filename}.png','wb') as out_file:
        shutil.copyfileobj(response.raw,out_file)
    del response
random = False
starturl = 'https://prnt.sc/'
if len(sys.argv) == 2 or len(sys.argv) == 1:
    print('random url')
    random = True
    urlnums = 0

    if len(sys.argv)==1:
        stop = 100
    else:
        stop = int(sys.argv[1])

else:

    letters = sys.argv[1]

    urlnums = sys.argv[2]

    #each time we run it goes for 100
    stop = int(urlnums) + int(sys.argv[3])

if(stop>=10000):
    print('start number too high for the amount of pictures you want')
    quit()
#The scraper has to emulate a browser so we have to create an instance of a nodejs server to do that cfscrape allows that and here
#the fake browser is created
scraper = cfscrape.create_scraper()

while int(urlnums) < stop:
    if not random:
        url = starturl + letters + urlnums
    else:
        url = starturl + randomurlletters() + randomurlnums()
    print(url)
    #sleep get url grab data feed it into beautiful soup so we can iterate over it better
    time.sleep(0.1)
    page = scraper.get(url)
    data = page.text

    soup = BeautifulSoup(data, 'html5lib')

    image = None
    #for every link we find with the img tag
    for link in soup.find_all('img'):
        #we check if starts with https://image which was gotten through source inspection of prnt.sc
        tempurl = str(link.get('src'))
        if tempurl.startswith('https://image'):
            #increment url number to bruteforce look
            urlnums = str(int(urlnums) + 1)
            #we save our found png with a file name so we can get back to it from the url.
            if not random:
                urlImg2File(tempurl,letters+urlnums)
            else:
                urlImg2File(tempurl,url[16:])
            break
    else:
        urlnums = str(int(urlnums) + 1)
        if random:
            stop+=1
    continue


import numpy as np
import pandas as pd
from sklearn import *
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import accuracy_score
from reportlab import platypus
from reportlab.lib.styles import ParagraphStyle as PS
from reportlab.platypus import SimpleDocTemplate

import sqlite3
con = sqlite3.connect('concrete1')
training_data = np.genfromtxt('conc.csv', delimiter=',')
inputs = training_data[:,:-1]
outputs = training_data[:, -1]
training_inputs = inputs[:1000]
training_outputs = outputs[:1000]
testing_inputs = inputs[1000:]
testing_outputs = outputs[1000:]
classifier = DecisionTreeClassifier()
classifier.fit(training_inputs, training_outputs)
with con:
  cur = con.cursor()
  cur.execute('SELECT * FROM mainingr');
  result1 = cur.fetchall()
  for row in result1:
    s2 = int(row[0])
    s3 = int(row[1])
    s4 = int(row[2])
    s5 = int(row[3])
  cur.execute('SELECT * FROM otheringr');
  result2 = cur.fetchall()
  for row in result2:
    s1 = float(row[0])
    s6 = float(row[1])
    s7 = float(row[2])
    s8 = int(row[3])
result = str(" ")
testSet = [[s1,s2,s3,s4,s5,s6,s7,s8]]
test = pd.DataFrame(testSet)
predictions = classifier.predict(test)
if (int(predictions) ==0):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 1 to 10 MPa')+ "
"
elif (int(predictions) ==1):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 11 to 20 MPa')+ "
"
elif (int(predictions) ==2):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 21 to 30 MPa')+ "
"
elif (int(predictions) ==3):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 31 to 40 MPa')+ "
"
elif (int(predictions) ==4):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 41 to 50 MPa')+ "
"
elif (int(predictions) ==5):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 51 to 60 MPa')+ "
"
elif (int(predictions) ==6):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 61 to 70 MPa')+ "
"
elif (int(predictions) ==7):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 71 to 80 MPa')+ "
"
elif (int(predictions) ==8):
   result = result + str('Predicted compressive strength range of the given concrete mix is: 81 to 90 MPa')+ "
"
else:
   result = result + str('Predicted compressive strength range of the given concrete mix is: 91 to 100 MPa')+ "
"
items = []
items.append(platypus.Paragraph(result,PS('body')))
doc = SimpleDocTemplate('concrep.pdf')
doc.multiBuild(items)
print('Concrete Compressive Strength Report Successfully Generated')


import enum
import logging

from aiohttp import web

from aiohttp.web_exceptions import (
    HTTPBadRequest, HTTPClientError
)
from aiohttp_swagger3.swagger_route import RequestValidationFailed

@enum.unique
class ApiErrorCode(enum.Enum):
    validation_error = 'validation_error'
    schema_validation_error = 'schema_validation_error'
    db_error = 'db_error'
    server_error = 'server_error'
    logical_error = 'logical_error'

class ApiBaseError(Exception):
    def __init__(self, error_code: ApiErrorCode, error_message: str, context: dict, http_status_code: int):
        self.status = 'error'
        self.error_code = error_code
        self.error_message = error_message
        if context is None:
            context = {}
        self.context = context
        self.http_status_code = http_status_code

    def as_dict(self) -> dict:
        data = {
            'status': self.status,
            'error_code': str(self.error_code.name),
            'error_message': self.error_message,
            'context': self.context
        }
        return data

@web.middleware
async def error_middleware(request: web.Request, handler) -> web.StreamResponse:
    try:
        return await handler(request)
    except RequestValidationFailed as e:
        err = ApiValidationError(
            error_message="Invalid params in body",
            error_code=ApiErrorCode.schema_validation_error,
            context=e.errors
        )
        return web.json_response(status=err.http_status_code, data=err.as_dict())
    except ApiBaseError as err:
        return web.json_response(status=err.http_status_code, data=err.as_dict())
    except HTTPBadRequest as e:
        err = ApiValidationError(
            error_message=str(e),
            context={'reason': e.reason}
        )
        return web.json_response(status=err.http_status_code, data=err.as_dict())
    except HTTPClientError:
        raise

    except Exception as e:
        logging.exception(f"Internal server error. Error: {e}")
        err = ApiInternalError(
            error_message='Internal server error',
            context={'Exception': str(e)}
        )
        return web.json_response(status=err.http_status_code, data=err.as_dict())

class ApiValidationError(ApiBaseError):
    def __init__(self, error_message: str, error_code: ApiErrorCode = ApiErrorCode.validation_error, context: dict=None):
        super(ApiValidationError, self).__init__(error_code, error_message, context, 400)

class ApiInternalError(ApiBaseError):
    def __init__(self, error_message: str, error_code: ApiErrorCode = ApiErrorCode.server_error, context: dict=None):
        super(ApiInternalError, self).__init__(error_code, error_message, context, 500)

class ApiLogicalError(ApiBaseError):
    def __init__(self, error_message: str, error_code: ApiErrorCode = ApiErrorCode.logical_error, context: dict=None):
        super(ApiLogicalError, self).__init__(error_code, error_message, context, 420)


import json
from datetime import timedelta

import structlog
from celery import shared_task
from django.apps import apps

from .celery import jbi_parse_buildinfo
from .celery import jbi_parse_hotfix
from .celery import process_result
from .conf import settings
from .utils import is_download_artifacts
from .utils import jenkins_get_artifact
from .utils import jenkins_get_build
from .utils import jenkins_get_console
from .utils import jenkins_get_env
from .utils import jenkins_remove_project_ppa
from tracker.conf import Tracker

logger = structlog.get_logger(__name__)

@shared_task(bind=True)
def jenkins_remove_project(self, jbi_id):
    JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
    jbi = JenkinsBuildInfo.objects.get(id=jbi_id)
    structlog.contextvars.bind_contextvars(
        jbi=str(jbi),
        result=jbi.result,
        gerrit_eventtype=jbi.gerrit_eventtype,
    )
    if (
        jbi.jobname.endswith("-repos")
        and jbi.result == "SUCCESS"
        and jbi.gerrit_eventtype == "change-merged"
    ):
        try:
            jenkins_remove_project_ppa(jbi.param_ppa, jbi.source)
            logger.info("triggered job for removal")
        except FileNotFoundError as exc:
            logger.warn("source is not there yet, try again in 60 secs")
            raise self.retry(exc=exc, countdown=60)
    else:
        logger.info("skip removal")

@shared_task(ignore_result=True)
def jbi_get_artifact(jbi_id, jobname, buildnumber, artifact_info):
    path = jenkins_get_artifact(jobname, buildnumber, artifact_info)
    if path.name == settings.HOTFIX_ARTIFACT:
        if settings.TRACKER_PROVIDER == Tracker.NONE:
            logger.info("no tracker defined, skip hotfix management")
            return
        jbi_parse_hotfix.delay(jbi_id, str(path))

@shared_task(ignore_result=True)
def get_jbi_files(jbi_id, jobname, buildnumber):
    jenkins_get_console(jobname, buildnumber)
    path_envVars = jenkins_get_env(jobname, buildnumber)
    path_build = jenkins_get_build(jobname, buildnumber)
    jbi_parse_buildinfo.delay(jbi_id, str(path_build))
    if is_download_artifacts(jobname):
        with open(path_build) as data_file:
            data = json.load(data_file)
        logger.debug("job_info", data=data)
        for artifact in data["artifacts"]:
            jbi_get_artifact.delay(jbi_id, jobname, buildnumber, artifact)
    else:
        logger.debug("skip artifacts download")
    if jobname in settings.RELEASE_CHANGED_JOBS:
        process_result.delay(jbi_id, str(path_envVars))

@shared_task(ignore_result=True)
def jbi_purge(release, weeks):
    JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
    JenkinsBuildInfo.objects.purge_release(release, timedelta(weeks=weeks))
    logger.info(f"purged release {release} jbi older than {weeks} weeks")


from collections import defaultdict, namedtuple
from typing import List

import SCons
from SCons.Tool import install

ALIAS_MAP = "AIB_ALIAS_MAP"
BASE_COMPONENT = "AIB_BASE_COMPONENT"
BASE_ROLE = "AIB_BASE_ROLE"
COMPONENT = "AIB_COMPONENT"
REVERSE_COMPONENT_DEPENDENCIES = "AIB_COMPONENTS_EXTRA"
DEFAULT_COMPONENT = "AIB_DEFAULT_COMPONENT"
INSTALLED_FILES = "AIB_INSTALLED_FILES"
META_COMPONENT = "AIB_META_COMPONENT"
META_ROLE = "AIB_META_ROLE"
ROLE = "AIB_ROLE"
ROLE_DECLARATIONS = "AIB_ROLE_DECLARATIONS"
SUFFIX_MAP = "AIB_SUFFIX_MAP"
TASKS = "AIB_TASKS"

SuffixMap = namedtuple(
    "SuffixMap",
    ["directory", "default_role"],
)

class RoleInfo:
    """A component/role union Node."""

    def __init__(self, component, role, files=None, dependencies=None):
        self.id = "{}-{}".format(component, role)
        self.component = component
        self.role = role
        if files is None:
            self.files = set()
        else:
            self.files = set(files)

        if dependencies is None:
            self.dependencies = set()
        else:
            self.dependencies = set(dependencies)

    def __str__(self):
        return "RoleInfo({})".format(self.id)

    def __repr__(self):
        return self.__str__()

class DeclaredRole:
    def __init__(self, name, dependencies=None, transitive=False, silent=False):
        self.name = name

        if dependencies is None:
            self.dependencies = set()
        else:
            self.dependencies = {dep for dep in dependencies if dep is not None}

        self.silent = silent

def declare_role(env, **kwargs):
    """Construct a new role declaration"""
    return DeclaredRole(**kwargs)

def declare_roles(env, roles, base_role=None, meta_role=None):
    """Given a list of role declarations, validate them and store them in the environment"""
    role_names = [role.name for role in roles]
    if len(role_names) != len(set(role_names)):
        raise Exception("Cannot declare duplicate roles")

    # Ensure that all roles named in dependency lists actually were
    # passed in as a role.
    for role in roles:
        for d in role.dependencies:
            if d not in role_names:
                raise Exception("Role dependency '{}' does not name a declared role".format(d))

    if isinstance(base_role, str):
        if base_role not in role_names:
            raise Exception(
                "A base_role argument was provided but it does not name a declared role")
    elif isinstance(base_role, DeclaredRole):
        if base_role not in roles:
            raise Exception("A base_role argument was provided but it is not a declared role")
    elif base_role is not None:
        raise Exception("The base_role argument must be a string name of a role or a role object")
    else:
        # Set it to something falsey
        base_role = str()

    if isinstance(meta_role, str):
        if meta_role not in role_names:
            raise Exception(
                "A meta_role argument was provided but it does not name a declared role")
    elif isinstance(meta_role, DeclaredRole):
        if meta_role not in roles:
            raise Exception("A meta_role argument was provided but it is not a declared role")
    elif meta_role is not None:
        raise Exception("The meta_role argument must be a string name of a role or a role object")
    else:
        # Set it to something falsy
        meta_role = str()

    silents = [role for role in roles if role.silent]
    if len(silents) > 1:
        raise Exception("No more than one role can be declared as silent")

    # If a base role was given, then add it as a dependency of every
    # role that isn't the base role (which would be circular).
    if base_role:
        for role in roles:
            if role.name != base_role:
                role.dependencies.add(base_role)

    # Become a dictionary, so we can look up roles easily.
    roles = {role.name: role for role in roles}

    # If a meta role was given, then add every role which isn't the
    # meta role as one of its dependencies.
    if meta_role:
        roles[meta_role].dependencies.update(r for r in roles.keys() if r != meta_role)

    # TODO: Check for DAG

    # TODO: What if base_role or meta_role is really None?
    env[BASE_ROLE] = base_role
    env[META_ROLE] = meta_role
    env[ROLE_DECLARATIONS] = roles

def generate_alias_name(env, component, role, task):
    """Generate a scons alias for the component and role combination"""
    return "{task}-{component}{role}".format(
        task=task,
        component=component,
        role="" if env[ROLE_DECLARATIONS][role].silent else "-" + role,
    )

def get_alias_map_entry(env, component, role):
    c_entry = env[ALIAS_MAP][component]

    try:
        return c_entry[role]
    except KeyError:
        r_entry = RoleInfo(component=component, role=role)
        c_entry[role] = r_entry

        declaration = env[ROLE_DECLARATIONS].get(role)
        for dep in declaration.dependencies:
            dep_entry = get_alias_map_entry(env, component, dep)
            r_entry.dependencies.add(dep_entry)

        meta_component = env.get(META_COMPONENT)
        if meta_component and component != meta_component:
            meta_c_entry = get_alias_map_entry(env, meta_component, role)
            meta_c_entry.dependencies.add(r_entry)

        base_component = env.get(BASE_COMPONENT)
        if base_component and component != base_component:
            base_c_entry = get_alias_map_entry(env, base_component, role)
            r_entry.dependencies.add(base_c_entry)

        meta_role = env.get(META_ROLE)
        if (meta_role and role != meta_role and meta_component and component != meta_component):
            meta_r_entry = get_alias_map_entry(env, component, meta_role)
            meta_c_r_entry = get_alias_map_entry(env, meta_component, meta_role)
            meta_c_r_entry.dependencies.add(meta_r_entry)

        return r_entry

def get_component(node):
    return getattr(node.attributes, COMPONENT, None)

def get_role(node):
    return getattr(node.attributes, ROLE, None)

def scan_for_transitive_install(node, env, _path):
    """Walk the children of node finding all installed dependencies of it."""
    component = get_component(node.sources[0])
    role = get_role(node.sources[0])
    if component is None:
        return []

    scanned = getattr(node.attributes, "AIB_SCANNED", None)
    if scanned is not None:
        return scanned

    # Access directly by keys because we don't want to accidentally
    # create a new entry via get_alias_map_entry and instead should
    # throw a KeyError if we got here without valid components and
    # roles
    alias_map = env[ALIAS_MAP]
    entry = alias_map[component][role]
    role_deps = env[ROLE_DECLARATIONS].get(role).dependencies
    results = set()

    # We have to explicitly look at the various BASE files here since it's not
    # guaranteed they'll be pulled in anywhere in our grandchildren but we need
    # to always depend upon them. For example if env.AutoInstall some file 'foo'
    # tagged as common base but it's never used as a source for the
    # AutoInstalled file we're looking at or the children of our children (and
    # so on) then 'foo' would never get scanned in here without this explicit
    # dependency adding.
    base_component = env.get(BASE_COMPONENT)
    if base_component and component != base_component:
        base_role_entry = alias_map[base_component][role]
        if base_role_entry.files:
            results.update(base_role_entry.files)

    base_role = env.get(BASE_ROLE)
    if base_role and role != base_role:
        component_base_entry = alias_map[component][base_role]
        if component_base_entry.files:
            results.update(component_base_entry.files)

    if (base_role and base_component and component != base_component and role != base_role):
        base_base_entry = alias_map[base_component][base_role]
        if base_base_entry.files:
            results.update(base_base_entry.files)

    installed_children = set(grandchild for child in node.children()
                             for direct_children in child.children()
                             for grandchild in direct_children.get_executor().get_all_targets()
                             if direct_children.get_executor() and grandchild.has_builder())

    for child in installed_children:
        auto_installed_files = get_auto_installed_files(env, child)
        if not auto_installed_files:
            continue

        child_role = get_role(child)
        if child_role == role or child_role in role_deps:
            child_component = get_component(child)
            child_entry = get_alias_map_entry(env, child_component, child_role)

            # This is where component inheritance happens. We need a default
            # component for everything so we can store it but if during
            # transitive scanning we see a child with the default component here
            # we will move that file to our component. This prevents
            # over-stepping the DAG bounds since the default component is likely
            # to be large and an explicitly tagged file is unlikely to depend on
            # everything in it.
            if child_component == env.get(DEFAULT_COMPONENT):
                setattr(node.attributes, COMPONENT, component)
                for f in auto_installed_files:
                    child_entry.files.discard(f)
                entry.files.update(auto_installed_files)
            elif component != child_component:
                entry.dependencies.add(child_entry)

            results.update(auto_installed_files)

    # Produce deterministic output for caching purposes
    results = sorted(results, key=str)
    setattr(node.attributes, "AIB_SCANNED", results)

    return results

def scan_for_transitive_install_pseudobuilder(env, node):
    return scan_for_transitive_install(node, env, None)

def tag_components(env, target, **kwargs):
    """Create component and role dependency objects"""
    target = env.Flatten([target])
    component = kwargs.get(COMPONENT)
    role = kwargs.get(ROLE)
    if component is not None and (not isinstance(component, str) or " " in component):
        raise Exception("AIB_COMPONENT must be a string and contain no whitespace.")

    if component is None:
        raise Exception("AIB_COMPONENT must be provided; untagged targets: {}".format(
            [t.path for t in target]))

    if role is None:
        raise Exception("AIB_ROLE was not provided.")

    for t in target:
        t.attributes.keep_targetinfo = 1
        setattr(t.attributes, COMPONENT, component)
        setattr(t.attributes, ROLE, role)

    entry = get_alias_map_entry(env, component, role)

    # We cannot wire back dependencies to any combination of meta role, meta
    # component or base component. These cause dependency cycles because
    # get_alias_map_entry will do that wiring for us then we will try to
    # map them back on themselves in our loop.
    if (component != env.get(BASE_COMPONENT) and role != env.get(META_ROLE)
            and component != env.get(META_COMPONENT)):
        for component in kwargs.get(REVERSE_COMPONENT_DEPENDENCIES, []):
            component_dep = get_alias_map_entry(env, component, role)
            component_dep.dependencies.add(entry)

    return entry

def auto_install_task(env, component, role):
    """Auto install task."""
    entry = get_alias_map_entry(env, component, role)
    return list(entry.files)

def auto_install_pseudobuilder(env, target, source, **kwargs):
    """Auto install pseudo-builder."""
    source = env.Flatten([source])
    source = [env.File(s) for s in source]
    entry = env.TagComponents(source, **kwargs)

    installed_files = []
    for s in source:

        target_for_source = target

        if not target_for_source:

            # AIB currently uses file suffixes to do mapping. However, sometimes we need
            # to do the mapping based on a different suffix. This is used for things like
            # dSYM files, where we really just want to describe where .dSYM bundles should
            # be placed, but need to actually handle the substructure. Currently, this is
            # only used by separate_debug.py.
            #
            # TODO: Find a way to do this without the tools needing to coordinate.
            suffix = getattr(s.attributes, "aib_effective_suffix", s.get_suffix())
            auto_install_mapping = env[SUFFIX_MAP].get(suffix)

            if not auto_install_mapping:
                raise Exception("No target provided and no auto install mapping found for:", str(s))

            target_for_source = auto_install_mapping.directory

        # We've already auto installed this file and it may have belonged to a
        # different role since it wouldn't get retagged above. So we just skip
        # this files since SCons will already wire the dependency since s is a
        # source and so the file will get installed. A common error here is
        # adding debug files to the runtime component file if we do not skip
        # this.
        existing_installed_files = get_auto_installed_files(env, s)
        if existing_installed_files:
            continue

        # We must do an early subst here so that the _aib_debugdir
        # generator has a chance to run while seeing 'source'. We need
        # to do two substs here.  The first is to expand an variables
        # in `target_for_source` while we can see `source`. This is
        # needed for things like _aib_debugdir. Then, we need to do a
        # second subst to expand DESTDIR, interpolating
        # `target_for_source` in as $TARGET. Yes, this is confusing.
        target_for_source = env.subst(target_for_source, source=s)
        target_for_source = env.Dir(env.subst('$DESTDIR/$TARGET', target=target_for_source))

        aib_additional_directory = getattr(s.attributes, "aib_additional_directory", None)
        if aib_additional_directory is not None:
            target_for_source = env.Dir(aib_additional_directory, directory=target_for_source)

        new_installed_files = env.Install(target=target_for_source, source=s)
        setattr(s.attributes, INSTALLED_FILES, new_installed_files)
        setattr(new_installed_files[0].attributes, 'AIB_INSTALL_FROM', s)
        installed_files.extend(new_installed_files)

    entry.files.update(installed_files)
    return installed_files

def finalize_install_dependencies(env):
    """Generates task aliases and wires install dependencies."""

    # Wire up component dependencies and generate task aliases
    for task, func in env[TASKS].items():
        generate_dependent_aliases = True

        # The task map is a map of string task names (i.e. "install" by default)
        # to either a tuple or function. If it's a function we assume that we
        # generate dependent aliases for that task, otherwise if it's a tuple we
        # deconstruct it here to get the function (the first element) and a
        # boolean indicating whether or not to generate dependent aliases for
        # that task. For example the "archive" task added by the auto_archive
        # tool disables them because tarballs do not track dependencies so you
        # do not want archive-foo to build archive-bar as well if foo depends on
        # bar.
        if isinstance(func, tuple):
            func, generate_dependent_aliases = func

        for component, rolemap in env[ALIAS_MAP].items():
            for role, info in rolemap.items():
                alias_name = generate_alias_name(env, component, role, task)
                alias = env.Alias(alias_name, func(env, component, role))
                if generate_dependent_aliases:
                    dependent_aliases = env.Flatten([
                        env.Alias(generate_alias_name(env, d.component, d.role, task))
                        for d in info.dependencies
                    ])
                    env.Alias(alias, dependent_aliases)

def auto_install_emitter(target, source, env):
    """When attached to a builder adds an appropriate AutoInstall to that Builder."""

    for t in target:
        if isinstance(t, str):
            t = env.File(t)

        if env.get("AIB_IGNORE", False):
            continue

        # There is no API for determining if an Entry is operating in
        # a SConf context. We obviously do not want to auto tag, and
        # install conftest Programs. So we filter them out the only
        # way available to us.
        #
        # We're working with upstream to expose this information.
        if "conftest" in str(t):
            continue

        # Get the suffix, unless overridden
        suffix = getattr(t.attributes, "aib_effective_suffix", t.get_suffix())
        auto_install_mapping = env[SUFFIX_MAP].get(suffix)

        if auto_install_mapping is not None:
            env.AutoInstall(
                auto_install_mapping.directory,
                t,
                AIB_COMPONENT=env.get(COMPONENT, env.get(DEFAULT_COMPONENT, None)),
                AIB_ROLE=env.get(ROLE, auto_install_mapping.default_role),
                AIB_COMPONENTS_EXTRA=env.get(REVERSE_COMPONENT_DEPENDENCIES, []),
            )

    return (target, source)

def add_suffix_mapping(env, suffix, role=None):
    """Map suffix to role"""
    if isinstance(suffix, str):
        if role not in env[ROLE_DECLARATIONS]:
            raise Exception("target {} is not a known role available roles are {}".format(
                role, env[ROLE_DECLARATIONS].keys()))
        env[SUFFIX_MAP][env.subst(suffix)] = role

    if not isinstance(suffix, dict):
        raise Exception("source must be a dictionary or a string")

    for _, mapping in suffix.items():
        role = mapping.default_role
        if role not in env[ROLE_DECLARATIONS]:
            raise Exception("target {} is not a known role. Available roles are {}".format(
                target, env[ROLE_DECLARATIONS].keys()))

    env[SUFFIX_MAP].update({env.subst(key): value for key, value in suffix.items()})

def suffix_mapping(env, directory="", default_role=False):
    """Generate a SuffixMap object from source and target."""
    return SuffixMap(directory=directory, default_role=default_role)

def get_auto_installed_files(env, node):
    return getattr(node.attributes, INSTALLED_FILES, [])

def list_components(env, **kwargs):
    """List registered components for env."""
    print("Known AIB components:")
    for key in env[ALIAS_MAP]:
        print("\t", key)

def list_hierarchical_aib_recursive(mapping, counter=0):
    if counter == 0:
        print("  " * counter, mapping.id)
    counter += 1
    for dep in mapping.dependencies:
        print("  " * counter, dep.id)
        list_hierarchical_aib_targets(dep, counter=counter)

def list_hierarchical_aib_targets(dag_mode=False):
    def target_lister(env, **kwargs):
        if dag_mode:
            installed_files = set(env.FindInstalledFiles())
            for f in installed_files:
                scan_for_transitive_install(f, env, None)

        mapping = env[ALIAS_MAP][env[META_COMPONENT]][env[META_ROLE]]
        list_hierarchical_aib_recursive(mapping)

    return target_lister

def list_recursive(mapping) -> List[str]:
    items = set()
    items.add(mapping.id)
    for dep in mapping.dependencies:
        items |= list_recursive(dep)
    return items

def list_targets():
    def target_lister(env, **kwargs):
        mapping = env[ALIAS_MAP][env[META_COMPONENT]][env[META_ROLE]]
        tasks = sorted(list(env[TASKS].keys()))
        roles = sorted(list(env[ROLE_DECLARATIONS].keys()))
        targets_with_role = list(list_recursive(mapping)) + [mapping.id]
        targets: List[str] = []
        for target_role in targets_with_role:
            # Does this target_role end with one of our speicifed roles
            matching_roles = list(filter(target_role.endswith, [f"-{role}" for role in roles]))
            assert len(matching_roles) == 1

            targets.append(target_role[:-len(matching_roles[0])])

        # dedup and sort targets
        targets = sorted(list(set(targets)))
        print(
            "The following are AIB targets. Note that runtime role is implied if not specified. For example, install-mongod"
        )
        tasks_str = ','.join(tasks)
        print(f"TASK={{{tasks_str}}}")
        roles_str = ','.join(roles)
        print(f"ROLE={{{roles_str}}}")
        for target in targets:
            print(f"  TASK-{target}-ROLE")

    return target_lister

def get_role_declaration(env, role):
    return env[ROLE_DECLARATIONS][role]

def exists(_env):
    """Always activate this tool."""
    return True

def generate(env):
    """Generate the auto install builders."""
    env["AUTO_INSTALL_ENABLED"] = True

    # Matches the autoconf documentation:
    # https://www.gnu.org/prep/standards/html_node/Directory-Variables.html
    env["DESTDIR"] = env.Dir(env.get("DESTDIR", "#install"))
    env["PREFIX"] = env.get("PREFIX", ".")
    env["PREFIX_BINDIR"] = env.get("PREFIX_BINDIR", "$PREFIX/bin")
    env["PREFIX_LIBDIR"] = env.get("PREFIX_LIBDIR", "$PREFIX/lib")
    env["PREFIX_SHAREDIR"] = env.get("PREFIX_SHAREDIR", "$PREFIX/share")
    env["PREFIX_DOCDIR"] = env.get("PREFIX_DOCDIR", "$PREFIX_SHAREDIR/doc")
    env["PREFIX_INCLUDEDIR"] = env.get("PREFIX_INCLUDEDIR", "$PREFIX/include")
    env[SUFFIX_MAP] = {}
    env[ALIAS_MAP] = defaultdict(dict)

    env.AppendUnique(AIB_TASKS={
        "install": auto_install_task,
    })

    env.AddMethod(
        scan_for_transitive_install_pseudobuilder,
        "GetTransitivelyInstalledFiles",
    )
    env.AddMethod(get_role_declaration, "GetRoleDeclaration")
    env.AddMethod(get_auto_installed_files, "GetAutoInstalledFiles")
    env.AddMethod(tag_components, "TagComponents")
    env.AddMethod(auto_install_pseudobuilder, "AutoInstall")
    env.AddMethod(add_suffix_mapping, "AddSuffixMapping")
    env.AddMethod(declare_role, "Role")
    env.AddMethod(declare_roles, "DeclareRoles")
    env.AddMethod(finalize_install_dependencies, "FinalizeInstallDependencies")
    env.AddMethod(suffix_mapping, "SuffixMap")
    env.Tool("install")

    # TODO: we should probably expose these as PseudoBuilders and let
    # users define their own aliases for them.
    env.Alias("list-aib-components", [], [list_components])
    env.AlwaysBuild("list-aib-components")

    env.Alias("list-hierarchical-aib-targets", [], [list_hierarchical_aib_targets(dag_mode=False)])
    env.AlwaysBuild("list-hierarchical-aib-targets")

    env.Alias("list-hierarchical-aib-dag", [], [list_hierarchical_aib_targets(dag_mode=True)])
    env.AlwaysBuild("list-hierarchical-aib-dag")

    env.Alias("list-targets", [], [list_targets()])
    env.AlwaysBuild("list-targets")

    for builder in ["Program", "SharedLibrary", "LoadableModule", "StaticLibrary"]:
        builder = env["BUILDERS"][builder]
        base_emitter = builder.emitter
        # TODO: investigate if using a ListEmitter here can cause
        # problems if AIB is not loaded last
        new_emitter = SCons.Builder.ListEmitter([base_emitter, auto_install_emitter])
        builder.emitter = new_emitter

    base_install_builder = install.BaseInstallBuilder
    assert base_install_builder.target_scanner is None

    base_install_builder.target_scanner = SCons.Scanner.Scanner(
        function=scan_for_transitive_install,
        path_function=None,
    )


from functools import partial

import torch
import torch.nn as nn
from timm.models.helpers import build_model_with_cfg
from timm.models.layers import DropPath, trunc_normal_
from timm.models.registry import register_model
from timm.models.vision_transformer import Mlp, PatchEmbed
from timm.models.vision_transformer_hybrid import HybridEmbed

from ..layers import build_rpe, get_rpe_config

default_cfgs = {
    'deit_small_patch16_224_ctx_product_50_shared_qkv': {
        'url':
        'https://github.com/wkcn/iRPE-model-zoo/releases/download/1.0/deit_small_patch16_224_ctx_product_50_shared_qkv.pth'  # noqa
    },
    'deit_base_patch16_224_ctx_product_50_shared_qkv': {
        'url':
        'https://github.com/wkcn/iRPE-model-zoo/releases/download/1.0/deit_base_patch16_224_ctx_product_50_shared_qkv.pth'  # noqa
    }
}

class RPEAttention(nn.Module):
    """Attention with image relative position encoding."""
    def __init__(self,
                 dim,
                 num_heads=8,
                 qkv_bias=False,
                 qk_scale=None,
                 attn_drop=0.,
                 proj_drop=0.,
                 rpe_config=None):
        super().__init__()
        self.num_heads = num_heads
        head_dim = dim // num_heads
        # NOTE scale factor was wrong in my original version,
        # can set manually to be compat with prev weights
        self.scale = qk_scale or head_dim**-0.5

        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
        self.attn_drop = nn.Dropout(attn_drop)
        self.proj = nn.Linear(dim, dim)
        self.proj_drop = nn.Dropout(proj_drop)

        # image relative position encoding
        self.rpe_q, self.rpe_k, self.rpe_v = \
            build_rpe(rpe_config,
                      head_dim=head_dim,
                      num_heads=num_heads)

    def forward(self, x):
        B, N, C = x.shape
        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads,
                                  C // self.num_heads).permute(2, 0, 3, 1, 4)
        q, k, v = qkv[0], qkv[1], qkv[
            2]  # make torchscript happy (cannot use tensor as tuple)

        q *= self.scale

        attn = (q @ k.transpose(-2, -1))

        # image relative position on keys
        if self.rpe_k is not None:
            attn += self.rpe_k(q)

        # image relative position on queries
        if self.rpe_q is not None:
            attn += self.rpe_q(k * self.scale).transpose(2, 3)

        attn = attn.softmax(dim=-1)
        attn = self.attn_drop(attn)

        out = attn @ v

        # image relative position on values
        if self.rpe_v is not None:
            out += self.rpe_v(attn)

        x = out.transpose(1, 2).reshape(B, N, C)
        x = self.proj(x)
        x = self.proj_drop(x)
        return x

class RPEBlock(nn.Module):
    def __init__(self,
                 dim,
                 num_heads,
                 mlp_ratio=4.,
                 qkv_bias=False,
                 qk_scale=None,
                 drop=0.,
                 attn_drop=0.,
                 drop_path=0.,
                 act_layer=nn.GELU,
                 norm_layer=nn.LayerNorm,
                 rpe_config=None):
        super().__init__()
        self.norm1 = norm_layer(dim)
        self.attn = RPEAttention(dim,
                                 num_heads=num_heads,
                                 qkv_bias=qkv_bias,
                                 qk_scale=qk_scale,
                                 attn_drop=attn_drop,
                                 proj_drop=drop,
                                 rpe_config=rpe_config)
        # NOTE: drop path for stochastic depth,
        # we shall see if this is better than dropout here
        self.drop_path = DropPath(
            drop_path) if drop_path > 0. else nn.Identity()
        self.norm2 = norm_layer(dim)
        mlp_hidden_dim = int(dim * mlp_ratio)
        self.mlp = Mlp(in_features=dim,
                       hidden_features=mlp_hidden_dim,
                       act_layer=act_layer,
                       drop=drop)

    def forward(self, x):
        x = x + self.drop_path(self.attn(self.norm1(x)))
        x = x + self.drop_path(self.mlp(self.norm2(x)))
        return x

class VisionTransformer(nn.Module):
    """Vision Transformer with support for patch or hybrid CNN input stage and
    image relative position encoding."""
    def __init__(self,
                 img_size=224,
                 patch_size=16,
                 in_chans=3,
                 num_classes=1000,
                 embed_dim=768,
                 depth=12,
                 num_heads=12,
                 mlp_ratio=4.,
                 qkv_bias=False,
                 qk_scale=None,
                 drop_rate=0.,
                 attn_drop_rate=0.,
                 drop_path_rate=0.,
                 hybrid_backbone=None,
                 norm_layer=nn.LayerNorm,
                 rpe_config=None):
        super().__init__()
        self.num_classes = num_classes
        # num_features for consistency with other models
        self.num_features = self.embed_dim = embed_dim

        if hybrid_backbone is not None:
            self.patch_embed = HybridEmbed(hybrid_backbone,
                                           img_size=img_size,
                                           in_chans=in_chans,
                                           embed_dim=embed_dim)
        else:
            self.patch_embed = PatchEmbed(img_size=img_size,
                                          patch_size=patch_size,
                                          in_chans=in_chans,
                                          embed_dim=embed_dim)
        num_patches = self.patch_embed.num_patches

        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
        self.pos_embed = nn.Parameter(
            torch.zeros(1, num_patches + 1, embed_dim))
        self.pos_drop = nn.Dropout(p=drop_rate)

        # stochastic depth decay rule
        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]
        self.blocks = nn.ModuleList([
            RPEBlock(dim=embed_dim,
                     num_heads=num_heads,
                     mlp_ratio=mlp_ratio,
                     qkv_bias=qkv_bias,
                     qk_scale=qk_scale,
                     drop=drop_rate,
                     attn_drop=attn_drop_rate,
                     drop_path=dpr[i],
                     norm_layer=norm_layer,
                     rpe_config=rpe_config) for i in range(depth)
        ])
        self.norm = norm_layer(embed_dim)

        # NOTE as per official impl, we could have a
        # pre-logits representation dense layer + tanh here
        # self.repr = nn.Linear(embed_dim, representation_size)
        # self.repr_act = nn.Tanh()

        # Classifier head
        self.head = nn.Linear(
            embed_dim, num_classes) if num_classes > 0 else nn.Identity()

        trunc_normal_(self.pos_embed, std=.02)
        trunc_normal_(self.cls_token, std=.02)
        self.apply(self._init_weights)

    def _init_weights(self, m):
        if isinstance(m, nn.Linear):
            trunc_normal_(m.weight, std=.02)
            if isinstance(m, nn.Linear) and m.bias is not None:
                nn.init.constant_(m.bias, 0)
        elif isinstance(m, nn.LayerNorm):
            nn.init.constant_(m.bias, 0)
            nn.init.constant_(m.weight, 1.0)

    @torch.jit.ignore
    def no_weight_decay(self):
        return {'pos_embed', 'cls_token'}

    def get_classifier(self):
        return self.head

    def reset_classifier(self, num_classes, global_pool=''):
        self.num_classes = num_classes
        self.head = nn.Linear(
            self.embed_dim, num_classes) if num_classes > 0 else nn.Identity()

    def forward_features(self, x):
        B = x.shape[0]
        x = self.patch_embed(x)

        cls_tokens = self.cls_token.expand(
            B, -1, -1)  # stole cls_tokens impl from Phil Wang, thanks
        x = torch.cat((cls_tokens, x), dim=1)
        x = x + self.pos_embed
        x = self.pos_drop(x)

        for blk in self.blocks:
            x = blk(x)

        x = self.norm(x)
        return x[:, 0]

    def forward(self, x):
        x = self.forward_features(x)
        x = self.head(x)
        return x

def _filter_fn(state_dict):
    state_dict = state_dict['model']
    return state_dict

def _irpe(arch, pretrained, **kwargs):
    model = build_model_with_cfg(VisionTransformer,
                                 arch,
                                 pretrained=pretrained,
                                 default_cfg=default_cfgs[arch],
                                 pretrained_filter_fn=_filter_fn,
                                 **kwargs)
    return model

@register_model
def deit_small_patch16_224_ctx_product_50_shared_qkv(pretrained=False,
                                                     **kwargs):
    rpe_config = get_rpe_config(
        ratio=1.9,
        method='product',
        mode='ctx',
        shared_head=True,
        skip=1,
        rpe_on='qkv',
    )
    return _irpe('deit_small_patch16_224_ctx_product_50_shared_qkv',
                 pretrained=pretrained,
                 rpe_config=rpe_config,
                 patch_size=16,
                 embed_dim=384,
                 depth=12,
                 num_heads=6,
                 mlp_ratio=4,
                 qkv_bias=True,
                 norm_layer=partial(nn.LayerNorm, eps=1e-6),
                 **kwargs)

@register_model
def deit_base_patch16_224_ctx_product_50_shared_qkv(pretrained=False,
                                                    **kwargs):
    rpe_config = get_rpe_config(
        ratio=1.9,
        method='product',
        mode='ctx',
        shared_head=True,
        skip=1,
        rpe_on='qkv',
    )
    return _irpe('deit_base_patch16_224_ctx_product_50_shared_qkv',
                 pretrained=pretrained,
                 rpe_config=rpe_config,
                 patch_size=16,
                 embed_dim=768,
                 depth=12,
                 num_heads=12,
                 mlp_ratio=4,
                 qkv_bias=True,
                 norm_layer=partial(nn.LayerNorm, eps=1e-6),
                 **kwargs)


from django.shortcuts import render,redirect
from django.http.response import HttpResponse
from django.contrib.auth.decorators import login_required
# Create your views here.

@login_required(login_url='/logreg/')
def content_view(request):
    return render(request,"lsssons-gal.html",{"token":1,"username":request.user.username})

@login_required(login_url='/logreg/')
def content_etails_view(request):
    return render(request,"lsssons-gal.html",{"token":1,"username":request.user.username})
def content_details_view(request,lesson_id):
    lessonurl='lessons/'
    if lesson_id==str(1):
        lessonurl+="1t.html"
    elif lesson_id==str(2):
        lessonurl+="2t.html"
    elif lesson_id==str(3):
        lessonurl+="3t.html"
    elif lesson_id==str(4):
        lessonurl+="4t.html"
    elif lesson_id==str(5):
        lessonurl+="5t.html"
    elif lesson_id==str(6):
        lessonurl+="6t.html"
    elif lesson_id==str(7):
        lessonurl+="7t.html"
    elif lesson_id==str(8):
        lessonurl+="8t.html"
    elif lesson_id==str(9):
        lessonurl+="9t.html"
    elif lesson_id==str(10):
        lessonurl+="10t.html"
    else:
        return HttpResponse("<h1> lesson is not here : "+str(lesson_id)+"</h1>")
        lessonurl+="10.html"
    return render(request,lessonurl,{"token":1,"username":request.user.username})

def contentnew_view(request):
    return render(request, "newgal.html")


# -*- coding: utf-8 -*-

#原本为获得as, cp, sin三个加密参数使用，后使用搜索功能不再需要

from selenium import webdriver
import time
import json
import requests
from urllib.parse import urlencode

def getUrls(page, keyWord):
    url = 'https://www.toutiao.com/search_content/'
    returnUrls = []
    for num in range(page):
        data = {'offset': str(num * 20), 'format': 'json', 'keyword': keyWord, 'autoload': 'true', 'count': '20',
                'cur_tab': '1', 'from': 'search_tab'}
        returnUrls.append(url + '?' + urlencode(data))
    return returnUrls

#下面为之前的函数，不用的
def getUrl():
    dic = getKey()
    hot_time = int(time.time())
    url = "https://www.toutiao.com/api/pc/feed/?category=news_tech&" \
          "utm_source=toutiao&widen=1&max_behot_time="+str(hot_time)+"&max_behot_time_tmp="+str(hot_time)+"&" \
          "tadrequire=true&as="+str(dic['as'])+"&cp="+str(dic['cp'])+"1&_signature="+str(dic['sin'])
    with open('filee.txt', 'wb') as f:
        f.write(bytes(url, encoding='utf-8'))
    return url

def getKey():
    web = webdriver.Chrome()
    web.get('https://www.toutiao.com/ch/news_tech/')
    ascp = web.execute_script('return ascp.getHoney()')
    strSin = web.execute_script('return TAC.sign(0)')
    web.quit()
    strAs = ascp['as']
    strCp = ascp['cp']
    dic = {'as': strAs, 'cp': strCp, 'sin': strSin}
    return dic


"""Api for working with piastrix."""
from hashlib import sha256
from time import time

from requests import post

class PiastrixApi:
    """Creates and sends all requests to work with the piastrix api."""

    def __init__(
        self,
        secret_key,
        shop_id,
        payway,
        url='https://core.piastrix.com/',
    ):
        """
        Keep base settings for all requests.

        Parameters:
            secret_key: str
            shop_id: str
            payway: str
            url: str
        """
        self.secret_key = secret_key
        self.shop_id = shop_id
        self.payway = payway
        self.url = url

    def _post(self, endpoint, data):
        response = post(f'{self.url}{endpoint}', json=data)
        return response.json()

    def _shop_order_id(self):
        return 'ORDER_{0}'.format(int(time()))

    def _sign(self, required_fields, data):
        sorted_data = [data[key] for key in sorted(required_fields)]
        signed_data = ':'.join(sorted_data) + self.secret_key
        data['sign'] = sha256(signed_data.encode('utf-8')).hexdigest()

    def create_bill(self, data):
        """
        Create bill.

        Parameters:
            data: dict, dict with bill information

        Returns:
            return answer from api
        """
        required_fields = [
            'payer_currency', 'shop_amount', 'shop_currency',
            'shop_id', 'shop_order_id',
        ]
        data.update({
            'shop_id': self.shop_id,
            'shop_order_id': self._shop_order_id(),
            'shop_amount': data['amount'],
            'shop_currency': data['currency'],
            'payer_currency': data['currency'],
        })
        self._sign(required_fields, data)
        answer = self._post('bill/create', data)
        answer.update({'type': 'bill'})
        return answer

    def create_invoice(self, data):
        """
        Create invoice.

        Parameters:
            data: dict, dict with invoice information

        Returns:
            return answer from api
        """
        required_fields = [
            'amount', 'currency', 'shop_id',
            'payway', 'shop_order_id',
        ]
        data.update({
            'shop_id': self.shop_id,
            'shop_order_id': self._shop_order_id(),
            'payway': self.payway,
        })
        self._sign(required_fields, data)
        answer = self._post('invoice/create', data)
        answer.update({'type': 'invoice'})
        return answer

    def create_pay_form(self, data):
        """
        Create invoice form.

        Parameters:
            data: dict, dict with invoice information

        Returns:
            return complete data for the invoice form
        """
        required_fields = ['amount', 'currency', 'shop_id', 'shop_order_id']
        data.update({
            'type': 'pay',
            'shop_id': self.shop_id,
            'shop_order_id': self._shop_order_id(),
        })
        self._sign(required_fields, data)
        return data


# This code flattens Amazon product reviews into csv file
import json
import os
import sys
import pandas
import flatten_json
sys.path.append("etl_lib")
import etl_lib

amazon_reviews ="/Volumes/My Passport/BU - Data warehousing/datasets/AmazonReviews"
top_level_keys =[u'Reviews', u'ProductInfo']
review_keys =[u'Author', u'ReviewID', u'Overall', u'Content', u'Title', u'Date']
product_keys =[u'Price', u'ProductID', u'Features', u'ImgURL', u'Name']

# For each product there are many reviews ie. 1 to many relationship

#Print columns first
category_list=[dir.strip() for dir in os.listdir(amazon_reviews) if dir != '.DS_Store']
log_file = open("print_amzn.log","w")
exception_log = open("print_amzn_except.log","w")

for category in category_list:
 full_dir_path = amazon_reviews + "/" + category
 file_list = [fl for fl in os.listdir(full_dir_path) if fl != '.DS_Store']
 for fl in file_list:
    try:
     with open(full_dir_path + "/" + fl) as f:
      data = f.read()
      jsondata = json.loads(data)
      for row in jsondata:
          # Pull the product info first
          #Notice the use of string to deal with None return values in case lookup of column fails
          product_info="@".join([jsondata['ProductInfo'][key] or '' for key in product_keys])
          number_reviews = len(jsondata['Reviews'])
          for i in range(number_reviews):
              review = jsondata['Reviews'][i]
              review_info="@".join([review[key] or '' for key in review_keys])
              #use * as record terminator
              sys.stdout.write(category + "@" + product_info + "@" + review_info + "*")
    except:
        exception_log.write(str(sys.exc_info()[0]) + "\n")
        log_file.write(full_dir_path + "/" + fl + "\n")

log_file.close()
exception_log.close()


import gym
import numpy as np
import pickle
import pandas as pd

env = gym.make('FishingDerby-ram-v4')
env.seed(42)

test = True

def bucket(v, mx, buckets=10):
	v = min(v, mx)
	return round(float(v / mx) * float(buckets))

def phi(x):

	line_x = int(x[32])
	line_y = int(x[67])
	fish6_top_x = int(x[70])

	x_dist = fish6_top_x - line_x

	xleft = abs(x_dist) if x_dist < 0 else 0
	xright = x_dist if x_dist > 0 else 0

	y_dist = 245 - line_y
	ytop = abs(y_dist) if y_dist < 0 else 0
	ybot = y_dist if y_dist > 0 else 0

	caught_fish_idx = 112
	v0 = 0 if x[caught_fish_idx] != 2 else 1

	res = np.clip([xleft, xright, ytop, ybot], 0, 20)
	return (res[0], res[1], res[2], res[3], v0)

observation = env.reset()
# state_size = phi(observation).shape[0]
state_size = len(phi(observation))

actions = [2,3,4,5]#,3,4,5]
n_actions = 4 #env.action_space.n

print(env.unwrapped.get_action_meanings())
print('State size:', state_size)

e = 1.0
e_decay_frames = 100000
e_min = 0.05

alpha = 0.1
gamma = 0.99

counter = 0

pending_reward_idx = 114
last_reward_frames = 0
caught_fish_idx = 112
def get_reward(obs, obs_):

	# if obs_[caught_fish_idx] == 0 and obs[caught_fish_idx] == 6 and obs_[pending_reward_idx] == 0:
	# 	return -0.5

	global last_reward_frames
	if last_reward_frames > 0:
		last_reward_frames -= 1
		return 0

	if obs_[caught_fish_idx] == 2 and obs_[67] <= 210:
		pending_reward = abs(obs_[caught_fish_idx] - 7)
		last_reward_frames = pending_reward + 1
		return pending_reward + 1

	return 0

Q = {}
if test:
	u = pickle._Unpickler(open("Q.dump", "rb"))
	u.encoding = 'latin1'
	Q = u.load()

def getQ(s, a):
	if (s,a) not in Q:
		return 0.0
	else:
		return Q[(s,a)]

def learnQ(state, action, reward, value):
	v = getQ(state, action)
	if v is None:
		Q[(state, action)] = reward
	else:
		Q[(state, action)] = v + alpha * (value - v)

def learn(state1, action1, reward, state2):
	maxqnew = max([getQ(state2, a) for a in actions])
	learnQ(state1, action1, reward, reward + gamma * maxqnew)

episode = 0
df = pd.DataFrame(columns=['episode', 'value'])
while episode < 100:
	observation = env.reset()

	total_catch_value = 0
	total_value = 0
	done = False
	while not done:
		# env.render()

		state = phi(observation)

		# Take a random action fraction e (epsilon) of the time
		action = np.random.choice(range(n_actions), p=[0.26,0.23,0.23,0.28])
		# action = np.random.choice(range(n_actions))

		# Take the chosen action
		observation_, reward, done, info = env.step(actions[action])
		# if reward > 0:
		total_catch_value += reward

		reward = get_reward(observation, observation_)

		# if reward == 0:
		# 	reward = -0.01

		total_value += reward

		# Store the tuple
		state_ = phi(observation_)
		# if not test:
		# 	learn(state, action, reward, state_)

		observation = observation_

		counter += 1

		# Anneal epsilon
		if e > e_min and not test:
			e -= (1.0 - e_min) / e_decay_frames
			e = max(e_min, e)

	df.loc[episode,:] = (episode, total_catch_value)
	df.to_csv('nonuniform_random_100.csv')
	print('Finished episode', episode, total_catch_value, total_value, counter, e)

	episode += 1

df.to_csv('nonuniform_random_100.csv')


#!/usr/bin/env python
# coding: utf-8

import re
import os
from itertools import chain

data_path = os.path.join(os.getcwd(), "data\en_words\words.txt")

word_regexes = {
    'en': r'[A-Za-z]+'
}

alphabets = {
    'en': 'abcdefghijklmnopqrstuvwxyz',
}

def load_data(file):
    f = open(file, 'r')
    words = {}
    for (i, word) in enumerate(f):
        word = word.strip('\n')
        pairs = word.split(" ")
        words[pairs[0]] = pairs[1]
    f.close()
    return words

class Correcter:
    def __init__(self, threshold=0, lang='en'):
        self.threshold = threshold
        self.nlp_data = load_data(data_path)
        self.lang = lang

        if threshold > 0:
            print(f'Original number of words: {len(self.nlp_data)}')
            self.nlp_data = {k: v for k, v in self.nlp_data.items()
                            if v > threshold}
            print(f'After applying threshold: {len(self.nlp_data)}')

    def existing(self, words):
        """{'the', 'teh'} => {'the'}"""
        return set(word for word in words
                   if word in self.nlp_data)

    def autocorrect_word(self, word):
        """most likely correction for everything up to a double typo"""
        w = Word(word, self.lang)
        candidates = (self.existing([word]) or
                      self.existing(list(w.typos())) or
                      self.existing(list(w.double_typos())) or
                      [word])

        return min(candidates, key=lambda k: self.nlp_data[k])

    def autocorrect_sentence(self, sentence):
        return re.sub(word_regexes[self.lang],
                      lambda match: self.autocorrect_word(match.group(0)),
                      sentence)

    def __call__(self, sentence):
        return(self.autocorrect_sentence(sentence))

class Word(object):
    """container for word-based methods"""

    def __init__(self, word, lang='en'):
        """
        Generate slices to assist with typo
        definitions.
        'the' => (('', 'the'), ('t', 'he'),
                  ('th', 'e'), ('the', ''))
        """
        slice_range = range(len(word) + 1)
        self.slices = tuple((word[:i], word[i:])
                            for i in slice_range)
        self.word = word
        self.alphabet = alphabets[lang]

    def _deletes(self):
        """th"""
        return (self.concat(a, b[1:])
                for a, b in self.slices[:-1])

    def _transposes(self):
        """teh"""
        return (self.concat(a, reversed(b[:2]), b[2:])
                for a, b in self.slices[:-2])

    def _replaces(self):
        """tge"""
        return (self.concat(a, c, b[1:])
                for a, b in self.slices[:-1]
                for c in self.alphabet)

    def _inserts(self):
        """thwe"""
        return (self.concat(a, c, b)
                for a, b in self.slices
                for c in self.alphabet)

    def concat(self, *args):
        """reversed('th'), 'e' => 'hte'"""
        try:
            return ''.join(args)
        except TypeError:
            return ''.join(chain.from_iterable(args))

    def typos(self):
        """letter combinations one typo away from word"""
        yield from self._deletes()
        yield from self._transposes()
        yield from self._replaces()
        yield from self._inserts()

    def double_typos(self):
        """letter combinations two typos away from word"""
        return (e2 for e1 in self.typos()
                for e2 in Word(e1).typos())

correct = Correcter(lang='en')
correct("th")

# In[ ]:


import argparse
import json
import logging

from nlp_architect.data.cdc_resources.relations.verbocean_relation_extraction import (
    VerboceanRelationExtraction,
)
from nlp_architect.models.cross_doc_coref.system.cdc_utils import load_mentions_vocab_from_files
from nlp_architect.utils import io

logger = logging.getLogger(__name__)

parser = argparse.ArgumentParser(description="Create Verb-Ocean dataset only dump")

parser.add_argument("--vo", type=str, help="Verb Ocean file", required=True)

parser.add_argument("--mentions", type=str, help="dataset mentions", required=True)

parser.add_argument("--output", type=str, help="location were to create dump file", required=True)

args = parser.parse_args()

def vo_dump():
    vo_file = args.vo
    out_file = args.output
    mentions_event_gold_file = [args.mentions]
    vocab = load_mentions_vocab_from_files(mentions_event_gold_file, True)
    vo = VerboceanRelationExtraction.load_verbocean_file(vo_file)
    vo_for_vocab = {}
    for word in vocab:
        if word in vo:
            vo_for_vocab[word] = vo[word]

    logger.info("Found %d words from vocabulary", len(vo_for_vocab.keys()))
    logger.info("Preparing to save refDict output file")

    with open(out_file, "w") as f:
        json.dump(vo_for_vocab, f)
    logger.info("Done saved to-%s", out_file)

if __name__ == "__main__":
    io.validate_existing_filepath(args.mentions)
    io.validate_existing_filepath(args.output)
    io.validate_existing_filepath(args.vo)
    vo_dump()


import discord
import asyncio
import pickle

with open("contestantsID.txt", "rb") as fp:
    local_contestantsID = pickle.load(fp)

with open("contestants.txt", "rb") as fp:
    local_contestants = pickle.load(fp)

commands = ["register","unregister","help","players"]
commands_text = ["Used followed by name like '>register PlayerName'","Used followed by name like '>unregister PlayerName'","Gives a list of commands","Give a list of players in tournaments"]

client = discord.Client()

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('-----------')

@client.event
async def on_message(message):
    b = message.author.id
    if message.content.startswith('>register'):
        if len(local_contestants) == 16:
            await message.channel.send("The tournament is full right now!")

        else:
            m = message.content.split()
            try:
                a = m[1].upper()
                await message.channel.send(a+", will be your name in the tournament!")
                local_contestants.append(a)
                local_contestantsID.append(b)
                with open("contestants.txt", "wb") as fp:
                    pickle.dump(local_contestants, fp)

                with open("contestantsID.txt", "wb") as fp:
                    pickle.dump(local_contestantsID, fp)

            except:
                await message.channel.send("You did not give me a name!")

    elif message.content.startswith('>unregister'):
        m = message.content.split()

        try:
            a = m[1].upper()

            for x in range(0,len(local_contestants)):
                if local_contestants[x] == a:
                    if message.author.id == local_contestantsID[x]:
                        local_contestants.pop(x)
                        local_contestantsID.pop(x)

                        with open("contestants.txt", "wb") as fp:
                            pickle.dump(local_contestants, fp)

                        with open("contestantsID.txt", "wb") as fp:
                            pickle.dump(local_contestantsID, fp)

                        await message.channel.send("Done!")

                    else:
                        await message.channel.send("That is not your registration!")

        except:
            await message.channel.send("You did not give me a name!")

    elif message.content.startswith('>help'):
        for i in range(0,len(commands)):
            await message.channel.send(commands[i])
            await message.channel.send(commands_text[i])
            await message.channel.send("-------------")

    elif message.content.startswith('>players'):

        await message.channel.send("Currently, there are "+ str(len(local_contestants)) +" players in the tournament:")

        for i in range(0,len(local_contestants)):
            await message.channel.send(local_contestants[i])

    elif message.content.startswith('>purge'):
        if message.author.id == 340610417967497216 or message.author.id == 364830006334980096:
            local_contestants.clear()
            local_contestantsID.clear()

            with open("contestants.txt", "wb") as fp:
                pickle.dump(local_contestants, fp)

            with open("contestantsID.txt", "wb") as fp:
                pickle.dump(local_contestantsID, fp)

            await message.channel.send("Done!")

    elif message.content.startswith('>delete'):
        if message.author.id == 340610417967497216 or message.author.id == 364830006334980096:
            m = message.content.split()
            a = m[1].upper()

            for l in range(0,len(local_contestants)):
                if local_contestants[l] == a:
                    local_contestants.pop(l)
                    local_contestantsID.pop(l)

                    with open("contestants.txt", "wb") as fp:
                        pickle.dump(local_contestants, fp)

                    with open("contestantsID.txt", "wb") as fp:
                        pickle.dump(local_contestantsID, fp)

                    await message.channel.send("Done!")

    elif message.content.startswith('>data'):
        if message.author.id == 340610417967497216 or message.author.id == 364830006334980096:
            await message.channel.send(local_contestants)
            await message.channel.send(local_contestantsID)

    elif message.content.startswith('>timer'):
        await message.channel.send("Tournaments are held every Friday at 5:00 PM BST.")

f = open("env.txt", "r")
contents = f.read()
client.run(contents)