#%%
import __init__ #type: ignore
from IPython.display import display
from scipy.integrate import solve_ivp #type: ignore
import matplotlib.pyplot as plt#type: ignore
import numpy as np
import sympy as sy
import plotly.express as px#type: ignore
from pandas import DataFrame #type: ignore
import math
from scipy.optimize import fsolve#type: ignore
from pyeq2orb.SymbolicOptimizerProblem import SymbolicProblem
from pyeq2orb.ScaledSymbolicProblem import ScaledSymbolicProblem
from pyeq2orb.Problems.ContinuousThrustCircularOrbitTransfer import ContinuousThrustCircularOrbitTransferProblem
from pyeq2orb.Numerical import ScipyCallbackCreators
from pyeq2orb.DemosAndPrototypes.LambdifyHelpers import OdeLambdifyHelperWithBoundaryConditions

import sympy as sy
from typing import List, Dict, Callable, Optional, Any
from pyeq2orb.SymbolicOptimizerProblem import SymbolicProblem
from pyeq2orb.Utilities.Typing import SymbolOrNumber

def plotSolution(helper, solution):

    xyz = np.zeros((len(tArray), 3))
    for i in range(0, len(solution[helper.NonTimeLambdifyArguments[0]])) :
        r = solution[helper.NonTimeLambdifyArguments[0]][i]
        theta = solution[helper.NonTimeLambdifyArguments[3]][i]
        x = r*math.cos(theta)
        y = r*math.sin(theta)
        xyz[i,0] = x
        xyz[i,1] = y
        xyz[i,2] = 0

    df = DataFrame(xyz)

    xf = np.array(xyz[:,0])
    yf = np.array(xyz[:,1])
    zf = np.array(xyz[:,2])
    df = DataFrame({"x": xf, "y":yf, "z":zf})
    fig = px.line_3d(df, x="x", y="y", z="z")
    fig.show()

from pyeq2orb.Utilities.SolutionDictionaryFunctions import GetValueFromStateDictionaryAtIndex
import scipyPaperPrinter as jh#type: ignore
from typing import cast
# constants
g = 9.80665
mu = 3.986004418e14
thrust = 20.0
isp = 6000.0
m0 = 1500.0

# initial values
r0 = 6678000.0
u0 = 0.0
v0 = sy.sqrt(mu/r0) # circular
lon0 = 0.0
# I know from many previous runs that this is the time needed to go from LEO to GEO.
# However, below works well wrapped in another fsolve to control the final time for a desired radius.
tfVal  = 3600*3.97152*24
tfOrg = tfVal
tArray = np.linspace(0.0, tfOrg, 1200)
#if scaleTime:
#    tfVal = 1.0
#    tArray = np.linspace(0.0, 1.0, 1200)

# your choice of the nu vector here controls which transversality condition we use
nus = [sy.Symbol('B_{u_f}'), sy.Symbol('B_{v_f}')]
#nus = []

mus = sy.Symbol(r'\mu', real=True, positive=True)
thrusts = sy.Symbol('T', real=True, positive=True)
m0s = sy.Symbol('m_0', real=True, positive=True)

gs = sy.Symbol('g', real=True, positive=True)

ispS = sy.Symbol('I_{sp}', real=True, positive=True)
ts = sy.Symbol('t', real=True)
t0s = sy.Symbol('t_0', real=True)
tfs = sy.Symbol('t_f', real=True, positive=True)

rs = sy.Function('r', real=True, positive=True)(ts)
us = sy.Function('u', real=True, nonnegative=True)(ts)
vs = sy.Function('v', real=True, nonnegative=True)(ts)
lonS =  sy.Function(r'\theta', real=True, nonnegative=True)(ts)

alps = sy.Function(r'\alpha', real=True)(ts)

bc1 = us.subs(ts, tfs)
bc2 = vs.subs(ts, tfs)-sy.sqrt(mu/rs.subs(ts, tfs))

terminalCost = rs.subs(ts, tfs) # maximization problem

mFlowRate = -1*thrusts/(ispS*gs)
mEq = m0s+ts*mFlowRate

rEom = us
uEom = vs*vs/rs - mus/(rs*rs) + thrusts*sy.sin(alps)/mEq
vEom = -vs*us/rs + thrusts*sy.cos(alps)/mEq
lonEom = vs/rs

rEquation = sy.Eq(rs.diff(ts), rEom)
uEquation = sy.Eq(us.diff(ts), uEom)
vEquation = sy.Eq(vs.diff(ts), vEom)
lonEquation = sy.Eq(lonS.diff(ts), lonEom)

helper = OdeLambdifyHelperWithBoundaryConditions(ts, t0s, tfs, [rEquation, uEquation, vEquation, lonEquation], [bc1, bc2], [], {gs:g, ispS:isp, m0s: m0, thrusts:thrust, mus:mu})

lmds = SymbolicProblem.CreateCoVector(helper.NonTimeLambdifyArguments, None, ts)
hamlt = SymbolicProblem.CreateHamiltonianStatic(helper.NonTimeLambdifyArguments, ts, helper.GetExpressionToLambdifyInMatrixForm(), 0, lmds)
lambdaEoms = SymbolicProblem.CreateLambdaDotEquationsStatic(hamlt, ts, helper.NonTimeArgumentsArgumentsInMatrixForm(), lmds)
helper.AddMoreEquationsOfMotion(lambdaEoms)
dHdu = SymbolicProblem.CreateHamiltonianControlExpressionsStatic(hamlt, alps)
controlSolved = sy.solve(dHdu, alps)[0]
helper.SubstitutionDictionary[alps] =  controlSolved

#TODO: Get the transversality conditions more generally
bc3 = 1-lmds[0].subs(ts, tfs)+0.5*lmds[2].subs(ts, tfs)*sy.sqrt(1/(rs.subs(ts, tfs)**3))
bc4 = lmds[3].subs(ts, tfs)
newBcs = [bc3, bc4]
helper.BoundaryConditionExpressions.extend(newBcs)
helper.SymbolsToSolveForWithBoundaryConditions.extend(SymbolicProblem.SafeSubs(lmds, {ts: t0s}))

# From working the problem enough, we find that one of the costate variables is constant and 0.
# As such, the related BC can be removed, one of the costates can be made constant, an EOM removed, etc..
# This needs to be improved, but for now, just do manually
del helper.BoundaryConditionExpressions[-1]
del helper.NonTimeLambdifyArguments[-1]
del helper.EquationsOfMotion[-1]
del helper.ExpressionsToLambdify[-1]
del helper.SymbolsToSolveForWithBoundaryConditions[-1]
helper.SubstitutionDictionary[lmds[3]] =0
helper.SubstitutionDictionary[lmds[3].subs(ts, tfs)]=0
helper.SubstitutionDictionary[lmds[3].subs(ts, t0s)]=0

ipvCallback = helper.CreateSimpleCallbackForSolveIvp()
def realIpvCallback(initialState) :
    solution = solve_ivp(ipvCallback, [tArray[0], tArray[-1]], initialState, t_eval=tArray, dense_output=True, method="LSODA", rtol=1.49012e-8, atol=1.49012e-11)
    solutionDictionary = ScipyCallbackCreators.ConvertEitherIntegratorResultsToDictionary(helper.NonTimeLambdifyArguments, solution)
    return solution

# In earlier iterations, I've found a way for this problem to get a not-horrible initial guess for the initial costates.  Not reproducing here
# for now
initialFSolveState = [0.0011569091762708, 0.00010000000130634948, 1.0]

justBcCb = helper.CreateCallbackForBoundaryConditionsWithFullState()
display(justBcCb[1](0, 6000, 7000, 8000, 0, *initialFSolveState, 5000000, 42164000, -2000, -3000, 17, 10000, 20000, 30000))

solverCb = helper.createCallbackToSolveForBoundaryConditions(realIpvCallback, tArray, [r0, u0, v0, lon0, *initialFSolveState])

#display(solverCb([1.0, 0.001, 0.001, 0.0]))
#display(helper.GetExpressionToLambdifyInMatrixForm())
#print(ipvCallback(0, [r0, u0, v0, lon0, 1.0, 0.001, 0.001, 0.0]))
solution = solve_ivp(ipvCallback, [tArray[0], tArray[-1]], [r0, u0, v0, lon0, *initialFSolveState], t_eval=tArray, dense_output=True, method="LSODA", rtol=1.49012e-8, atol=1.49012e-11)
solutionDictionary = ScipyCallbackCreators.ConvertEitherIntegratorResultsToDictionary(helper.NonTimeLambdifyArguments, solution)
plotSolution(helper, solutionDictionary)

fSolveSol = fsolve(solverCb, initialFSolveState, epsfcn=0.000001, full_output=True)
display(fSolveSol)
fSolveSolution = solve_ivp(ipvCallback, [tArray[0], tArray[-1]], [r0, u0, v0, lon0, *fSolveSol[0]], t_eval=tArray, dense_output=True, method="LSODA", rtol=1.49012e-8, atol=1.49012e-11)
fSolveSolutionDictionary = ScipyCallbackCreators.ConvertEitherIntegratorResultsToDictionary(helper.NonTimeLambdifyArguments, fSolveSolution)

plotSolution(helper, fSolveSolutionDictionary)

#%%
# these are options to switch to try different things
scaleElements = True
scaleTime = scaleElements and True

baseProblem = ContinuousThrustCircularOrbitTransferProblem()
initialStateValues = baseProblem.CreateVariablesAtTime0(baseProblem.StateVariables)
problem = cast(SymbolicProblem, baseProblem)

if scaleElements :
    newSvs = ScaledSymbolicProblem.CreateBarVariables(problem.StateVariables, problem.TimeSymbol)
    problem = ScaledSymbolicProblem(baseProblem, newSvs, {problem.StateVariables[0]: initialStateValues[0],
                                                          problem.StateVariables[1]: initialStateValues[2],
                                                          problem.StateVariables[2]: initialStateValues[2],
                                                          problem.StateVariables[3]: 1} , scaleTime) # note the int here for the scaling, not a float
stateAtTf = SymbolicProblem.SafeSubs(problem.StateVariables, {problem.TimeSymbol: problem.TimeFinalSymbol})
# make the time array
tArray = np.linspace(0.0, tfOrg, 1200)
if scaleTime:
    tfVal = 1.0
    tArray = np.linspace(0.0, 1.0, 1200)
jh.t = problem._timeSymbol # needed for cleaner printed equations

# register constants
constantsSubsDict = problem.SubstitutionDictionary
constantsSubsDict[baseProblem.Isp] = isp
constantsSubsDict[baseProblem.MassInitial] = m0
constantsSubsDict[baseProblem.Gravity] = g
constantsSubsDict[baseProblem.Mu]= mu
constantsSubsDict[baseProblem.Thrust] = thrust

# register initial state values
constantsSubsDict.update(zip(initialStateValues, [r0, u0, v0, lon0]))
if scaleElements :
    # and reset the real initial values using tau_0 instead of time
    initialValuesAtTau0 = SymbolicProblem.SafeSubs(initialStateValues, {baseProblem.TimeInitialSymbol: problem.TimeInitialSymbol})
    constantsSubsDict.update(zip(initialValuesAtTau0, [r0, u0, v0, lon0]))

    r0= r0/r0
    u0=u0/v0
    v0=v0/v0
    lon0=lon0/1.0
    # add the scaled initial values (at tau_0).  We should NOT need to add these at t_0
    initialScaledStateValues = problem.CreateVariablesAtTime0(problem.StateVariables)
    constantsSubsDict.update(zip(initialScaledStateValues, [r0, u0, v0, lon0]))

# this next block does most of the problem, pretty standard optimal control actions
problem.Lambdas.extend(problem.CreateCoVector(problem.StateVariables, r'\lambda', problem.TimeSymbol))
lambdas = problem.Lambdas
hamiltonian = problem.CreateHamiltonian(lambdas)
lambdaDotExpressions = problem.CreateLambdaDotCondition(hamiltonian)
dHdu = problem.CreateHamiltonianControlExpressions(hamiltonian)[0]
controlSolved = sy.solve(dHdu, problem.ControlVariables[0])[0] # something that may be different for other problems is when there are multiple control variables

# you are in control of the order of integration variables and what equations of motion get evaluated, start updating the problem
# NOTE that this call adds the lambdas to the integration state
problem.EquationsOfMotion.update(zip(lambdas, lambdaDotExpressions))
SymbolicProblem.SafeSubs(problem.EquationsOfMotion, {problem.ControlVariables[0]: controlSolved})
# the trig simplification needs the deep=True for this problem to make the equations even cleaner
for (key, value) in problem.EquationsOfMotion.items() :
    problem.EquationsOfMotion[key] = value.trigsimp(deep=True).simplify() # some simplification to make numerical code more stable later, and that is why this code forces us to do things somewhat manually.  There are often special things like this that we ought to do that you can't really automate.

## Start with the boundary conditions
if scaleTime : # add BC if we are working with the final time (kind of silly for this example, but we need an equal number of in's and out's for fsolve later)
    problem.BoundaryConditions.append(baseProblem.TimeFinalSymbol-tfOrg)

# make the transversality conditions
if len(nus) != 0:
    transversalityCondition = problem.TransversalityConditionsByAugmentation(nus)
else:
    transversalityCondition = problem.TransversalityConditionInTheDifferentialForm(hamiltonian, sy.Symbol(r'dt_f'))
# and add them to the problem
problem.BoundaryConditions.extend(transversalityCondition)

initialFSolveStateGuess = ContinuousThrustCircularOrbitTransferProblem.CreateInitialLambdaGuessForLeoToGeo(problem, controlSolved)

# lambda_lon is always 0, so do that cleanup
del problem.EquationsOfMotion[lambdas[3]]
problem.BoundaryConditions.remove(transversalityCondition[-1])
lmdTheta = lambdas.pop()
problem.IntegrationSymbols.pop()
constantsSubsDict[lmdTheta]=0
constantsSubsDict[lmdTheta.subs(problem.TimeSymbol, problem.TimeFinalSymbol)]=0
constantsSubsDict[lmdTheta.subs(problem.TimeSymbol, problem.TimeInitialSymbol)]=0

# start the conversion to a numerical problem
if scaleTime :
    initialFSolveStateGuess.append(tfOrg)

otherArgs = []
if scaleTime :
    otherArgs.append(baseProblem.TimeFinalSymbol)
if len(nus) > 0 :
    otherArgs.extend(nus)
stateAndLambdas = []
stateAndLambdas.extend(problem.StateVariables)
stateAndLambdas.extend(lambdas)
odeState = [problem.TimeSymbol, stateAndLambdas, otherArgs]

def safeSubs(exprs, toBeSubbed):
    tbr = []
    for eom in exprs :
        if hasattr(eom, "subs"):
            tbr.append(eom.subs(toBeSubbed))
        else :
            tbr.append(eom)
    return tbr

class OdeHelper :
    lambdifyStateFlattenOption = "flatten"
    lambdifyStateGroupedAllOption = "group"
    lambdifyStateGroupedAllButParametersOption = "groupFlattenParameters"

    lambdifyStateOrderOptionTimeFirst = "Time,StateVariables,MissingInitialValues,Parameters"
    lambdifyStateOrderOptionTimeMiddle = "StateVariables,Time,MissingInitialValues,Parameters"
    def __init__(self, t) :
        self.equationsOfMotion = []
        self.initialSymbols = []
        self.stateFunctionSymbols = []
        self.t = t
        self.constants = {}
        self.lambdifyParameterSymbols = []

    def setStateElement(self, sympyFunctionSymbol, symbolicEom, initialSymbol) :
        self.equationsOfMotion.append(symbolicEom)
        self.stateFunctionSymbols.append(sympyFunctionSymbol)
        self.initialSymbols.append(initialSymbol)

    def makeStateForLambdifiedFunction(self, groupOrFlatten=lambdifyStateGroupedAllButParametersOption, orderOption=lambdifyStateOrderOptionTimeFirst):
        arrayForLmd = []
        if orderOption == OdeHelper.lambdifyStateOrderOptionTimeFirst :
            arrayForLmd.append(self.t)
        stateArray = []
        for svf in self.stateFunctionSymbols :
            stateArray.append(svf)
        if groupOrFlatten != OdeHelper.lambdifyStateFlattenOption :
            arrayForLmd.append(stateArray)
        else :
            arrayForLmd.extend(stateArray)
        if orderOption == OdeHelper.lambdifyStateOrderOptionTimeMiddle :
            arrayForLmd.append(self.t)

        if len(self.lambdifyParameterSymbols) != 0 :
            if groupOrFlatten == OdeHelper.lambdifyStateGroupedAllButParametersOption or groupOrFlatten == OdeHelper.lambdifyStateFlattenOption:
                arrayForLmd.extend(self.lambdifyParameterSymbols)
            elif groupOrFlatten == OdeHelper.lambdifyStateGroupedAllOption :
                arrayForLmd.append(self.lambdifyParameterSymbols)
        return arrayForLmd

    def _createParameterOptionalWrapperOfLambdifyCallback(self, baseLambdifyCallback) :
        def callbackWrapper(a, b, *args) :
            if len(self.lambdifyParameterSymbols) == 0 :
                return baseLambdifyCallback(a, b)
            else :
                return baseLambdifyCallback(a, b, *args)
        return callbackWrapper

    def createLambdifiedCallback(self, groupOrFlatten=lambdifyStateGroupedAllButParametersOption, orderOption=lambdifyStateOrderOptionTimeFirst) :
        arrayForLmd=self.makeStateForLambdifiedFunction(groupOrFlatten, orderOption)
        subbedEom = safeSubs(self.equationsOfMotion, self.constants)
        baseLambdifyCallback = sy.lambdify(arrayForLmd, subbedEom, 'numpy')
        return self._createParameterOptionalWrapperOfLambdifyCallback(baseLambdifyCallback)

thisOdeHelper = OdeHelper(problem.TimeSymbol)
for key, value in problem.EquationsOfMotion.items() :
    thisOdeHelper.setStateElement(key, value, key.subs(problem.TimeSymbol, problem.TimeInitialSymbol) )

if scaleTime:
    thisOdeHelper.lambdifyParameterSymbols.append(baseProblem.TimeFinalSymbol)

if len(nus) != 0:
    thisOdeHelper.lambdifyParameterSymbols.append(problem.CostateSymbols[1])
    thisOdeHelper.lambdifyParameterSymbols.append(problem.CostateSymbols[2])

thisOdeHelper.constants = problem.SubstitutionDictionary
display(thisOdeHelper.makeStateForLambdifiedFunction())
odeIntEomCallback = thisOdeHelper.createLambdifiedCallback()

if len(nus) > 0 :
    # run a test solution to get a better guess for the final nu values, this is a good technique, but
    # it is still a custom-to-this-problem piece of code because it is still initial-guess work
    initialFSolveStateGuess.append(initialFSolveStateGuess[1])
    initialFSolveStateGuess.append(initialFSolveStateGuess[2])
    argsForOde = []
    if scaleTime :
        argsForOde.append(tfOrg)
    argsForOde.append(initialFSolveStateGuess[1])
    argsForOde.append(initialFSolveStateGuess[2])
    print("solving ivp for final adjoined variable guess")
    testSolution = solve_ivp(odeIntEomCallback, [tArray[0], tArray[-1]], [r0, u0, v0, lon0, *initialFSolveStateGuess[0:3]], args=tuple(argsForOde), t_eval=tArray, dense_output=True, method="LSODA", rtol=1.49012e-8, atol=1.49012e-11)
    #testSolution = odeint(odeIntEomCallback, [r0, u0, v0, lon0, *initialFSolveStateGuess[0:3]], tArray, args=tuple(argsForOde))
    finalValues = ScipyCallbackCreators.GetFinalStateFromIntegratorResults(testSolution)
    initialFSolveStateGuess[-2] = finalValues[5]
    initialFSolveStateGuess[-1] = finalValues[6]

print(initialFSolveStateGuess)

stateForBoundaryConditions = []
stateForBoundaryConditions.extend(SymbolicProblem.SafeSubs(problem.IntegrationSymbols, {problem.TimeSymbol: problem.TimeInitialSymbol}))
stateForBoundaryConditions.extend(SymbolicProblem.SafeSubs(problem.IntegrationSymbols, {problem.TimeSymbol: problem.TimeFinalSymbol}))
stateForBoundaryConditions.extend(lambdas)
stateForBoundaryConditions.extend(otherArgs)

fSolveCallback = ContinuousThrustCircularOrbitTransferProblem.createSolveIvpSingleShootingCallbackForFSolve(problem, problem.IntegrationSymbols, [r0, u0, v0, lon0], tArray, odeIntEomCallback, problem.BoundaryConditions, SymbolicProblem.SafeSubs(lambdas, {problem.TimeSymbol: problem.TimeInitialSymbol}), otherArgs)
fSolveSol = fsolve(fSolveCallback, initialFSolveStateGuess, epsfcn=0.000001, full_output=True) # just to speed things up and see how the initial one works
print(fSolveSol)

# final run with answer
solution = solve_ivp(odeIntEomCallback, [tArray[0], tArray[-1]], [r0, u0, v0, lon0, *fSolveSol[0][0:3]], args=tuple(fSolveSol[0][3:len(fSolveSol[0])]), t_eval=tArray, dense_output=True, method="LSODA", rtol=1.49012e-8, atol=1.49012e-11)
#solution = odeint(odeIntEomCallback, [r0, u0, v0, lon0, *fSolveSol[0][0:3]], tArray, args=tuple(fSolveSol[0][3:len(fSolveSol[0])]))
#solution = odeint(odeIntEomCallback, [r0, u0, v0, lon0, 26.0, 1.0, 27.0], tArray, args=(tfOrg,))
solutionDictionary = ScipyCallbackCreators.ConvertEitherIntegratorResultsToDictionary(problem.IntegrationSymbols, solution)
unscaledResults = solutionDictionary
unscaledTArray = tArray
unscaledResults = problem.DescaleResults(solutionDictionary)
if scaleTime:
    unscaledTArray=tfOrg*tArray

if scaleElements:
    finalState = GetValueFromStateDictionaryAtIndex(solutionDictionary, -1)
    jh.showEquation(stateAtTf[0], finalState[problem.StateVariables[0]])
    jh.showEquation(stateAtTf[1], finalState[problem.StateVariables[1]])
    jh.showEquation(stateAtTf[2], finalState[problem.StateVariables[2]])
    jh.showEquation(stateAtTf[3], (finalState[problem.StateVariables[3]]%(2*math.pi))*180.0/(2*math.pi))

baseProblem.PlotSolution(tArray*tfOrg, unscaledResults, "Test")
jh.showEquation(baseProblem.StateVariables[0].subs(problem.TimeSymbol, problem.TimeFinalSymbol), unscaledResults[baseProblem.StateVariables[0]][-1])
jh.showEquation(baseProblem.StateVariables[1].subs(problem.TimeSymbol, problem.TimeFinalSymbol), unscaledResults[baseProblem.StateVariables[1]][-1])
jh.showEquation(baseProblem.StateVariables[2].subs(problem.TimeSymbol, problem.TimeFinalSymbol), unscaledResults[baseProblem.StateVariables[2]][-1])
jh.showEquation(baseProblem.StateVariables[3].subs(problem.TimeSymbol, problem.TimeFinalSymbol), (unscaledResults[baseProblem.StateVariables[3]][-1]%(2*math.pi))*180.0/(2*math.pi))

[hamiltonVals, dhduValues, d2hdu2Values] = problem.EvaluateHamiltonianAndItsFirstTwoDerivatives(solutionDictionary, tArray, hamiltonian, {problem.ControlVariables[0]: controlSolved}, {baseProblem.TimeFinalSymbol: tfOrg})
plt.title("Hamiltonian and its derivatives")
plt.plot(tArray/86400, hamiltonVals, label="Hamiltonian")
plt.plot(tArray/86400, dhduValues, label=r'$\frac{dH}{du}$')
plt.plot(tArray/86400, d2hdu2Values, label=r'$\frac{d^2H}{du^2}$')

plt.tight_layout()
plt.grid(alpha=0.5)
plt.legend(framealpha=1, shadow=True)
plt.show()

xyz = np.zeros((len(tArray), 3))
for i in range(0, len(unscaledResults[baseProblem.StateVariables[0]])) :
    r = unscaledResults[baseProblem.StateVariables[0]][i]
    theta = unscaledResults[baseProblem.StateVariables[3]][i]
    x = r*math.cos(theta)
    y = r*math.sin(theta)
    xyz[i,0] = x
    xyz[i,1] = y
    xyz[i,2] = 0

df = DataFrame(xyz)

xf = np.array(xyz[:,0])
yf = np.array(xyz[:,1])
zf = np.array(xyz[:,2])
df = DataFrame({"x": xf, "y":yf, "z":zf})
fig = px.line_3d(df, x="x", y="y", z="z")
fig.show()


# import required files and modules

import requests
from bs4 import BeautifulSoup
import pandas as pd

# set the headers and user string
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"
}

def product(url):
    # send a request to fetch HTML of the page

    response = requests.get(url, headers=headers)
    if response.status_code != 200:
      print('Sorry cannot fetch data for this product right now!!')
      exit()
    #print(response)
    # create the soup object
    soup = BeautifulSoup(response.content, 'html.parser')

    # change the encoding to utf-8
    soup.encode('utf-8')

    title = soup.find('span',attrs={'class': 'B_NuCI'}).get_text()
    price = soup.find('div',attrs={'class': '_30jeq3 _16Jk6d'}).get_text()

    th=[]
    th.append('Title')
    th.append('Price')
    for td in soup.find_all('td',attrs={'class':'_1hKmbr col col-3-12'},text=True):
        th.append(td.get_text())

    td=[]
    td.append(title)
    td.append(price)
    for t in soup.find_all('li',attrs={'class':'_21lJbe'},text=True):
        td.append(t.get_text())

    if len(th)==2:
      for i in soup.find_all('div',attrs={'class':'col col-3-12 _2H87wv'},text=True):
        th.append(i.get_text())

      for t in soup.find_all('div',attrs={'class':'col col-9-12 _2vZqPX'},text=True):
        td.append(t.get_text())

    if len(th)>len(td):
      th.pop(2)
    else:
      if len(th)<len(td):
        td.pop(2)

    data={"Features": th,
      "Details": td
       }

    df=pd.DataFrame(data=data)

    return df
n=int(input())

list_of_df=[]

for i in range(0,n):
  url=input()
  list_of_df.append(product(url))

feature={ }
final={}

list_of_feature=[]

for df in list_of_df :
  for index,rows in df.iterrows():
    if rows["Features"] in feature:
       h=1
    else :
      list_of_feature.append(rows["Features"])
      feature[rows["Features"]]=1

final["Features"]=list_of_feature
heading='Details of Product '
for i in range(0,n):
  final[heading+str(i+1)]=[]

for f in list_of_feature:
  i=0
  for df in list_of_df :
    flag=0
    for index,rows in df.iterrows():
      if rows["Features"]==f:
        flag=1
        final[heading+str(i+1)].append(rows["Details"])
    if flag==0:
       final[heading+str(i+1)].append('NA')
    i+=1

final_df=pd.DataFrame(data=final)
final_df.to_excel("data.xlsx",index=False)


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from . import views

router = DefaultRouter()
router.register(r"categorias", views.CategoriaViewSet)
router.register(r"tipoGastos", views.TipoGastoViewSet)
router.register(r"proveedores", views.ProveedorViewSet)
router.register(r"gastos", views.GastoViewSet)

urlpatterns = [
  path('gastos/sumatoria/', views.total_gastos),
  path('', include(router.urls))
]


import pytest

from maasserver.certificates import get_maas_certificate
from maasserver.secrets import SecretManager
from provisioningserver.testing.certificates import (
    get_sample_cert_with_cacerts,
)

@pytest.mark.usefixtures("maasdb")
class TestGetMAASCertificate:
    def test_no_secret(self):
        assert (
            SecretManager().get_composite_secret("tls", default=None) is None
        )
        assert get_maas_certificate() is None

    def test_no_all_entries_secret(self):
        SecretManager().set_composite_secret("tls", {"cert": "ABC"})
        assert get_maas_certificate() is None

    def test_certificate(self):
        cert = get_sample_cert_with_cacerts()
        SecretManager().set_composite_secret(
            "tls",
            {
                "cert": cert.certificate_pem(),
                "key": cert.private_key_pem(),
                "cacert": cert.ca_certificates_pem(),
            },
        )
        maas_cert = get_maas_certificate()
        assert maas_cert is not None
        assert maas_cert.certificate_pem() == cert.certificate_pem()
        assert maas_cert.private_key_pem() == cert.private_key_pem()
        assert maas_cert.ca_certificates_pem() == cert.ca_certificates_pem()


"""
A tool that connects with the available blendshapes.
|-----------------------------------|
|  control_name:       [          ] |
|  connection_name:    [translateX] |
|  -------------------------------- |
|  blend_shape_node    [          ] |
|  blend_shape_attrib  [          ] |
| --------------------------------- |
|   [     button: connect       ]   |
|---------------------------------- |
"""

# import standard modules
from PySide2 import QtCore, QtGui, QtWidgets

# import local modules
from deformers import blendshape_utils
from maya_utils import ui_utils
from maya_utils import object_utils
from maya_utils import animation_utils

# reloads
reload(ui_utils)
reload(object_utils)

# define global variables
window_title = 'agBlendShapeConnector'
open_windows = {}

class Form(QtWidgets.QDialog):
    WIDGETS = {}

    def __init__(self, parent=None):
        super(Form, self).__init__(parent)
        self.setWindowTitle(window_title)

        # main layour
        layout = QtWidgets.QVBoxLayout()

        # controller object
        h_layout = QtWidgets.QHBoxLayout()
        self.WIDGETS['controllerLineEdit'] = QtWidgets.QLineEdit("Please enter the controller name.")
        self.WIDGETS['controllerLineEdit'].setReadOnly(True)
        self.WIDGETS['lineEditCtrlAddButton'] = QtWidgets.QPushButton(" <<< Control")
        h_layout.addWidget(self.WIDGETS['controllerLineEdit'], 0)
        h_layout.addWidget(self.WIDGETS['lineEditCtrlAddButton'], 1)

        layout.addLayout(h_layout) # 0
        layout.addWidget(QtWidgets.QLabel("--------------------------------------------------------------------------------")) # 2
        layout.insertSpacerItem(2, QtWidgets.QSpacerItem(5, 10)) # 2

        # attributes object
        h_layout2 = QtWidgets.QHBoxLayout()
        self.WIDGETS["blendShapeLabel"] = QtWidgets.QLabel("Please choose the available blend-shapes.")
        self.WIDGETS['blendShapeComboBox'] = QtWidgets.QComboBox()
        self.WIDGETS['blendShapeComboBox'].setContextMenuPolicy(QtCore.Qt.ActionsContextMenu)
        self.add_menu_actions(self.WIDGETS['blendShapeComboBox'])
        self.populate_blendshapes()
        h_layout2.addWidget(self.WIDGETS["blendShapeLabel"], 0)
        h_layout2.addWidget(self.WIDGETS['blendShapeComboBox'], 1)

        # attributes object
        h_layout3 = QtWidgets.QHBoxLayout()
        self.WIDGETS["blendShaoeAttributesLabel"] = QtWidgets.QLabel("Please select available blend-shape attributes")
        self.WIDGETS['blendShaoeAttributesComboBox'] = QtWidgets.QComboBox()
        h_layout3.addWidget(self.WIDGETS["blendShaoeAttributesLabel"], 0)
        h_layout3.addWidget(self.WIDGETS['blendShaoeAttributesComboBox'], 1)

        layout.addLayout(h_layout2) # 3
        layout.addLayout(h_layout3) # 4
        layout.addWidget(QtWidgets.QLabel("Connect the blendShape with one of these attributes: "))  # 6
        layout.insertSpacerItem(5, QtWidgets.QSpacerItem(4, 10)) # 5

        self.WIDGETS['performConnectionButton'] = QtWidgets.QPushButton("Connect the attributes.")
        h_layout4 = QtWidgets.QHBoxLayout()
        self.WIbutton_group = QtWidgets.QButtonGroup()
        self.WIDGETS["attributesButtonGroup"] = QtWidgets.QButtonGroup()
        translateX = QtWidgets.QCheckBox("translateX")
        translateX.setChecked(True)
        translateY = QtWidgets.QCheckBox("translateY")
        translateZ = QtWidgets.QCheckBox("translateZ")

        rotateX = QtWidgets.QCheckBox("rotateX")
        rotateY = QtWidgets.QCheckBox("rotateY")
        rotateZ = QtWidgets.QCheckBox("rotateZ")
        self.WIDGETS["attributesButtonGroup"].addButton(translateX)
        self.WIDGETS["attributesButtonGroup"].addButton(translateY)
        self.WIDGETS["attributesButtonGroup"].addButton(translateZ)
        self.WIDGETS["attributesButtonGroup"].addButton(rotateX)
        self.WIDGETS["attributesButtonGroup"].addButton(rotateY)
        self.WIDGETS["attributesButtonGroup"].addButton(rotateZ)
        h_layout4.addWidget(translateX)
        h_layout4.addWidget(translateY)
        h_layout4.addWidget(translateZ)
        h_layout4.addWidget(rotateX)
        h_layout4.addWidget(rotateY)
        h_layout4.addWidget(rotateZ)
        layout.addLayout(h_layout4)
        layout.addWidget(self.WIDGETS['performConnectionButton'])

        self.setLayout(layout)
        self.connect_buttons()
        self.populate_shapes()

    def add_menu_actions(self, widget):
        """
        adds menu actions.
        """
        select = QtWidgets.QAction(self)
        select.setText("Select")
        select.triggered.connect(self.select_blendshape)
        widget.addAction(select)
        return True

    def select_blendshape(self):
        """
        selects the blendshape.
        """
        data = self.get_text_data()
        object_utils.select_object(data['blendShapeNode'])
        return True

    def populate_menu_bar(self):
        """
        populates the menu bar with options
        """
        file_menu = self.menuBar().addMenu('&BlendShape')
        file_menu.addAction("&Select BlendShape Node")
        return True

    def connect_buttons(self):
        """
        connects the abailable buttons
        """
        self.WIDGETS['lineEditCtrlAddButton'].clicked.connect(self.populate_controller)
        self.WIDGETS['performConnectionButton'].clicked.connect(self.connect_blendshape_attribute)

    def get_text_data(self):
        """
        get text data from the UI
        :return: <dict> return the line edit, comboBox text data as dictionary
        """
        text_data = {}
        text_data['controllerLineEdit'] = self.WIDGETS['controllerLineEdit'].text()
        item_idx = self.WIDGETS['blendShapeComboBox'].currentIndex()
        text_data['blendShapeNode'] = self.WIDGETS['blendShapeComboBox'].itemText(item_idx)
        item_idx = self.WIDGETS['blendShaoeAttributesComboBox'].currentIndex()
        text_data['blendShapeNodeAttributes'] = self.WIDGETS['blendShaoeAttributesComboBox'].itemText(item_idx)
        text_data['attributeName'] = self.WIDGETS['attributesButtonGroup'].checkedButton().text()
        return text_data

    def connect_blendshape_attribute(self):
        """
        make a dynamic fk chain through selection.
        :return: <bool> True for success.
        """
        data = self.get_text_data()
        controller_text = data['controllerLineEdit']
        driver_attrib_text = data['attributeName']
        blend_node_text = data['blendShapeNode']
        blend_shape_attribute = data['blendShapeNodeAttributes']
        print(controller_text, driver_attrib_text, ' >>> ', blend_node_text, blend_shape_attribute)
        animation_utils.connect_anim(controller_text, driver_attrib_text, blend_node_text, blend_shape_attribute)
        return True

    def populate_blendshapes(self):
        """
        populates the controller text edit box.
        :return:
        """
        blend_shape_nodes = blendshape_utils.get_scene_blendshapes(as_strings=True)
        self.WIDGETS['blendShapeComboBox'].addItems(blend_shape_nodes.values())

    def populate_shapes(self):
        """
        populates the controller text edit box.
        :return:
        """
        data = self.get_text_data()
        blend_node_text = data['blendShapeNode']
        shape_names = blendshape_utils.get_shapes(blend_node_text)
        self.WIDGETS['blendShaoeAttributesComboBox'].addItems(shape_names)

    def populate_controller(self):
        """
        populates the controller text edit box.
        :return:
        """
        selected_object = object_utils.get_selected_node(single=True)
        self.WIDGETS['controllerLineEdit'].setText(selected_object)
        self.setWindowTitle("{} :: {}".format(window_title, selected_object))

    def close(self):
        self.deleteLater()

def open_it():
    """
    opens the tool in whatever application we are using.
    :return: <bool> True for success.
    """
    global open_windows
    if window_title in open_windows:
        open_windows[window_title].close()
    open_windows[window_title] = Form(parent=ui_utils.get_maya_parent_window())
    open_windows[window_title].show()
    return open_windows


"""trydjango URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from pages import views
from product.views import *
from django.contrib.auth import views as auth_views

urlpatterns = [
	path('', views.home_view, name='home_view'),
	path('search/', views.search_view, name= 'search_view'),
	path('price/', price_view, name ='price_view'),
	path('load/', load_base_view, name ='load_base_view'),
	path('modify/', modify_product_view, name ='modify_product_view'),
    path('delete/', delete_single_price_view, name ='delete_single_price'),
    path('update_single/', update_single_price_view, name ='update_single_price'),
    path('update/<int:position_id>/', update_price_view, name ='update'),
    path('from_file/', modify_price_from_file_view, name ='modify_from_file_view'),
    path('admin/', admin.site.urls),
    path('', include("django.contrib.auth.urls")),
    #path('', auth_views.LoginView ,{'template_name' : 'core/login.html'}, name='login'),
]


import pysal as ps
import pandas as pd
import numpy as np
import os
import csv

#function that averages all the values inside the uni id zones
#it gets the sum of rows, columns and zones that both rows and columns are in uni id list
#set all the values in targeted ids to 0
#set super zone rows, columns and intrazone to correspodning values
#get average value of rows, columns and superzone.
def avg_uni(matrix, uni_ids, univ_super):
    row_value = matrix.loc[uni_ids].sum()
    col_value = matrix[uni_ids].sum(axis = 1)
    in_zones =  matrix.ix[uni_ids, uni_ids].values.sum()

    matrix.loc[uni_ids] = 0
    matrix[uni_ids] = 0
    matrix.ix[uni_ids, uni_ids] = 0

    matrix.loc[univ_super] = row_value
    matrix[univ_super] = col_value
    matrix.ix[uni_ids, univ_super] = 0
    matrix.ix[univ_super, uni_ids] = 0

    matrix.set_value(univ_super, univ_super, in_zones)

    matrix.loc[univ_super] = matrix.loc[univ_super] / len(uni_ids)
    matrix[univ_super] = matrix[univ_super] / len(uni_ids)
    return matrix

#boolean to indentify if the model run is for the submodel unm dc_mc_property
#If true return True and ends function scope
#else retrun False
def is_uni(id):
    if id in [3,4]:
        return True
    return False

# Creates a df from a dbf
def dbf2df(dbfile, lower=True):  # Reads in DBF files and returns Pandas DF
    db = ps.open(dbfile)  # Pysal to open DBF
    d = {col: db.by_col(col) for col in db.header}  # Convert dbf to dictionary
    pandasDF = pd.DataFrame(d)  # Convert to Pandas DF
    if lower: #Make columns uppercase if wanted
        pandasDF.columns = pandasDF.columns.str.lower()
    db.close()
    return pandasDF

# functions that reads a txt file and return a dataframe matrix
# it creates a 919x919 dataframe
# reads the input file
# updates the 919 df with file values
def load_skim(skim, frame):
    with open(skim) as f:
        all_zones = pd.DataFrame(0.0, index=frame, columns=frame)
        df = pd.read_table(f, delim_whitespace=True,
                           dtype={'A': np.int64,
                                  'B': np.int64, 'C': np.float64},
                           names=('A', 'B', 'C'))
        df = df.pivot(index="A", columns="B", values="C")
        df = df.rename_axis(None)
        df.columns.name = None
        all_zones.update(df)
    return all_zones

# Retruns the name of the file, no extensions and path
def get_file_name(path):
    return os.path.basename(path).split(".")[0].strip().lower()

# Reads input files and returns the proper dataframe
# after reding the file it lower all column names
# set index to passed field
# updates to a dataframe with the  919 zones from 14 to 907
def read_file(file_input, frame, index_field=None):
    filename, file_extension = os.path.splitext(file_input)
    if file_extension == ".dbf":
        df = dbf2df(file_input)

    elif file_extension == ".csv":
        df = pd.read_csv(file_input)

    try:
        df.columns = map(str.lower, df.columns.values)
        df.index = df[index_field].astype(int)
        df.sort_index()
        all_zones = pd.DataFrame(0.0, index=frame, columns=df.columns.values)
        all_zones.update(df)
        return all_zones
    except ValueError:
        df = df[df[index_field] != 0]
        df.index = df[index_field].astype(int)
        df = df.rename_axis(None)
        return df

# creates a zeries of 1 and 0 depending if the zone is a model zone or not
# currenly zone models are 14 to 907
def get_model_zones(start=1, end=917, model_start=14, model_end=919):
    model_zones_list = []
    for i, x in enumerate(range(start, end + 1), 1):
        if model_start <= i <= model_end:
            z = 1
        else:
            z = 0
        model_zones_list.append(z)
    return pd.Series(data=model_zones_list, index=[i for i in range(start, end + 1)])

# set density variable
def set_denisty_variables(df, variable, area, numbin=4):
    # 1- it creates a new density column by dividing the vari_name by area.
    # 1.a vari_name comes from the dc_mc_property.xml Create_Density_Quartile_Variable tag.
    # 1.b area comes from the dc_mc_property.xml Area_Variable_Name tag.
    df.density = df['{}'.format(variable)].div(df['{}'.format(area)]).fillna(0)

    # 2 creates an ascending sorted list of the density field
    dsort = df.density.copy()
    dsort.sort_values(inplace=True)
    dsort = dsort.tolist()

    # Step3: Find density values at the location of 25%, 50%, and 75% in dsort
    # for bins it uses the number of rows of the socio table / num of bins
    bins = len(df.index) / numbin
    # treshold is  a list of 4 ranges value, each value is depending on evry
    # bins value.
    threshold = dsort[::bins]
    # by default is uses the last value of dsort +1 to ensure that we have a
    # max number for comparison
    threshold[-1] = dsort[-1] + 1
    return df.density, threshold

def emp_within(min, source, emp_vari, frame):
    empwithin = pd.Series(data=0.0, index=frame)
    for index, row in source.iterrows():
        temp = ((source.loc[index] > 0) & (source.loc[index] < min)) * 1
        indices = [key for key, value in temp.iteritems() if value == 1]
        try:
            numemp = emp_vari[indices].sum()
            empwithin[index] = np.float(numemp)
        except IndexError:
            empwithin[index] = np.float(0.0)
    return empwithin.divide(emp_vari.sum()).fillna(0)

# set density variable
# uses a threshold time such as 30 or 50 min
# It adds different skims to get total time stored in timedata
# temp is a pd series of 1 or 0 depending if the timedata.row[i] is less than the min threshold
# indices is a list that holds all the zones that are whitin the time treshold
# empvari is a series of the tot_emp field in the socio table. it sums all the emp inside indices
# for example if zone 17 and 18 are accesible within 30 min, it get totemp
# for zone 17 and 18 as empwithin
def set_accessibility_variables(min, matrices, skims_vari, emp_vari, frame):
    timedata = pd.DataFrame(0.0, index=frame, columns=frame)
    for i in skims_vari:
        timedata += matrices[i]
    empwithin = emp_within(min, timedata, emp_vari, frame)
    return empwithin

# return the model attributes from the xml file
#is uni and enrollment attributes needed for uni submodel
# enrollmnet values come from the dc_property xml via model enrollment value
def get_model_attributes(model, doubly=None, is_uni = False, enr_val = None):
    name = model.get('Name')
    model_id = model.get('Id')
    model_desc = model.get('Description')
    if is_uni:
        enroll_type =  model.get("Enrollment").lower()
        enrollment = enr_val[enroll_type]
        trip_rate = float(model.get("TripRate"))
        trip_count = trip_rate * enrollment
        if model.attrib.get("PandA") == 'Attraction':
            isPro = False
        else:
            isPro = True

        return {"name": model.get('Name'), "model_id": model.get('Id'), "model_desc": model_desc,
                "enrollment": enrollment, "isProduct": isPro, "trip_rate": trip_rate, "trip_count": trip_count}
    else:
        if model.get("Productions"):
            product = model.get("Productions").split(",")
            isPro = True
            if model.get("Attractions"):
                doubly = model.get("Attractions").split(",")
        else:
            product = model.get("Attractions").split(",")
            isPro = False

        return {"name": model.get('Name'), "model_id": model.get('Id'), "model_desc": model_desc,
                "product_columns": product, "isProduct": isPro, "doubly_columns": doubly}

# function that aggregates either productions or attracations based of product or doubly columns
# the columns come from the xml file through the get model attribute function
def get_panda(columns, panda_table, taz_start_id, taz_end_id):
    serie = pd.Series(
        data=0.0, index=[i for i in range(taz_start_id, taz_end_id + 1)])
    for col in columns:
        serie += panda_table[col.lower()]
    return serie

# Function to balance a matrix by doing an IPF
# it will balance rows for productions
# it will balance cols for attractions
def balancing_matrix(matrix, product, attract, log):
    tol = 0.005
    maxiter = 20
    relative = 1.0
    numzones = len(matrix.index)
    numiter = 0
    result = matrix

    while numiter < maxiter and relative > tol:

        normal_row = nomral_factor(result, attract)
        result = result.mul(normal_row, axis=1)

        log.msg("Iteration {}".format(numiter))
        normal_col = nomral_factor(result, product, index=1)
        result = result.mul(normal_col, axis=0)

        relative = computeErrors(result, attract,log)

        numiter += 1

    return result.fillna(0)

# Function used in the balancing matrix functions
# it gets the row or columns total depending on the sum index.
# index=0 means sumrows and index=1 means colsums
# it divides prod or attractions by sumtotal depending.
def nomral_factor(matrix, target, index=0):
    np.seterr(invalid='ignore')
    targetsum = matrix.sum(axis=index)
    return target.divide(targetsum).fillna(0)

# get the absolute and relative error for the ipf process
def computeErrors(matrix, attract,log):
    rowsum = matrix.sum(axis=0)
    adifference = pd.Series.abs(rowsum - attract)
    lowvalue = np.minimum(rowsum, attract)
    rdifference = adifference.div(lowvalue).fillna(0).max()
    log.msg("Relative error: {}".format(rdifference))
    log.msg("Absolute error: {}".format(adifference.max()))
    return rdifference

# function that aggregates any dataframe by area types.
# it groups by row then it groups by column
# manipulates the dataframe to exclude the 0 group which is usally non existent
def aggregated_result(matrix, group):
    max_group = int(group.max())
    if int(group.min()) <= 0:
        min_group = 1
    else:
        min_group = int(group.min())

    matrix = matrix.groupby(group).sum().groupby(group, axis=1).sum()
    matrix = matrix.rename_axis(None)
    matrix.columns = matrix.columns.astype(int)
    matrix.index = matrix.index.astype(int)
    return matrix.ix[:, min_group:max_group].loc[min_group:max_group]

def aggregated_result_validation(result, dis_mtx, socio, output_name, is_uni = False, superzone = None):
    #get index of the / to add aggregate to the name
    sindex = output_name.rfind("/", 1)
    output_file = output_name[:sindex + 1] + \
        '_aggregated_' + output_name[sindex + 1:]

    bins = {1: [0.0, 1.0], 2: [1.0, 2.5], 3: [2.5, 5.0], 4: [5.0, 7.5], 5: [7.5, 10.0], 6: [10.0, 15.0], 7: [15.0, 20.0],
            8: [20.0, 25.0], 9: [25.0, 30.0], 10: [30.0, 35.0], 11: [35.0, 40.0], 12: [40.0, 45.0], 13: [45.0, 30000.0]}

    labels = {1: '<=1.0', 2: '1.0>,<=2.5', 3: '2.5>,<=5.0', 4: '5.0>,<=7.5', 5: '7.5>,<=10', 6: '10>,<=15',
              7: '15>,<=20', 8: '20>,<=25', 9: '25>,<=30', 10: '30>,<=35', 11: '35>,<=40', 12: '40>,<=45', 13: '>45'}

    if is_uni:
        aggregated_result_dc_uni(result,output_file,dis_mtx,socio,bins,labels,superzone)
    else :
        aggregated_result_dc(result,output_file,dis_mtx,socio,bins,labels)

#Writes summary report for the aggregated pk,op files
def  aggregated_result_dc_uni(result, output_file, dis_mtx, socio, bins, labels, superzone):
    with open(output_file, "wb") as f:
        writer = csv.writer(f)
        ### finish this later. i dont think result series need to be transposed
        if result[superzone].sum() < result.loc[superzone].sum():
            result_series = result.loc[superzone]
            dis_mtx = dis_mtx.loc[superzone]
        else:
            result_series = result[superzone]
            dis_mtx = dis_mtx[superzone]

        # Writes distance summary
        for key in bins.keys():
            bin_ = bins[key]
            label = labels[key]
            temp = result_series.mul(((dis_mtx <= bin_[1]) & (dis_mtx > bin_[0])) * 1)
            trips = temp.values.sum()
            if key == 1:
                trips = result_series[superzone]
            if key == 2:
                trips = trips - result_series[superzone]
            writer.writerow([label,trips])
        intra_camp_trips = result_series.ix[superzone,superzone]
        writer.writerow(["Intra_Campus",intra_camp_trips])

        writer.writerow([])
        writer.writerow([])

        # Wrties aggregation by district summary
        agg_district = aggregated_result(result, socio.district)
        writer.writerow(["district"] + agg_district.columns.tolist())
        for row in agg_district.iterrows():
            index, data = row
            writer.writerow([index] + data.tolist())

        writer.writerow([])
        writer.writerow([])

        # Wrties aggregation by area type
        #since there a two county fields in the socio df, the country filed with int
        #is county_y. since it was read second. in order to change this, county from the socio dbf
        #needs have a different name
        agg_areatypes = aggregated_result(result, socio.county_y)
        writer.writerow(["county"] + agg_areatypes.columns.tolist())
        for row in agg_areatypes.iterrows():
            index, data = row
            writer.writerow([index] + data.tolist())

#Writes summary report for the aggregated pk,op files
def  aggregated_result_dc(result,output_file,dis_mtx,socio,bins,labels):
    with open(output_file, "wb") as f:
        writer = csv.writer(f)
        # Writes distance summary
        for key in bins.keys():
            bin_ = bins[key]
            label = labels[key]
            temp = result.mul(((dis_mtx <= bin_[1]) & (dis_mtx > bin_[0])) * 1)
            trips = temp.values.sum()
            writer.writerow([label, trips])
        total_miles = result * dis_mtx
        avg_miles = total_miles.values.sum() / result.values.sum()
        writer.writerow(['average distance', avg_miles])

        writer.writerow([])
        writer.writerow([])

        # Wrties aggregation by district summary
        agg_district = aggregated_result(result, socio.district)
        writer.writerow(["district"] + agg_district.columns.tolist())
        for row in agg_district.iterrows():
            index, data = row
            writer.writerow([index] + data.tolist())

        writer.writerow([])
        writer.writerow([])

        # Wrties aggregation by area type
        agg_areatypes = aggregated_result(result, socio["at"])
        writer.writerow(["areatypes"] + agg_areatypes.columns.tolist())
        for row in agg_areatypes.iterrows():
            index, data = row
            writer.writerow([index] + data.tolist())

#mode choice functions to determine walk times and shares from zone to zone
def intra_walk_share(zone, df, field):
    val = df[field].loc[df.index == zone].values[0]
    temp = df[field] * val
    return temp.values.tolist()

def external_walk_share(zone, df, field1, field2):
    val1 = df[field1].loc[df.index == zone].values[0]
    val2 = df[field2].loc[df.index == zone].values[0]
    temp = df[field2] * val1 +  df[field1] * val2
    return temp.values.tolist()

def intra_walk_time(zone, df, field):
    val = df[field].loc[df.index == zone].values[0]
    temp = df[field] + val
    return temp.values.tolist()

def external_walk_time(zone, df, field1, field2, field3, field4):
    val1 = df[field1].loc[df.index == zone].values[0]
    val2 = df[field2].loc[df.index == zone].values[0]
    val3 = df[field3].loc[df.index == zone].values[0]
    val4 = df[field4].loc[df.index == zone].values[0]
    temp = ((df[field3] + val1) * (df[field4] * val2) + (df[field1] + val3)  * (df[field2] * val4))
    return temp.values.tolist()

def drive_access_transit_shares(zone, df, field):
    val = 1 - df.sh_share.loc[df.index == zone].values[0] - df.md_share.loc[df.index == zone].values[0] - df.lg_share.loc[df.index == zone].values[0]
    temp = val * df[field]
    return temp.values.tolist()

#create walking time,share base dataframe
def create_walking_share_time_df(d):
    df = pd.DataFrame(index=pd.MultiIndex.from_product([d.index, d.index])).reset_index()
    df.columns = ['i', 'j']

    col = ["s_s_walk_share" ,"s_s_walk_time","s_m_walk_share","s_m_walk_time",
           "s_l_walk_share","s_l_walk_time","m_m_walk_share","m_m_walk_time",
           "m_l_walk_share","m_l_walk_time","l_l_walk_share","l_l_walk_time",
           "d_access_t_short_walk_share","d_access_t_short_walk_time",
           "d_access_t_med_walk_share","d_access_t_med_walk_time",
           "d_access_t_long_walk_share","d_access_t_long_walk_time","drive_only"]

    for c in col:
        df[c] = -99

    return df

#Function that deletes success file to run python script
def del_checker(file):
    filename = os.path.join(file, "01_Inputs/common/success.txt")
    if os.path.exists(filename):
        os.remove(filename)

def create_checker(file):
    filename = os.path.join(file, "01_Inputs/common/success.txt")
    with open(filename,"w") as f:
        f.write("1")


from builtins import object
from bq_data_access.v2.clinical_data import CLINICAL_FEATURE_TYPE
from bq_data_access.v2.feature_search.common import InvalidFieldException, EmptyQueryException
from bq_data_access.data_types.clinical import BIGQUERY_CONFIG
from scripts.feature_def_gen.clinical_features import CLINDataSourceConfig
from bq_data_access.v2.schema.program_schemas import TABLE_TO_SCHEMA_MAP

class ClinicalSearcher(object):
    config_instance = CLINDataSourceConfig.from_dict(BIGQUERY_CONFIG)
    feature_search_valid_fields = ['keyword']

    @classmethod
    def get_datatype_identifier(cls):
        return CLINICAL_FEATURE_TYPE

    @classmethod
    def get_searchable_fields(cls):
        return [
            {'name': 'keyword',
             'label': 'Keyword',
             'static': False},
        ]

    @classmethod
    def build_feature_label(cls, column_name):
        name_parts = []
        for part in column_name.split('_'):
            name_parts.append(part.capitalize())
        human_readable_name = ' '.join(name_parts)
        return human_readable_name

    def filter_by_name(self, keyword):
        result = []
        found_columns = set()
        for table_config in self.config_instance.data_table_list:
            schema = TABLE_TO_SCHEMA_MAP[table_config.table_id]

            for field_item in schema:
                name = field_item['name']
                if (name not in found_columns) and (name.find(keyword.replace(' ', '_').lower()) != -1):
                    found_columns.update([name])
                    result.append(field_item)

        return result

    def validate_feature_search_input(self, parameters):
        # Check that the input contains only allowed fields
        for field, keyword in parameters.items():
            if field not in self.feature_search_valid_fields:
                raise InvalidFieldException(self.get_datatype_identifier(), keyword, field)

        # At least one field has to have a non-empty keyword
        found_field = False
        for field, keyword in parameters.items():
            if len(keyword) > 0:
                found_field = True
                continue

        if not found_field:
            raise EmptyQueryException(self.get_datatype_identifier())

    def search(self, parameters):
        self.validate_feature_search_input(parameters)
        search_result = self.filter_by_name(parameters['keyword'])

        found_features = []
        for feature_item in search_result:
            column_name = feature_item['name']
            human_readable_name = self.build_feature_label(column_name)
            internal_id = 'v2:CLIN:' + column_name
            value_type = "N"

            if feature_item['type'] == "STRING" :
                value_type = "C"

            found_features.append({
                'feature_type': 'CLIN',
                'column_name': column_name,
                'internal_feature_id': internal_id,
                'label': human_readable_name,
                'type': value_type
            })

        return found_features


import time

from starlette.middleware.base import BaseHTTPMiddleware
from starlette.middleware.base import RequestResponseEndpoint
from starlette.requests import Request
from starlette.responses import Response
from starlette.types import ASGIApp

from opencopilot.logger import api_logger
from opencopilot.service.error_responses import APIErrorResponse
from opencopilot.service.middleware import util
from opencopilot.service.middleware.entities import RequestStateKey
from opencopilot.utils.http_headers import add_response_headers

logger = api_logger.get()

class MainMiddleware(BaseHTTPMiddleware):
    def __init__(self, app: ASGIApp) -> None:
        super().__init__(app)

    async def dispatch(
        self, request: Request, call_next: RequestResponseEndpoint
    ) -> Response:
        request_id = util.get_state(request, RequestStateKey.REQUEST_ID)
        ip_address = util.get_state(request, RequestStateKey.IP_ADDRESS)
        country = util.get_state(request, RequestStateKey.COUNTRY)
        try:
            logger.info(
                f"REQUEST STARTED "
                f"request_id={request_id} "
                f"request_path={request.url.path} "
                f"ip={ip_address} "
                f"country={country} "
            )
            before = time.time()
            response: Response = await call_next(request)

            duration = time.time() - before
            process_time = (time.time() - before) * 1000
            formatted_process_time = "{0:.2f}".format(process_time)
            if response.status_code != 404:
                logger.info(
                    f"REQUEST COMPLETED "
                    f"request_id={request_id} "
                    f"request_path={request.url.path} "
                    f"completed_in={formatted_process_time}ms "
                    f"status_code={response.status_code}"
                )
            return await _get_response_with_headers(response, duration)
        except Exception as error:
            if isinstance(error, APIErrorResponse):
                is_exc_info = error.to_status_code() == 500
                logger.error(
                    f"Error while handling request. request_id={request_id} "
                    f"request_path={request.url.path}"
                    f"status code={error.to_status_code()}"
                    f"code={error.to_code()}"
                    f"message={error.to_message()}",
                    exc_info=is_exc_info,
                )
            else:
                logger.error(
                    f"Error while handling request. request_id={request_id} "
                    f"request_path={request.url.path}",
                    exc_info=True,
                )
            raise error

async def _get_response_with_headers(response, duration):
    response = await add_response_headers(response, duration)
    return response


import requests
from bs4 import BeautifulSoup
import tkinter
from tkinter.ttk import *
from PIL import Image, ImageTk

def dir_link_finder(url):
    dir_link_list = list()
    dir_link_list.append(url)
    req = requests.get(url)
    repo_link_protocol = url[:8]
    link_divide = url.split("/")
    domain_name = link_divide[2]
    soup = BeautifulSoup(req.content, "html.parser")
    all_hyperlink_tags = soup.find_all('a')
    for hyper_link_tags in all_hyperlink_tags:
        str_hyper_link_tag = str(hyper_link_tags)
        if f'{link_divide[3]}/{link_divide[4]}/tree/master/' in str_hyper_link_tag:
            split_file_tags = str_hyper_link_tag.split(' ')
            for href in split_file_tags:
                if href[0:4] == "href":
                    link = href[6:-1]
                    dir_link_list.append(repo_link_protocol + domain_name + link)
    return dir_link_list

def code_link_finder(url_list: list):
    code_links = list()
    for url in url_list:
        req = requests.get(url)
        repo_link_protocol = url[:8]
        link_divide = url.split("/")
        domain_name = link_divide[2]
        repo_name = f'/{link_divide[3]}/{link_divide[4]}'
        soup = BeautifulSoup(req.content, "html.parser")
        code_extensions = [".py", ".cpp", ".java", "json", ".c"]

        all_hyperlink_tags = soup.find_all('a')
        for hyperlink_tag in all_hyperlink_tags:
            for ext in code_extensions:
                str_hyperlink_tag = str(hyperlink_tag)
                if ext in str_hyperlink_tag and not (".com" in str_hyperlink_tag):
                    split_a_tag = str_hyperlink_tag.split(' ')
                    for href in split_a_tag:
                        if href[0:4] == "href":
                            link_1 = href[6:-1]
                            code_links.append(repo_link_protocol + domain_name + link_1)
        for link_2 in code_links:
            if f'{repo_link_protocol + domain_name + repo_name}/blob/master/' in link_2:
                pass
            else:
                code_links.remove(link_2)
    return code_links

def open_code(code_link_list: list):  # will receive Code links from code_link_finder function
    program = []
    for code_link in code_link_list:
        req = requests.get(code_link)
        soup = BeautifulSoup(req.content, "html.parser")
        code = soup.text
        code_1 = code.split('\n')  # to ignore the '\n' char in the variable code
        code_2 = [ele for ele in code_1 if ele.strip()]  # to ignore the empty or blank elements in variable code_1
        for i in range(len(code_2)):
            found = 0
            if "View blame" in code_2[i]:
                found = True
            while found:
                if "Copy lines" in code_2[i]:
                    found = False
                if code_2[i] == "                View blame" or code_2[i] == "            Copy lines":
                    pass
                else:
                    program.append(code_2[i])
                i = i + 1
    return program

# Function Which clear the frame
def clear_frame(frame):
    for widgets in frame.winfo_children():
        widgets.destroy()

# Function to Display all the Code in the Repo
def display_submit_func(frame, link):
    code = open_code(code_link_finder(dir_link_finder(link)))
    vul_syn_list, vul_des_list, vul_rating_list = return_vul(link)
    clear_frame(frame)
    vertical_scrollbar = tkinter.Scrollbar(frame, orient=tkinter.VERTICAL)
    vertical_scrollbar.grid(column=1, sticky='ns')
    text_box = tkinter.Text(frame, yscrollcommand=vertical_scrollbar.set)
    for i in range(len(code)):
        text_box.insert(float(i + 1), code[i] + '\n')
    vertical_scrollbar.config(command=text_box.yview)
    text_box.grid(column=0, row=0, sticky='nsew', pady=5, padx=5)
    # Rate Repo Button
    rate_button = tkinter.Button(frame, text='Rate Repo',
                                 command=lambda: rate_repo_submit(frame, vul_rating_list))
    rate_button.grid(row=1, column=0, sticky='s')

# Function which take GitHub URl as input for displaying the Code in the Repo
def display_code(frame):
    def temp_text(e):
        url_entry.delete(0, "end")

    clear_frame(frame)  # clear the widgets inside the frame

    # input URL variable
    input_repo_url = tkinter.StringVar()

    # Right frame for working palate
    right_frame = tkinter.Frame(frame)
    right_frame.grid(sticky='n')

    # Right frame display frame column configure
    right_frame.columnconfigure(0, weight=10)
    right_frame.columnconfigure(1, weight=1)

    # Instruction Label
    title_label = tkinter.Label(right_frame, text="To Display All the Code in the Repo", font=('Arial', 20))
    title_label.grid(row=0, columnspan=2)
    github_link_label = tkinter.Label(right_frame, text="Enter the Repo link Below")
    github_link_label.grid(row=1, column=0, columnspan=2)
    url_entry = tkinter.Entry(right_frame, textvariable=input_repo_url, width=50)
    url_entry.insert(0, "Enter the Github URL of the Repo")
    url_entry.grid(row=2, column=0)
    url_entry.bind("<FocusIn>", temp_text)
    url_entry.update()
    submit_button = tkinter.Button(right_frame, text="Submit", width=5,
                                   command=lambda: display_submit_func(frame, input_repo_url.get()))
    submit_button.grid(row=2, column=1)

# this adds the vulnerabilities into the file
def vul_submit_func(syn, des, rate, frame_1):
    f = open("vulnerable_syntax.txt", "a")
    f.write(f'{syn.get()}, {des.get()}, {rate.get()}\n')
    f.close()
    add_vulnerable_syntax(frame_1)

# Shows when Add Vulnerability is Clicked
def add_vulnerable_syntax(frame):
    def syn_temp_text(w):
        syntax_entry.delete(0, "end")

    def desc_temp_text(w):
        desc_entry.delete(0, "end")

    def rate_temp_text(w):
        danger_rating_entry.delete(0, "end")

    clear_frame(frame)

    # Entry Variable declaration
    syntax = tkinter.StringVar()
    desc = tkinter.StringVar()
    rate_syn = tkinter.StringVar()

    # right frame for adding widgets
    right_frame = tkinter.Frame(frame)
    right_frame.grid(sticky='nsew', padx=5)

    # right frame column Configuration
    right_frame.columnconfigure(0, weight=1)
    right_frame.columnconfigure(1, weight=10)

    # title label and all functionality widgets
    title_label = tkinter.Label(right_frame, text='Add Vulnerable Syntax', font=('Arial', 20))
    title_label.grid(row=0, column=0, columnspan=2, sticky='ew', pady=5, padx=5)

    # Form Frame
    form_frame = tkinter.Frame(right_frame)
    form_frame.grid(row=1, column=1, pady=5, padx=5, sticky='ns')

    # Syntax
    syntax_label = tkinter.Label(form_frame, text='Syntax', font=('Arial', 15))
    syntax_label.grid(column=0, row=0, sticky='e', padx=5, pady=5)

    # Syntax Entry
    syntax_entry = tkinter.Entry(form_frame, textvariable=syntax, width=50)
    syntax_entry.insert(0, "Enter the Vulnerable Syntax")
    syntax_entry.grid(column=1, row=0, sticky='w', padx=5, pady=5)
    syntax_entry.bind("<FocusIn>", syn_temp_text)

    # Description
    desc_label = tkinter.Label(form_frame, text='Description', font=('Arial', 15))
    desc_label.grid(column=0, row=1, sticky='e', padx=5, pady=5)

    # Description Entry
    desc_entry = tkinter.Entry(form_frame, textvariable=desc, width=50)
    desc_entry.insert(0, "Describe the Vulnerability")
    desc_entry.grid(column=1, row=1, sticky='w', padx=5, pady=5)
    desc_entry.bind("<FocusIn>", desc_temp_text)

    # Rating
    danger_rating_label = tkinter.Label(form_frame, text='Syntax Danger Rating', font=('Arial', 15))
    danger_rating_label.grid(column=0, row=2, sticky='e', padx=5, pady=5)

    # Danger Rating Entry
    danger_rating_entry = tkinter.Entry(form_frame, textvariable=rate_syn, width=50)
    danger_rating_entry.insert(0, "Severity of Syntax 1: Low, 2: Medium, 3: High")
    danger_rating_entry.grid(column=1, row=2, sticky='w', padx=5, pady=5)
    danger_rating_entry.bind("<FocusIn>", rate_temp_text)

    frame.update()

    # Submit Button
    submit_button = tkinter.Button(form_frame, text='Submit',
                                   command=lambda: vul_submit_func(syntax, desc, rate_syn, frame))
    submit_button.grid(column=0, row=4, columnspan=2, pady=5)

# Function for UI for taking GitHub URL as input to check Repo for Vulnerability
def check_repo_code(frame):
    def temp_text(e):
        url_entry.delete(0, "end")

    clear_frame(frame)  # clear the widgets inside the frame

    # input URL variable
    input_repo_url = tkinter.StringVar()

    # Right frame for working palate
    right_frame = tkinter.Frame(frame)
    right_frame.grid(sticky='n')

    # Right frame display frame column configure
    right_frame.columnconfigure(0, weight=10)
    right_frame.columnconfigure(1, weight=1)

    # Instruction Label
    title_label = tkinter.Label(right_frame, text="To check for Vulnerability in the Repo", font=('Arial', 20))
    title_label.grid(row=0, columnspan=2)

    # Github Link Label
    github_link_label = tkinter.Label(right_frame, text="Enter the Repo link Below")
    github_link_label.grid(row=1, column=0, columnspan=2)

    # URL Entry Label
    url_entry = tkinter.Entry(right_frame, textvariable=input_repo_url, width=50)
    url_entry.insert(0, "Enter the Github URL of the Repo")
    url_entry.grid(row=2, column=0)
    url_entry.bind("<FocusIn>", temp_text)
    url_entry.update()

    # Submit Button
    submit_button = tkinter.Button(right_frame, text="Submit", width=5,
                                   command=lambda: check_repo(input_repo_url.get(), frame))
    submit_button.grid(row=2, column=1)

# Function which returns the Vulnerabilities
def return_vul(link):
    f = open("vulnerable_syntax.txt", "r")
    code = open_code(code_link_finder(dir_link_finder(link)))
    a = f.read().split('\n')
    a.pop()
    b = list()
    vul_syn_list = list()
    vul_des_list = list()
    vul_rating_list = list()
    for i in range(len(a)):
        if a[i] == '':
            pass

        else:
            c = tuple(a[i].split(', '))
            b.append(c)
            syn, des, rate = b[i]
            for syntax in code:
                if syn in syntax and syn not in vul_syn_list:
                    vul_syn_list.append(syn)
                    vul_des_list.append(des)
                    vul_rating_list.append(rate)
    return vul_syn_list, vul_des_list, vul_rating_list

# Function to Show the Vulnerabilities in the Repo
def check_repo(link, frame):
    vul_syn_list = list()
    vul_des_list = list()
    vul_rating_list = list()
    vul_syn_list, vul_des_list, vul_rating_list = return_vul(link)

    # Clear the Frame
    clear_frame(frame)

    if len(vul_syn_list) == 0:
        zero_vul_label = tkinter.Label(frame, text="The Repo is safe and have Zero Vulnerabilities", fg='green',
                                       font=('Arial', 20))
        zero_vul_label.grid()
    else:
        # Text Box for Vulnerabilities
        text_box = tkinter.Text(frame, font=('Arial', 15))
        text_box.insert(1.0, f'Following are the Vulnerabilities in the {link} Repo\n')
        text_box.insert(2.0, '\n')
        i = 3.0
        syn_max_length = len(max(vul_syn_list)) + 5
        des_max_length = len(max(vul_des_list)) + 5
        for index in range(len(vul_syn_list)):
            text_box.insert(i, f'{vul_syn_list[index]: >{syn_max_length}s}:-- {vul_des_list[index]: ^{des_max_length}s}\n')
            i = i + 1
        text_box.grid(row=0, column=0, sticky='nsew', padx=5, pady=5)

        # Rate Repo Button
        rate_button = tkinter.Button(frame, text='Rate Repo',
                                     command=lambda: rate_repo_submit(frame, vul_rating_list))
        rate_button.grid(row=1, column=0, sticky='s')

# Function which show the Rating the for the Repo
def rate_repo_submit(frame, rating_list):
    if len(rating_list) == 0:
        rate_label = tkinter.Label(frame, text="The Repo is safe and have Zero Vulnerabilities", fg='green',
                                   font=('Arial', 20))
        rate_label.grid()
    else:
        rate = 0.0
        for r in rating_list:
            rate = rate + int(r)
        rate = float(rate / len(rating_list))
        rate_value = (rate / 3.0) * 100
        lower_frame = tkinter.Frame(frame)
        lower_frame.grid(sticky='s')
        # Rate Heading
        rate_label_heading = tkinter.Label(lower_frame, text="The Repo have Vulnerabilities", font=('Arial', 20),
                                           fg='red')
        rate_label_heading.grid(row=0, column=0, sticky='n', columnspan=3)

        # Label to Show the rating
        rate_label = tkinter.Label(lower_frame, text=f'{rate:.3f}', font=('Arial', 20), fg='red')
        rate_label.grid(row=1, columnspan=3)

        # Progress bar
        good_label = tkinter.Label(lower_frame, text='Good: 0', fg='green')
        good_label.grid(row=2, column=0)
        progress_bar = Progressbar(lower_frame, orient=tkinter.HORIZONTAL, length=300, mode='determinate',
                                   value=rate_value)
        progress_bar.grid(row=2, column=1, sticky='n')
        bad_label = tkinter.Label(lower_frame, text='Bad: 3', fg='red')
        bad_label.grid(row=2, column=2)

# Function which calls the
def check_rate(link, frame):
    vul_syn_list, vul_des_list, vul_rating_list = return_vul(link)
    rate_repo_submit(frame, vul_rating_list)

# Function for UI to take GitHub URL as input for rating the Repo
def rate_repo_code(frame):
    def temp_text(e):
        url_entry.delete(0, "end")

    clear_frame(frame)  # clear the widgets inside the frame

    # input URL variable
    input_repo_url = tkinter.StringVar()

    # Right frame for working palate
    right_frame = tkinter.Frame(frame)
    right_frame.grid(sticky='ns')

    # Right frame display frame column configure
    right_frame.columnconfigure(0, weight=10)
    right_frame.columnconfigure(1, weight=1)

    # Instruction Label
    title_label = tkinter.Label(right_frame, text="To Rate the Repo", font=('Arial', 20))
    title_label.grid(row=0, columnspan=2)

    # Github Link Label
    github_link_label = tkinter.Label(right_frame, text="Enter the Repo link Below")
    github_link_label.grid(row=1, column=0, columnspan=2)

    # URL Entry Label
    url_entry = tkinter.Entry(right_frame, textvariable=input_repo_url, width=50)
    url_entry.insert(0, "Enter the Github URL of the Repo")
    url_entry.grid(row=2, column=0)
    url_entry.bind("<FocusIn>", temp_text)
    url_entry.update()

    # Submit Button
    submit_button = tkinter.Button(right_frame, text="Submit", width=5,
                                   command=lambda: check_rate(input_repo_url.get(), frame))
    submit_button.grid(row=2, column=1)

# Function to Display Team Information
def team_info_display(frame):
    clear_frame(frame)

    # Row Configure
    frame.rowconfigure(0, weight=1)
    frame.rowconfigure(1, weight=1)
    frame.rowconfigure(2, weight=1)
    frame.rowconfigure(3, weight=1)

    # Column Configure
    frame.columnconfigure(0, weight=1)
    frame.columnconfigure(1, weight=1)
    frame.columnconfigure(2, weight=1)
    frame.columnconfigure(3, weight=1)

    team_post = tkinter.Label(frame, text='Team Member', font=('Arial', 15))
    team_post.grid(column=0, row=0, sticky='n')
    member_pic = tkinter.Label(frame, text='Profile\nPicture', font=('Arial', 15))
    member_pic.grid(column=0, row=1, sticky='n')
    member_name = tkinter.Label(frame, text='Name', font=('Arial', 15))
    member_name.grid(column=0, row=2, sticky='n')
    institute_name = tkinter.Label(frame, text='Institute Name', font=('Arial', 15))
    institute_name.grid(column=0, row=3, sticky='n')
    passing_batch = tkinter.Label(frame, text='Passing Batch', font=('Arial', 15))
    passing_batch.grid(column=0, row=4, sticky='n')

    # LEADER

    # Leader Label
    leader_label = tkinter.Label(frame, text='Leader', font=('Arial', 15))
    leader_label.grid(row=0, column=1, sticky='N')

    # Leader Profile Picture
    leader_pic = Image.open('Images/Profile Pic.jpg')
    leader_pic = leader_pic.resize((150, 150))
    leader_pic = ImageTk.PhotoImage(leader_pic)
    leader_pic_label = tkinter.Label(frame, image=leader_pic)
    leader_pic_label.image = leader_pic
    leader_pic_label.grid(row=1, column=1, sticky='N')

    # Leader Name
    leader_name = tkinter.Label(frame, text="Kartik Bind", font=('Arial', 15))
    leader_name.grid(column=1, row=2, sticky='N')

    # Leader Institute Name
    leader_institute_name = tkinter.Label(frame, text='Dr. Akhilesh Das Gupta Institute\nof\nTechnology & Management')
    leader_institute_name.grid(column=1, row=3, sticky='N')

    # Leader Passing Batch
    leader_passing_batch_label = tkinter.Label(frame, text='2023', font=('Arial', 15))
    leader_passing_batch_label.grid(column=1, row=4, sticky='n')

    # Member-1

    # Member-1 Label
    leader_label = tkinter.Label(frame, text='Member-1', font=('Arial', 15))
    leader_label.grid(row=0, column=2, sticky='N')

    # Member-1 Profile Picture
    member_1_pic = Image.open('Images/Kabir_Profile_Pic.jpeg')
    member_1_pic = member_1_pic.resize((150, 150))
    member_1_pic = ImageTk.PhotoImage(member_1_pic)
    member_1_pic_label = tkinter.Label(frame, image=member_1_pic)
    member_1_pic_label.image = member_1_pic
    member_1_pic_label.grid(row=1, column=2, sticky='n')

    # Member-1 Name
    member_1_name = tkinter.Label(frame, text='Kabir Sharma', font=('Arial', 15))
    member_1_name.grid(row=2, column=2, sticky='n')

    # Member-1 Institute Name
    member_1_institute_name = tkinter.Label(frame, text='Dr. Akhilesh Das Gupta Institute\nof\nTechnology & Management')
    member_1_institute_name.grid(column=2, row=3, sticky='N')

    # Member-1 Passing Batch
    member_1_passing_batch_label = tkinter.Label(frame, text='2023', font=('Arial', 15))
    member_1_passing_batch_label.grid(column=2, row=4, sticky='n')

    # Member-2

    # Member-2 Label
    leader_label = tkinter.Label(frame, text='Member-2', font=('Arial', 15))
    leader_label.grid(row=0, column=3, sticky='N')

    # Member-2 Profile Picture
    member_2_pic = Image.open('Images/Yukti_Profile_Pic.jpeg')
    member_2_pic = member_2_pic.resize((150, 150))
    member_2_pic = ImageTk.PhotoImage(member_2_pic)
    member_2_pic_label = tkinter.Label(frame, image=member_2_pic)
    member_2_pic_label.image = member_2_pic
    member_2_pic_label.grid(row=1, column=3, sticky='n')

    # Member-2 Name
    member_2_name = tkinter.Label(frame, text=' Yukti', font=('Arial', 15))
    member_2_name.grid(row=2, column=3, sticky='n')

    # Member-2 Institute Name
    member_2_institute_name = tkinter.Label(frame, text='Dr. Akhilesh Das Gupta Institute\nof\nTechnology & Management')
    member_2_institute_name.grid(column=3, row=3, sticky='N')

    # Member-2 Passing Batch
    member_2_passing_batch = tkinter.Label(frame, text='2023', font=('Arial', 15))
    member_2_passing_batch.grid(column=3, row=4)

def gui():
    window = tkinter.Tk()
    window.title("Flipkart GRiD 4.0")

    # To find the center of the screen according to the app dimension
    app_width = 1000
    app_height = 750
    screen_width = window.winfo_screenwidth()
    screen_height = window.winfo_screenheight()
    center_width = int((screen_width / 2) - (app_width / 2))
    center_height = int((screen_height / 2) - (app_height / 2))

    # To give geometry to the app
    window.geometry(f'{app_width}x{app_height}+{center_width}+{center_height}')

    # Column width
    window.columnconfigure(0, weight=1)
    window.columnconfigure(1, weight=10)
    window.columnconfigure(2, weight=10)

    # Row width
    window.rowconfigure(0, weight=1)
    window.rowconfigure(1, weight=1)
    window.rowconfigure(2, weight=10)
    window.rowconfigure(3, weight=1)

    # Label for team name Cache of Clan
    team_name_label = tkinter.Label(window, text="Cache of Clan", font=('Arial', 20))
    team_name_label.grid(columnspan=3, row=0, sticky="NS", padx=5)

    # Image of team
    pic = Image.open('Images/Flipkart-GRiD-4.0.png')
    pic = pic.resize((360, 200))
    pic = ImageTk.PhotoImage(pic)
    pic_label = tkinter.Label(window, image=pic)
    pic_label.image = pic
    pic_label.grid(row=1, columnspan=3)

    # Left Menu
    left_menu_frame = tkinter.LabelFrame(window, text='Menu', font=('Arial', 15))
    left_menu_frame.grid(column=0, row=2, sticky='nsew', pady=10, padx=10)

    # Left Menu Frame Column Configure
    left_menu_frame.columnconfigure(0, weight=1)

    # Right Frame
    right_frame = tkinter.Frame(window)
    right_frame.grid(column=1, row=2, columnspan=2, sticky='nsew')

    # Right Frame Column Configure
    right_frame.columnconfigure(0, weight=1)

    # Right Frame Row Configure
    right_frame.rowconfigure(0, weight=1)  # it's centering widgets vertically

    # Buttons for Left Menu
    button_width = 20
    button_pad_x = 5

    # Check Repo Button
    check_repo_button = tkinter.Button(left_menu_frame, text="Check Repo", width=button_width,
                                       command=lambda: check_repo_code(right_frame))
    check_repo_button.grid(padx=button_pad_x, pady=5)

    # Rate Repo Button
    rate_repo_button = tkinter.Button(left_menu_frame, text="Rate Repo", width=button_width,
                                      command=lambda: rate_repo_code(right_frame))
    rate_repo_button.grid(padx=button_pad_x, pady=5)

    # Add Vulnerabilities Button
    add_vulnerabilities = tkinter.Button(left_menu_frame, text=" Add Vulnerable Syntax", width=button_width,
                                         command=lambda: add_vulnerable_syntax(right_frame))
    add_vulnerabilities.grid(padx=button_pad_x, pady=5)

    # Display Code Button
    display_code_button = tkinter.Button(left_menu_frame, text="Display Codes", width=button_width,
                                         command=lambda: display_code(right_frame))
    display_code_button.grid(padx=button_pad_x, pady=5)

    # Team Information Button
    team_info = tkinter.Button(window, text="Team Information", width=button_width,
                               command=lambda: team_info_display(right_frame))
    team_info.grid(padx=button_pad_x, pady=5)

    # Welcome Label
    welcome_label = tkinter.Label(right_frame, text='Welcome to OSS Security Inspector', font=('Arial', 25))
    welcome_label.grid(column=0, row=0, columnspan=1, sticky='ew')

    # Close Button
    close_button = tkinter.Button(window, text="Close", command=window.quit)
    close_button.grid(row=3, column=2, columnspan=2, sticky='e', padx=20)

    window.mainloop()

gui()


import torch
import torch.nn as nn

import utils.utils as utils
off_diagonal = utils.off_diagonal

class BarlowTwinsLoss(nn.Module):
	def __init__(self, cfg, ncrops):
		super().__init__()
		self.cfg = cfg
		self.ncrops = ncrops
		self.bn = nn.BatchNorm1d(cfg.projector_out_dim, affine=False)

	def forward_loss(self, z1, z2):
		# empirical cross-correlation matrix
		c = self.bn(z1).T @ self.bn(z2)
		# sum the cross-correlation matrix between all gpus
		c.div_(z1.shape[0])
		if utils.is_dist_avail_and_initialized():
			torch.distributed.all_reduce(c)

		on_diag = torch.diagonal(c).add_(-1).pow_(2).sum()
		if self.cfg.HSIC:
			# encouraging off_diag to be negative ones
			off_diag = off_diagonal(c).add_(1).pow_(2).sum()
		else:
			off_diag = off_diagonal(c).pow_(2).sum()
		loss = self.cfg.alpha * on_diag + self.cfg.lmbda * off_diag
		return loss

	def forward(self, student_output, teacher_output, ngcrops_each=1):

		student_out = student_output.chunk(self.ncrops - (2-ngcrops_each))
		teacher_out = teacher_output.chunk(ngcrops_each)  # teacher only gets 1 global crop

		total_loss = 0
		n_loss_terms = 0
		for q in range(len(teacher_out)):
			for v in range(len(student_out)):
				if len(teacher_out) > 1:
					if q == v:
						continue
				loss = self.forward_loss(teacher_out[q], student_out[v])
				total_loss += loss
				n_loss_terms += 1
		total_loss /= n_loss_terms
		return total_loss


import sys
import os
import argparse
import pickle
# FAISS will crash if you don't import it here!
import faiss
import glob

import pandas as pd
from tqdm import tqdm
from sklearn.preprocessing import normalize

sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..'))

import configs
from core.clip import CLIP
from core.faiss_index import FaissIndex
from utils import pytorch_utils as ptu
from utils import logging_utils as logu
from utils.logging_utils import print_verbose

if __name__ == '__main__':
    # ----- Get arguments from input -----
    parser = argparse.ArgumentParser()

    # Path
    parser.add_argument('--dataframe_path', type=str, default=os.path.join('ilsvrc2012', 'imagenet_captions.parquet'))
    parser.add_argument('--index2wnid_path', type=str,
                        default=os.path.join('ilsvrc2012', 'processed', 'labels', 'icimagename2wnid.pkl'))
    parser.add_argument('--labels_path', type=str, default=os.path.join('laion400m', 'processed', 'ilsvrc_labels'))
    parser.add_argument('--labels_filter', type=str, default='wnid2laionindices(substring_matched_part*).pkl')

    parser.add_argument('--indices_path', type=str,
                        default=os.path.join('laion400m', 'processed', 'clip_text_indices', 'all_indices.npy'))

    parser.add_argument('--faiss_index_path', type=str,
                        default=os.path.join('laion400m', 'processed', 'faiss_index', 'knn.index'))

    parser.add_argument('--save_path', type=str, default=os.path.join('laion400m', 'processed', 'ilsvrc_labels'))

    # Compute
    parser.add_argument('--no_gpu', action='store_true')
    parser.add_argument('--gpu_id', type=int, default=0)

    # Logging
    parser.add_argument('--no_verbose', dest='verbose', action='store_false')

    # Overwrite?
    parser.add_argument('--no_safe', dest='safe', action='store_false')

    # Convert to dictionary
    params = vars(parser.parse_args())

    # ----- Init. -----
    logu.verbose = params['verbose']

    print_verbose('initializing ...')

    # Env
    ptu.init_gpu(use_gpu=not params['no_gpu'], gpu_id=params['gpu_id'])

    # Path
    os.makedirs(params['save_path'], exist_ok=True)

    # Safety
    open_type = 'xb' if params['safe'] else 'wb'

    print_verbose('done!\n')

    # ----- Init. CLIP -----
    print_verbose('init clip ...')

    clip = CLIP()

    print_verbose('done!\n')

    # ----- Load dataframe labels -----
    print_verbose('loading dataframe labels ...')

    with open(os.path.join(params['index2wnid_path']), 'rb') as f:
        index2wnid = pickle.load(f)

    print_verbose('done!\n')

    # ----- Load LAION labels -----
    print_verbose('loading laion labels (maps) ...')

    maps_paths = glob.glob(os.path.join(params['labels_path'], params['labels_filter']))

    print_verbose(f'\tfound {len(maps_paths)} wnid2laion maps:\n')
    print_verbose('\t- ' + '\n\t- '.join(maps_paths))

    wnid2laionindices = {}
    for path in tqdm(maps_paths):
        with open(path, 'rb') as f:
            wnid2laionindices_i = pickle.load(f)

        for wnid, laionindices in wnid2laionindices_i.items():
            if wnid not in wnid2laionindices:
                wnid2laionindices[wnid] = []
            wnid2laionindices[wnid].extend(laionindices)

    print_verbose('done!\n')

    # ----- Load the text index -----
    faiss_index = FaissIndex.load(params['faiss_index_path'], params['indices_path'])

    # ----- Load dataframe -----
    print_verbose('loading dataframe ...')

    df = pd.read_parquet(params['dataframe_path'])

    print_verbose(f'\tfound {len(df)} rows.')

    print_verbose('done!\n')

    # ----- Collect downloads and calc. embeddings -----
    # Init.
    index2laionindices = {}
    index2sims = {}

    indices_batch = []
    wnids_batch = []
    texts_batch = []
    i_row = -1
    for idx, row in tqdm(df.iterrows(), desc='calc embeds and find most similars', total=len(df)):
        i_row += 1

        # Parse
        text = row[configs.LAIONConfig.TEXT_COL]

        # Add to batch
        indices_batch.append(idx)
        wnids_batch.append(index2wnid[idx])
        texts_batch.append(text)

        if len(indices_batch) < configs.CLIPConfig.BATCH_SIZE and i_row < (len(df) - 1):
            continue
        if len(indices_batch) == 0:
            continue

        # Get CLIP embedding
        embeds_batch = clip.text_embeds(texts_batch)
        embeds_batch_norm = normalize(embeds_batch, norm='l2', axis=1)

        # Search the index
        k = 512
        laion_indices_bk, sims_bk = faiss_index.search(embeds_batch_norm, k=k)

        laion_indices_b, sims_b = [], []
        for i_row in range(len(laion_indices_bk)):
            laion_indices_k, sims_k = laion_indices_bk[i_row], sims_bk[i_row]

            laion_index, sim = None, None

            wnid = wnids_batch[i_row]
            if wnid in wnid2laionindices:
                for k_res in range(k):
                    if laion_indices_k[k_res] in wnid2laionindices[wnid]:
                        laion_index = laion_indices_k[k_res]
                        sim = sims_k[k_res]
                        break

            laion_indices_b.append(laion_index)
            sims_b.append(sim)

        # Update
        for i_row, index in enumerate(indices_batch):
            if laion_indices_b[i_row] is None:
                continue
            laion_idx = laion_indices_b[i_row]
            sim = sims_b[i_row]

            # For compatibility with subset_laion, use array
            index2laionindices[index] = [laion_idx]
            index2sims[index] = [sim]

        # Step
        indices_batch = []
        wnids_batch = []
        texts_batch = []

    # ----- Save -----
    print_verbose('saving ...')

    prefix = configs.LAIONConfig.SUBSET_IC_MOST_SIMILAR_TXT_TXT_WITH_SIMILAR_LABELS_PREFIX

    file_name = f'icimagename2laionindices({prefix[:-1]}).pkl'
    with open(os.path.join(params['save_path'], file_name), open_type) as f:
        pickle.dump(index2laionindices, f)

    sim_file_name = f'icimagename2sims({prefix[:-1]}).pkl'
    with open(os.path.join(params['save_path'], sim_file_name), open_type) as f:
        pickle.dump(index2sims, f)

    print_verbose('done!\n')


from Observer import Observer
from EventListener import Observable
import os
from tkinter import *
from tkinter import filedialog, messagebox, ttk, Listbox, Label, Canvas, Frame, Entry, Tk
import youtubesearchpython
from youtubesearchpython import ResultMode

class window():

    def checkplaylist(self):
        desktop = os.path.join(os.path.join(os.environ['USERPROFILE']), 'Desktop')
        isFolder = os.path.isdir(desktop + "\\" + self.getEntryVal())
        return isFolder

    def createfolder(self):
        desktop = os.path.join(os.path.join(os.environ['USERPROFILE']), 'Desktop')
        desktop = os.path.join(desktop, self.getEntryVal())
        os.mkdir(desktop)

    def enter(self,search):
        self.topResultList = youtubesearchpython.VideosSearch(search, limit=1)
        self.updatewindow("Searching song")
        self.result = self.topResultList.result(mode=ResultMode.dict)
        self.updatewindow('song found.')
        title = ((self.result["result"])[0])["title"]
        self.title = title
        url = ((self.result["result"])[0])["link"]
        self.url = url
        SAVE_PATH = '/'.join(os.getcwd().split('/')[:3]) + '/Downloads'
        print(title)
        YoutubeVideo = {
            "format": "best", "postprocessors": [{
                "key": "FFmpegExtractAudio",
                "preferredcodec": "mp3",
                "preferredquality": "192",

            }],
            "outtmpl": '%USERPROFILE%\Desktop\\' + self.getEntryVal() + '\%(title)s-%(id)s.%(ext)s',

        }
        # path + "/%(title)s.mp3
        self.update()
        self.updatewindow("Downloading song. ")
        self.updatewindow("Song title: " + self.title)
        self.entercountid =+ 1
        entity = Observer(self.entercountid,"song",YoutubeVideo,self.url)
        print("Count run:" + str(self.entercountid))
        self.observerlist.attach(entity)

    def loadFile(self):
        try:
            if self.checkplaylist():
                messagebox.showerror(title="Error", message="Songlist name has already existed!!!!!!")
                return

            filename = filedialog.askopenfilename(initialdir="/", title="Select File",
                                                  filetypes=(("text files", "*.txt"),
                                                             ("all files", "*.*")))
            if filename == "":
                return
            self.createfolder()
            with open(filename, "r") as file:
                for line in file:
                    print("print song name: "+line)
                    songlist.append(line)
                    self.enter(line)
            self.observerlist.exec()
        except FileNotFoundError:
            print("exit window without action")
            messagebox.showinfo("Please input a file with at least 1 song")
        print("Finished downloads for playlist.")

    def getEntryVal(self):
        return self.entry.get()

    def update(self):
        self.root.update()

    def updatewindow(self,text):
        self.listbox.insert(self.linecount,text)
        self.linecount = self.linecount + 1
        self.update()

    def check(self,app):
        if app['status'] != 'finished':
            self.update()
        else:
            return

    def __init__(self, root):
        self.root = root
        self.root.title("Youtube songs downloader")
        self.root.resizable(width=False, height=False)
        self.canvas = Canvas(self.root, height=600, width=600, bg="#2c2f33")
        self.canvas.pack()
        self.entercountid = 0
        self.linecount = 0
        self.observerlist = Observable()
        self.topFrame = Frame(self.root, bg="#7289da")
        self.topFrame.place(relwidth=0.8, relheight=0.4, relx=0.1, rely=0.1)
        self.label = Label(self.topFrame, text="Enter the song list name:", bg="#7289da")
        self.label.place(relx=0.36, rely=0.16)
        self.entry = Entry(self.topFrame, text="Example playlist Name")
        self.entry.insert(0, "SongList")
        self.entry.place(relwidth=0.5, relheight=0.1, relx=0.25, rely=0.25, )
        combobox = ttk.Combobox(self.topFrame, state="readonly")
        self.combobox = combobox
        self.combobox['values'] = ("Download songs from local .txt",
                                   "Download from youtube online playlist")
        self.combobox.place(relwidth=0.5, relheight=0.1, relx=0.25, rely=0.37)
        self.combobox.set("Download songs from local .txt")
        self.combobox.current(0)

        self.buttomFrame = Frame(self.root, bg="White")
        self.buttomFrame.place(relwidth=0.8, relheight=0.4, relx=0.1, rely=0.51)

        self.listbox = Listbox(self.buttomFrame,bg="White")
        self.listbox.pack(side="left", fill= BOTH, expand=1)

        self.scrollbar = Scrollbar(self.buttomFrame)
        self.scrollbar.pack(side="right", fill=Y)

        self.listbox.config(yscrollcommand = self.scrollbar.set)
        self.scrollbar.config(command = self.listbox.yview)

        self.myButton = Button(self.topFrame, text="Click here to Download", padx=10, pady=0, fg="#2c2f33",
                               bg="#99aab5",
                               command=self.loadFile)
        self.myButton.place(relx=0.34, rely=0.5)

root = Tk()
songlist = []
window = window(root)
root.mainloop()

# if __name__ == "__main__":
#     main()