#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 25 21:24:35 2019

@author: behrense
"""

from netCDF4 import Dataset
import matplotlib.pyplot as plt
import numpy as np
import  cmocean

def initial_conditions(din,ratio=1.005):
    rn15std=0.0036765
    tmp=ratio*rn15std*din/(1+ratio*rn15std)
    return tmp

def delta(C,C15):
    rn15std=0.0036765
    tmp=(C15/(C-C15)/rn15std-1)*1000

    return tmp

# both euqations verified

d1=Dataset('DIN.nc')
lon=d1.variables['nav_lon'][:]
lat=d1.variables['nav_lat'][:]
DIN=d1.variables['DIN'][:]
d3=Dataset('tmask.nc')
tmask=d3.variables['tmaskutil'][:]

#%% create initial conditions for tracer concentrations as per MEDUSA code

PHN=np.zeros(DIN.shape)
PHN[:14,:,:]=0.1 # over the frist 200 m else 0
PHD=PHN.copy()+.2
ZMI=PHN.copy()+.5
ZME=PHN.copy()+.7
DET=PHN.copy()+.25

PHN_N15=initial_conditions(PHN,1.005)
PHD_N15=initial_conditions(PHD,1.005)
ZMI_N15=initial_conditions(ZMI,1.005)
ZME_N15=initial_conditions(ZME,1.005)
DIN_N15=initial_conditions(DIN,1.005)
DET_N15=initial_conditions(DET,1.005)

#%%  initial tracer concentrations next time step to 0
PHN_next=np.zeros(PHN.shape)
PHD_next=np.zeros(PHN.shape)
ZMI_next=np.zeros(PHN.shape)
ZME_next=np.zeros(PHN.shape)
DIN_next=np.zeros(PHN.shape)
DET_next=np.zeros(PHN.shape)

PHN_N15_next=np.zeros(PHN.shape)
PHD_N15_next=np.zeros(PHN.shape)
ZMI_N15_next=np.zeros(PHN.shape)
ZME_N15_next=np.zeros(PHN.shape)
DIN_N15_next=np.zeros(PHN.shape)
DET_N15_next=np.zeros(PHN.shape)

#%% check that things are ok ratios are 5 permil
plt.close('all')
fig = plt.figure(figsize=(15,10))
ax1 = fig.add_subplot(231)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(PHN[0,0,:,:],PHN_N15[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'PHN',color='white')
ax1 = fig.add_subplot(232)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(PHD[0,0,:,:],PHD_N15[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'PHD',color='white')
ax1 = fig.add_subplot(233)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(ZMI[0,0,:,:],ZMI_N15[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'ZMI',color='white')
ax1 = fig.add_subplot(234)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(ZME[0,0,:,:],ZME_N15[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'ZME',color='white')
ax1 = fig.add_subplot(235)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(DIN[0,0,:,:],DIN_N15[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'DIN',color='white')
ax1 = fig.add_subplot(236)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(DET[0,0,:,:],DET_N15[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'DET',color='white')

plt.savefig('Initial_surface_concentration.png',dpi=200,pading=.1,bbox_inches='tight')

#%% code as similar to Fortran

def calculate_ratios():
    global fprn,zphn,fprd,zphd
    global DIN,DIN_N15,PHN,PHN_N15,PHD,PHD_N15,ZMI,ZMI_N15,ZME,ZME_N15
    global rn15std
    global un,rdin,bassimphn,fcassimphn,bassimphd,fcassimphd,rtdin15,rtphnn15,rtphdn15
    global rtzmin15,rtzmen15,rtdetrn15,rmi,bmiexcr,fcmiexcr,rme,bmeexcr,fcmeexcr

    eps_assim = 0.
    eps_excr= 0.

    rmin=0.000000001
    dmin=0.000001
    rmax=0.999999999
    rdt=40*60

    #keep code as similar to Fortran so we can copy across

    zdim=0
    tdim=0

    for jj in range(fprn.shape[0]):
        for ji in range(fprn.shape[1]):
              #uno3 = npp*dtbio/biono3
              un[jj,ji]=(fprn[jj,ji] * zphn[jj,ji] + fprd[jj,ji] * zphd[jj,ji]) * rdt /max(DIN[tdim,zdim,jj,ji],dmin)

              un[jj,ji] = min(un[jj,ji], rmax)
              un[jj,ji] = max(un[jj,ji], rmin)
              #rno3 = biodin15/(biono3-biodin15)
              rdin[jj,ji] = DIN_N15[tdim,zdim,jj,ji]/(max(DIN[tdim,zdim,jj,ji]-DIN_N15[tdim,zdim,jj,ji],dmin))
              rdin[jj,ji] = min(rdin[jj,ji],2*rn15std)
              rdin[jj,ji] = max(rdin[jj,ji],rn15std/2)
              #print(rdin[0,0])
              #bassim = rno3 + eps_assim*(1-uno3)/uno3*log(1-uno3)*rno3/1000.
              bassimphn[jj,ji]=rdin[jj,ji]+rdin[jj,ji]*eps_assim*(1-un[jj,ji])/un[jj,ji]*np.log(1-un[jj,ji])/1000.
              fcassimphn[jj,ji]=bassimphn[jj,ji]/(1+bassimphn[jj,ji])
              #print(fcassimphn[0,0])
              #bassim = rno3 + eps_assim*(1-uno3)/uno3*log(1-uno3)*rno3/1000.
              bassimphd[jj,ji]=rdin[jj,ji]+rdin[jj,ji]*eps_assim*(1-un[jj,ji])/un[jj,ji]*np.log(1-un[jj,ji])/1000.
              fcassimphd[jj,ji]=bassimphd[jj,ji]/(1+bassimphd[jj,ji])
              #rtdin15 = biodin15/biono3
              rtdin15[jj,ji] = DIN_N15[tdim,zdim,jj,ji] /max(DIN[tdim,zdim,jj,ji],rmin)
              rtdin15[jj,ji] = min(rtdin15[jj,ji],2*rn15std/(1+rn15std))
              rtdin15[jj,ji] = max(rtdin15[jj,ji],rn15std/(1+rn15std)/2.)
              #rtphytn15 = biophytn15/biophyt
              rtphnn15[jj,ji] = PHN_N15[tdim,zdim,jj,ji] /max(PHN[tdim,zdim,jj,ji],rmin)
              rtphdn15[jj,ji] = PHD_N15[tdim,zdim,jj,ji] /max(PHD[tdim,zdim,jj,ji],rmin)
              rtphnn15[jj,ji] = min(rtphnn15[jj,ji],2.*rn15std/(1+rn15std))
              rtphnn15[jj,ji] = max(rtphnn15[jj,ji],rn15std/(1+rn15std)/2.)
              rtphdn15[jj,ji] = min(rtphdn15[jj,ji],2.*rn15std/(1+rn15std))
              rtphdn15[jj,ji] = max(rtphdn15[jj,ji],rn15std/(1+rn15std)/2.)
              #rtzoopn15 = biozoopn15/biozoop
              rtzmin15[jj,ji] = ZMI_N15[tdim,zdim,jj,ji] /max(ZMI[tdim,zdim,jj,ji],rmin)
              rtzmen15[jj,ji] = ZME_N15[tdim,zdim,jj,ji] /max(ZME[tdim,zdim,jj,ji],rmin)
              rtzmin15[jj,ji] = min(rtzmin15[jj,ji],2.*rn15std/(1+rn15std))
              rtzmin15[jj,ji] = max(rtzmin15[jj,ji],rn15std/(1+rn15std)/2.)
              rtzmen15[jj,ji] = min(rtzmen15[jj,ji],2.*rn15std/(1+rn15std))
              rtzmen15[jj,ji] = max(rtzmen15[jj,ji],rn15std/(1+rn15std)/2.)
              #rtdetrn15 = biodetrn15/biodetr
              rtdetrn15[jj,ji] = DET_N15[tdim,zdim,jj,ji] /max(DET[tdim,zdim,jj,ji],rmin)
              rtdetrn15[jj,ji] = min(rtdetrn15[jj,ji],2.*rn15std/(1+rn15std))
              rtdetrn15[jj,ji] = max(rtdetrn15[jj,ji],rn15std/(1+rn15std)/2.)
              #rzoop = biozoopn15/(biozoop-biozoopn15)
              rmi[jj,ji] = ZMI_N15[tdim,zdim,jj,ji] /max(ZMI[tdim,zdim,jj,ji]-ZMI_N15[tdim,zdim,jj,ji],rmin)
              rmi[jj,ji] = min(rmi[jj,ji], 2.*rn15std)
              rmi[jj,ji] = max(rmi[jj,ji], rn15std/2.)
              #bexcr = rzoop - eps_excr*rzoop/1000.
              bmiexcr[jj,ji] = rmi[jj,ji] - eps_excr*rmi[jj,ji]/1000.
              fcmiexcr[jj,ji] = bmiexcr[jj,ji]/(1+bmiexcr[jj,ji])
              #rzoop = biozoopn15/(biozoop-biozoopn15)
              rme[jj,ji] = ZME_N15[tdim,zdim,jj,ji] /max(ZME[tdim,zdim,jj,ji]-ZME_N15[tdim,zdim,jj,ji],rmin)
              rme[jj,ji] = min(rme[jj,ji], 2.*rn15std)
              rme[jj,ji] = max(rme[jj,ji], rn15std/2.)
              #bexcr = rzoop - eps_excr*rzoop/1000.
              bmeexcr[jj,ji] = rme[jj,ji] - eps_excr*rme[jj,ji]/1000.
              fcmeexcr[jj,ji] = bmeexcr[jj,ji]/(1+bmeexcr[jj,ji])

    return

#%%

def calc_growth():

        global fprn,fgmipn,fgmid
        global fstarmi,fmith,ficmi,ficmin15
        xphi=.2
        xthetapn=6.625
        xbetan=0.77
        xthetazmi=5.625
        xbetac=0.64
        xkc=0.80
        xthetazme=5.625
        xthetapd=6.625

        finmi=np.zeros(fprn.shape)
        finme=np.zeros(fprn.shape)
        ficmi=np.zeros(fprn.shape)
        finmin15=np.zeros(fprn.shape)
        finmen15=np.zeros(fprn.shape)
        ficmin15=np.zeros(fprn.shape)
        fmigrow=np.zeros(fprn.shape)
        fmigrown15=np.zeros(fprn.shape)
        ficme=np.zeros(fprn.shape)
        fgmidc=np.zeros(fprn.shape)+.01
        fgmedc=np.zeros(fprn.shape)+.1
        ficmen15=np.zeros(fprn.shape)
        fmegrow=np.zeros(fprn.shape)
        fmegrown15=np.zeros(fprn.shape)

        for jj in range(fprn.shape[0]): #loop j

            for ji in range(fprn.shape[1]): # loop i

               finmi[jj,ji]  = (1.0 - xphi) * (fgmipn[jj,ji]+ fgmid[jj,ji])
               ficmi[jj,ji]  = (1.0 - xphi) * ((xthetapn * fgmipn[jj,ji]) + fgmidc[jj,ji])
               finmin15[jj,ji]  = (1.0 - xphi) * (rtphnn15[jj,ji]*fgmipn[jj,ji]+ rtdetrn15[jj,ji]*fgmid[jj,ji])
               ficmin15[jj,ji]  = (1.0 - xphi) * rtphnn15[jj,ji]*((xthetapn * fgmipn[jj,ji]) +  fgmidc[jj,ji])

               fstarmi = (xbetan * xthetazmi) / (xbetac * xkc)
               fmith = (ficmi[jj,ji]/ (finmi[jj,ji]+ finmi[jj,ji]))

               if (fmith >= fstarmi):
                  fmigrow[jj,ji]= xbetan * finmi[jj,ji]
                  fmiexcr[jj,ji]= 0.0
                  fmigrown15[jj,ji]= xbetan * finmin15[jj,ji]

               else:
                  fmigrow[jj,ji]= (xbetac * xkc * ficmi[jj,ji]) / xthetazmi
                  fmiexcr[jj,ji]= ficmi[jj,ji]* ((xbetan / (fmith + fmith)) - ((xbetac * xkc) / xthetazmi))
                  fmigrown15[jj,ji]= (xbetac * xkc * ficmin15[jj,ji]) / xthetazmi

               finme[jj,ji]   = (1.0 - xphi) \
                   *(fgmepn[jj,ji] + fgmepd[jj,ji] \
                   +fgmezmi[jj,ji] + fgmed[jj,ji])
               ficme[jj,ji]   = (1.0 - xphi) *\
                                ((xthetapn * fgmepn[jj,ji]) +\
                                (xthetapd * fgmepd[jj,ji]) +\
                                (xthetazmi * fgmezmi[jj,ji]) + fgmedc[jj,ji])

               finmen15[jj,ji]   = (1.0 - xphi) *\
                                (rtphnn15[jj,ji]*fgmepn[jj,ji] + rtphdn15[jj,ji]*fgmepd[jj,ji] +\
                                rtzmin15[jj,ji]*fgmezmi[jj,ji] + rtdetrn15[jj,ji]*fgmed[jj,ji])
               total=(xthetapn  * fgmepn[jj,ji]) + (xthetapd * fgmepd[jj,ji]) + (xthetazmi * fgmezmi[jj,ji]) + fgmedc[jj,ji]
               frac1=(xthetapn  * fgmepn[jj,ji]+1/3* fgmedc[jj,ji])/total
               frac2=(xthetapd  * fgmepd[jj,ji]+1/3* fgmedc[jj,ji])/total
               frac3=(xthetazmi * fgmezmi[jj,ji]+1/3* fgmedc[jj,ji])/total

               ficmen15[jj,ji]   = (1.0 - xphi) *\
                                 (frac1*rtphnn15[jj,ji]+frac2*rtphdn15[jj,ji]+frac3*rtzmin15[jj,ji])*((xthetapn * fgmepn[jj,ji]) +\
                                 (xthetapd * fgmepd[jj,ji]) + \
                                 (xthetazmi * fgmezmi[jj,ji]) + fgmedc[jj,ji])

               fstarme        = (xbetan * xthetazme) / (xbetac * xkc)

               fmeth   = (ficme[jj,ji] / (finme[jj,ji] + finme[jj,ji]))
               if (fmeth >= fstarme):
                  fmegrow[jj,ji] = xbetan * finme[jj,ji]
                  fmeexcr[jj,ji] = 0.0
                  fmegrown15[jj,ji] = xbetan * finmen15[jj,ji]
               else:
                  fmegrow[jj,ji] = (xbetac * xkc * ficme[jj,ji]) / xthetazme
                  fmeexcr[jj,ji] = ficme[jj,ji] * ((xbetan / (fmeth + fmeth)) -  ((xbetac * xkc) / xthetazme))
                  fmegrown15[jj,ji] = (xbetac * xkc * ficmen15[jj,ji]) / xthetazme

        return finmi,finme,fmigrow,fmigrown15,fmegrow,fmegrown15

#%%

def compute_fluxes():
    global fprn,zphn,fprd,zphd
    global DIN,DIN_N15,PHN,PHN_N15,PHD,PHD_N15,ZMI,ZMI_N15,ZME,ZME_N15
    global PHN_N15_flux,PHD_N15_flux,ZMI_N15_flux,ZME_N15_flux,DET_N15_flux,DIN_N15_flux
    global rn15std
    global un,rdin,bassimphn,fcassimphn,bassimphd,fcassimphn,rtdin15,rtphnn15,rtphdn15
    global rtzmin15,rtzmen15,rtdetrn15,rmi,bmiexcr,fcmiexcr,rme,bmeexcr,fcmeexcr,fmiexcr,fmeexcr,freminn
    global PHN_flux,PHD_flux,ZMI_flux,ZME_flux,DET_flux,DIN_flux

    finmi,finme,fmigrow,fmigrown15,fmegrow,fmegrown15=calc_growth()

    fn_cons_n15=np.zeros(fprn.shape)
    fn_prod_n15=np.zeros(fprn.shape)
    fn_cons=np.zeros(fprn.shape)
    fn_prod=np.zeros(fprn.shape)
    b0=1
    xfdfrac1=0.333
    xfdfrac2=1
    xbetan=0.77
    xphi=.2

    for jj in range(fprn.shape[0]): #loop j
        for ji in range(fprn.shape[1]): # loop i
               PHN_flux[jj,ji] = b0 * ( fprn[jj,ji] * zphn[jj,ji] \
                                     - fdpn[jj,ji] \
                                     - fdpn2[jj,ji] \
                                     - fgmipn[jj,ji] \
                                     - fgmepn[jj,ji] )

               PHD_flux[jj,ji] = b0 * ( fprd[jj,ji] * zphd[jj,ji] \
                                      - fdpd[jj,ji] \
                                      - fdpd2[jj,ji] \
                                      - fgmepd[jj,ji])

               ZMI_flux[jj,ji] = b0 * (fmigrow[jj,ji] \
                                   - fgmezmi[jj,ji] \
                                   - fdzmi[jj,ji]\
                                   - fdzmi2[jj,ji])

               ZME_flux[jj,ji] = b0 * (fmegrow[jj,ji]            \
                                    - fdzme[jj,ji] \
                                    - fdzme2[jj,ji])

               DET_flux[jj,ji] = b0 * (fdpn[jj,ji] \
                                    +  (1.0 - xfdfrac1) * fdpd[jj,ji] \
                                    + fdzmi[jj,ji]                    \
                                    + (1.0 - xfdfrac2) *fdzme[jj,ji] \
                                    + ((1.0 - xbetan) *  (finmi[jj,ji] + finme[jj,ji])) \
                                    - fgmid[jj,ji]                   \
                                    - fgmed[jj,ji]                   \
                                    - fdd[jj,ji]                     \
                                    + fslowgain[jj,ji] - fslowloss[jj,ji]             \
                                    - (f_sbenin_n[jj,ji] / fse3t[jj,ji])           \
                                    + ffast2slown[jj,ji] )

               fn_cons[jj,ji] = -(fprn[jj,ji] * zphn[jj,ji])\
                                 -(fprd[jj,ji] * zphd[jj,ji])

               fn_prod[jj,ji] =(xphi * (fgmipn[jj,ji] + fgmid[jj,ji])) \
                            + (xphi * (fgmepn[jj,ji]   \
                                    + fgmepd[jj,ji]    \
                                    + fgmezmi[jj,ji]   \
                                    + fgmed[jj,ji]))  \
                                    + fmiexcr[jj,ji]                           \
                                    + fmeexcr[jj,ji]                           \
                                    + fdd[jj,ji]              \
                                    + freminn[jj,ji]                           \
                                    + fdpn2[jj,ji]             \
                                    + fdpd2[jj,ji]             \
                                    + fdzmi2[jj,ji]            \
                                    + fdzme2[jj,ji]

               DIN_flux[jj,ji] = b0 * ( fn_prod[jj,ji] + fn_cons[jj,ji] )

               PHN_N15_flux[jj,ji] = b0 * ( fcassimphn[jj,ji]*fprn[jj,ji] * zphn[jj,ji] \
                                     - rtphnn15[jj,ji]*fdpn[jj,ji] \
                                     - rtphnn15[jj,ji]*fdpn2[jj,ji] \
                                     - rtphnn15[jj,ji]*fgmipn[jj,ji] \
                                     - rtphnn15[jj,ji]*fgmepn[jj,ji] )

               PHD_N15_flux[jj,ji] = b0 * ( fcassimphd[jj,ji]*fprd[jj,ji] * zphd[jj,ji] \
                                      - rtphdn15[jj,ji]*fdpd[jj,ji] \
                                      - rtphdn15[jj,ji]* fdpd2[jj,ji] \
                                      - rtphdn15[jj,ji]*fgmepd[jj,ji])

               ZMI_N15_flux[jj,ji] = b0 * (fmigrown15[jj,ji] \
                                   - rtzmin15[jj,ji]*fgmezmi[jj,ji] \
                                   - rtzmin15[jj,ji]*fdzmi[jj,ji]\
                                   - rtzmin15[jj,ji]*fdzmi2[jj,ji])

               ZME_N15_flux[jj,ji] = b0 * (fmegrown15[jj,ji]            \
                                    - rtzmen15[jj,ji]*fdzme[jj,ji] \
                                    - rtzmen15[jj,ji]*fdzme2[jj,ji])

               DET_N15_flux[jj,ji] = b0 * (rtphnn15[jj,ji]*fdpn[jj,ji] \
                                    +  (1.0 - xfdfrac1) *rtphdn15[jj,ji]* fdpd[jj,ji] \
                                    + rtzmin15[jj,ji]*fdzmi[jj,ji]                    \
                                    + (1.0 - xfdfrac2) * rtzmen15[jj,ji]*fdzme[jj,ji] \
                                    + ((1.0 - xbetan) *  (rtzmin15[jj,ji]*finmi[jj,ji] + rtzmen15[jj,ji]*finme[jj,ji])) \
                                    - rtdetrn15[jj,ji]*fgmid[jj,ji]                   \
                                    - rtdetrn15[jj,ji]*fgmed[jj,ji]                   \
                                    - rtdetrn15[jj,ji]*fdd[jj,ji]                     \
                                    + rtdetrn15[jj,ji]*(fslowgain[jj,ji] - fslowloss[jj,ji])             \
                                    - (rtdetrn15[jj,ji]*f_sbenin_n[jj,ji] / fse3t[jj,ji])           \
                                    + rtdetrn15[jj,ji]*ffast2slown[jj,ji] )

               fn_cons_n15[jj,ji] = -(fcassimphn[jj,ji]*fprn[jj,ji] * zphn[jj,ji])\
                                      -(fcassimphd[jj,ji]*fprd[jj,ji] * zphd[jj,ji])

               fn_prod_n15[jj,ji] =(xphi * (rtphnn15[jj,ji]*fgmipn[jj,ji] + rtdetrn15[jj,ji]*fgmid[jj,ji])) \
                           + (xphi * (rtphnn15[jj,ji]*fgmepn[jj,ji]   \
                                   + rtphdn15[jj,ji]*fgmepd[jj,ji]    \
                                   + rtzmin15[jj,ji]*fgmezmi[jj,ji]   \
                                   + rtdetrn15[jj,ji]*fgmed[jj,ji]))  \
                                   + rtzmin15[jj,ji]*fmiexcr[jj,ji]                           \
                                   + rtzmen15[jj,ji]*fmeexcr[jj,ji]                           \
                                   + rtdetrn15[jj,ji]*fdd[jj,ji]              \
                                   + (rtphdn15[jj,ji]+rtzmen15[jj,ji])/2*freminn[jj,ji]                           \
                                   + rtphnn15[jj,ji]*fdpn2[jj,ji]             \
                                   + rtphdn15[jj,ji]*fdpd2[jj,ji]             \
                                   + rtzmin15[jj,ji]*fdzmi2[jj,ji]            \
                                   + rtzmen15[jj,ji]*fdzme2[jj,ji]

               DIN_N15_flux[jj,ji] = b0 * ( fn_prod_n15[jj,ji] + fn_cons_n15[jj,ji] )
    return

def compute_new_concentrations():
    dayinsec=86400.
    global DIN_next,DET_next,PHN_next,PHD_next,ZMI_next,ZME_next
    global DIN_N15_next,DET_N15_next,PHN_N15_next,PHD_N15_next,ZMI_N15_next,ZME_N15_next
    DIN_next[0,0,:,:]=DIN[0,0,:,:]+DIN_flux/dayinsec
    DET_next[0,0,:,:]=DET[0,0,:,:]+DET_flux/dayinsec
    PHN_next[0,0,:,:]=PHN[0,0,:,:]+PHN_flux/dayinsec
    PHD_next[0,0,:,:]=PHD[0,0,:,:]+PHD_flux/dayinsec
    ZMI_next[0,0,:,:]=ZMI[0,0,:,:]+ZMI_flux/dayinsec
    ZME_next[0,0,:,:]=ZME[0,0,:,:]+ZME_flux/dayinsec
    DIN_N15_next[0,0,:,:]=DIN_N15[0,0,:,:]+DIN_N15_flux/dayinsec
    DET_N15_next[0,0,:,:]=DET_N15[0,0,:,:]+DET_N15_flux/dayinsec
    PHN_N15_next[0,0,:,:]=PHN_N15[0,0,:,:]+PHN_N15_flux/dayinsec
    PHD_N15_next[0,0,:,:]=PHD_N15[0,0,:,:]+PHD_N15_flux/dayinsec
    ZMI_N15_next[0,0,:,:]=ZMI_N15[0,0,:,:]+ZMI_N15_flux/dayinsec
    ZME_N15_next[0,0,:,:]=ZME_N15[0,0,:,:]+ZME_N15_flux/dayinsec
    return

#%% define fluxes for the computation of tracer fluxes and assign some random values
rn15std=0.0036765
fprn=np.zeros((DIN.shape[2],DIN.shape[3]))+.5   #production phyt non-diatoms
zphn=np.zeros(fprn.shape)+.2   #non-diatom concentraion
zphd=np.zeros(fprn.shape)+.2   #diatom concentraion
fprd=np.zeros(fprn.shape)+.5   #production phyt diatoms

fdpn=np.zeros(fprn.shape)+1   #loss term
fdpn2=np.zeros(fprn.shape)+2  #loss term 2
fgmipn=np.zeros(fprn.shape)+1  #eaten by zmi
fgmepn=np.zeros(fprn.shape)+1  #eaten by zme

fdpd=np.zeros(fprn.shape)+.2 #loss term
fdpd2=np.zeros(fprn.shape)+.3 #loss term2
fgmepd=np.zeros(fprn.shape)+.5 #eaten by zme

fmigrown15=np.zeros(fprn.shape)+.05 #grow term
fgmezmi=np.zeros(fprn.shape)+.15 #eaten by zme
fdzmi=np.zeros(fprn.shape)+.25 #loss term
fdzmi2=np.zeros(fprn.shape)+.35 #loss term2

fmegrown15=np.zeros(fprn.shape)+.15  #grow term
fdzme=np.zeros(fprn.shape)+.25 #loss term
fdzme2=np.zeros(fprn.shape)+.35 #loss term2

fgmid=np.zeros(fprn.shape)+.3 # grazing zmi on det
fgmed=np.zeros(fprn.shape)+.2 # grazing zme on det
fdd=np.zeros(fprn.shape)+.22  #remin.
fslowgain=np.zeros(fprn.shape)+.1 #sinking
fslowloss=np.zeros(fprn.shape)+.1 #sinking
f_sbenin_n=np.zeros(fprn.shape)+.1 #sinking
fse3t=np.zeros(fprn.shape)+.1 #sinking
ffast2slown=np.zeros(fprn.shape)+.1 #sinking

fmiexcr=np.zeros(fprn.shape)+.1 #excr
fmeexcr=np.zeros(fprn.shape)+.2
freminn=np.zeros(fprn.shape)+.2

bmiexcr=np.zeros(fprn.shape)
bmeexcr=np.zeros(fprn.shape)
#%% initialise N15 ratio fields
un=np.zeros(fprn.shape)
bassimphn=np.zeros(fprn.shape)
bassimphd=np.zeros(fprn.shape)
rdin=np.zeros(fprn.shape)
fcassimphd=np.zeros(fprn.shape)
fcassimphn=np.zeros(fprn.shape)
rtdin15=np.zeros(fprn.shape)
rtphnn15=np.zeros(fprn.shape)
rtphdn15=np.zeros(fprn.shape)
rtzmin15=np.zeros(fprn.shape)
rtzmen15=np.zeros(fprn.shape)
rtdetrn15=np.zeros(fprn.shape)
fcmiexcr=np.zeros(fprn.shape)
fcmeexcr=np.zeros(fprn.shape)
rmi=np.zeros(fprn.shape)
rme=np.zeros(fprn.shape)
#%% define tracer fluxes and set zeros
PHN_flux=np.zeros(fprn.shape)  #phyto non-diatom
PHD_flux=np.zeros(fprn.shape)  #phyto diatom
ZMI_flux=np.zeros(fprn.shape)  #zoo micro
ZME_flux=np.zeros(fprn.shape)  #zoo meso
DET_flux=np.zeros(fprn.shape)  #detritus
DIN_flux=np.zeros(fprn.shape)  #DIN

PHN_N15_flux=np.zeros(fprn.shape)
PHD_N15_flux=np.zeros(fprn.shape)
ZMI_N15_flux=np.zeros(fprn.shape)
ZME_N15_flux=np.zeros(fprn.shape)
DET_N15_flux=np.zeros(fprn.shape)
DIN_N15_flux=np.zeros(fprn.shape)
#%% model run
calculate_ratios()
compute_fluxes()
compute_new_concentrations()

#%% delta ratio after update

plt.close('all')
fig = plt.figure(figsize=(15,10))
ax1 = fig.add_subplot(231)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(PHN_next[0,0,:,:],PHN_N15_next[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'PHN',color='white')
ax1 = fig.add_subplot(232)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(PHD_next[0,0,:,:],PHD_N15_next[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'PHD',color='white')
ax1 = fig.add_subplot(233)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(ZMI_next[0,0,:,:],ZMI_N15_next[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'ZMI',color='white')
ax1 = fig.add_subplot(234)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(ZME_next[0,0,:,:],ZME_N15_next[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'ZME',color='white')
ax1 = fig.add_subplot(235)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(DIN_next[0,0,:,:],DIN_N15_next[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'DIN',color='white')
ax1 = fig.add_subplot(236)
plt.pcolormesh(np.where(tmask[0,:,:]==0,1,delta(DET_next[0,0,:,:],DET_N15_next[0,0,:,:])),vmin=4.7,vmax=5.4)
plt.colorbar()
plt.text(10,10,'DET',color='white')

plt.savefig('Updated_surface_concentration.png',dpi=200,pading=.1,bbox_inches='tight')

#%%
plt.close('all')
fig = plt.figure(figsize=(15,10))
ax1 = fig.add_subplot(231)
plt.pcolormesh(np.where(tmask[0,:,:]==0,np.NaN,delta(PHN_next[0,0,:,:],PHN_N15_next[0,0,:,:])-delta(PHN[0,0,:,:],PHN_N15[0,0,:,:])),vmin=-0.00001,vmax=0.00001,cmap='cmo.balance')
plt.colorbar()
plt.text(10,10,'PHN',color='white')
ax1 = fig.add_subplot(232)
plt.pcolormesh(np.where(tmask[0,:,:]==0,np.NaN,delta(PHD_next[0,0,:,:],PHD_N15_next[0,0,:,:])-delta(PHD[0,0,:,:],PHD_N15[0,0,:,:])),vmin=-0.00001,vmax=0.00001,cmap='cmo.balance')
plt.colorbar()
plt.text(10,10,'PHD',color='white')
ax1 = fig.add_subplot(233)
plt.pcolormesh(np.where(tmask[0,:,:]==0,np.NaN,delta(ZMI_next[0,0,:,:],ZMI_N15_next[0,0,:,:])-delta(ZMI[0,0,:,:],ZMI_N15[0,0,:,:])),vmin=-0.00001,vmax=0.00001,cmap='cmo.balance')
plt.colorbar()
plt.text(10,10,'ZMI',color='white')
ax1 = fig.add_subplot(234)
plt.pcolormesh(np.where(tmask[0,:,:]==0,np.NaN,delta(ZME_next[0,0,:,:],ZME_N15_next[0,0,:,:])-delta(ZME[0,0,:,:],ZME_N15[0,0,:,:])),vmin=-0.00001,vmax=0.00001,cmap='cmo.balance')
plt.colorbar()
plt.text(10,10,'ZME',color='white')
ax1 = fig.add_subplot(235)
plt.pcolormesh(np.where(tmask[0,:,:]==0,np.NaN,delta(DIN_next[0,0,:,:],DIN_N15_next[0,0,:,:])-delta(DIN[0,0,:,:],DIN_N15[0,0,:,:])),vmin=-0.00001,vmax=0.00001,cmap='cmo.balance')
plt.colorbar()
plt.text(10,10,'DIN',color='white')
ax1 = fig.add_subplot(236)
plt.pcolormesh(np.where(tmask[0,:,:]==0,np.NaN,delta(DET_next[0,0,:,:],DET_N15_next[0,0,:,:])-delta(DET[0,0,:,:],DET_N15[0,0,:,:])),vmin=-0.00001,vmax=0.00001,cmap='cmo.balance')
plt.colorbar()
plt.text(10,10,'DET',color='white')

plt.savefig('Updated_surface_concentration_difference.png',dpi=200,pading=.1,bbox_inches='tight')

#%%% test for PHN according to Chris with eps=0 the delta should not change

# rdt=3
# rmin=0.1
# aa=np.zeros(fprn.shape)
# for jj in range(fprn.shape[0]): #loop j
#     for ji in range(fprn.shape[1]): # loop i
#         #un[jj,ji]=max(DIN[0,0,jj,ji],rmin)
#         un[jj,ji]=(fprn[jj,ji] * zphn[jj,ji] + fprd[jj,ji] * zphd[jj,ji]) * rdt /max(DIN[0,0,jj,ji],rmin)

plt.close('all')
fig = plt.figure(figsize=(15,10))
ax1 = fig.add_subplot(111)
plt.pcolormesh(fcassimphn)
plt.colorbar()


import json

def agregar_expedientes(datos_nuevos):
    """
    Guarda los datos de un nuevo expediente en el archivo de expedientes.json
    """
    with open('Modulo-3/files/expedientes.json','r') as archivo:
        lista_expedientes = json.load(archivo) #carga tos los expedientes
    if not lista_expedientes:
        id_nuevo = 1
    else:
        id_nuevo = int(max(lista_expedientes, key=lambda x:x['id'])['id']) + 1
    datos_nuevos['id'] = id_nuevo
    lista_expedientes.append(datos_nuevos)
    with open('Modulo-3/files/expedientes.json','w') as archivo:
        json.dump(lista_expedientes,archivo,indent=4)


from fsspec.implementations.local import LocalFileSystem

from scanner import Scanner

import asyncio

fs = LocalFileSystem()
scanner = Scanner(
    [
        "79852227949",
        "79934962253",
        "79037895690",
        "79934957590",
    ],
    fs)

async def main():
    async with scanner.session():
        async for msg in scanner.get_chat_history("@sea_aparts"):
            print(msg.text)

asyncio.run(main())


import streamlit as st
from dotenv import load_dotenv
from htmlTemplates import css, bot_template, user_template
from Backend_logic import pdf_to_text , text_to_chunk , get_vectorstore , conversation_chain

def handle_userinput(user_question):
    response = st.session_state.conversation({'question': user_question})
    st.session_state.chat_history = response['chat_history']

    for i, message in enumerate(st.session_state.chat_history):
        if i % 2 == 0:
            st.write(user_template.replace(
                "{{MSG}}", message.content), unsafe_allow_html=True)
        else:
            st.write(bot_template.replace(
                "{{MSG}}", message.content), unsafe_allow_html=True)

def main():
    load_dotenv()
    st.set_page_config(page_title="Chat with multiple PDFs",
                       page_icon=":books:")
    st.write(css, unsafe_allow_html=True)

    if "conversation" not in st.session_state:
        st.session_state.conversation = None
    if "chat_history" not in st.session_state:
        st.session_state.chat_history = None

    st.header("Chat with multiple PDFs :books:")
    user_question = st.text_input("Ask a question about your documents:")
    if user_question:
        handle_userinput(user_question)

    with st.sidebar:
        st.subheader("Your documents")
        pdf_docs = st.file_uploader(
            "Upload your PDFs here and click on 'Process'", accept_multiple_files=True)
        if st.button("Process"):
            with st.spinner("Processing"):
                # get pdf text
                raw_text = pdf_to_text(pdf_docs)

                # get the text chunks
                text_chunks = text_to_chunk(raw_text)

                # create vector store
                vectorstore = get_vectorstore(text_chunks)

                # create conversation chain
                st.session_state.conversation = conversation_chain(vectorstore)

if __name__ == '__main__':
    main()


from math import floor as floor

import matplotlib.animation as animation
import matplotlib.pyplot as plot
import matplotlib.style as style
import numpy as np
from configparser import ConfigParser
from tkinter import *

# ----------------------------------------------------------------
# Declaration of classes used in the Simulation
# Classes were required as this was not a simple model to solve
# ----------------------------------------------------------------
class Ecoli:
    def __init__(self, count):
        self.ecolicount = count

    @property
    def ecoli_count(self):
        return self.ecolicount

class Glucose:
    def __init__(self, count):
        self.glucosecount = count

    @property
    def glucose_count(self):
        return self.glucosecount

class Galactose:
    def __init__(self, count):
        self.galactosecount = count

    @property
    def galactose_count(self):
        return self.galactosecount

class Lactose:
    def __init__(self, count):
        self.lactosecount = count

    @property
    def lactose_count(self):
        return self.lactosecount

class LactaseEnzyme:
    def __init__(self, count):
        self.lactase = count

    @property
    def lactose_count(self):
        return self.lactase

class LactaseSwapPeriod:
    def __init__(self, count):
        self.lactaseSwap = count

    @property
    def lactase_swap_period(self):
        return self.lactaseSwap

# Create a parser object to read and read the conf.ini file
parser = ConfigParser()
parser.read("conf.ini")

# --------------------------------------------------------------------------
# Generate the option selection window and take the input from the user
# --------------------------------------------------------------------------
master = Tk()
var = StringVar()
Label(master, text="Select Simulation to run").grid(row=0, sticky=W)
Radiobutton(master, text="Normal", variable=var, value='Normal').grid(row=1, sticky=W)
Radiobutton(master, text="OnlyLactose", variable=var, value='OnlyLactose').grid(row=2, sticky=W)
Radiobutton(master, text="Starvation", variable=var, value='Starvation').grid(row=3, sticky=W)
Radiobutton(master, text="stable_colony", variable=var, value='stable_colony').grid(row=4, sticky=W)
Radiobutton(master, text="stable_colony_glucose", variable=var, value='stable_colony_glucose').grid(row=5, sticky=W)
Radiobutton(master, text="stable_colony_lactose", variable=var, value='stable_colony_lactose').grid(row=6, sticky=W)
Radiobutton(master, text="custom", variable=var, value='custom').grid(row=7, sticky=W)
Button(master, text="OK", command=master.quit).grid(row=8, sticky=W)
mainloop()

# ------------------------------------------------------------------------------------------------------------------
# Extract the Independent variable parameters from the Configuration file (conf.ini) based on the selection
# ------------------------------------------------------------------------------------------------------------------
selection = var.get()

# Set number of iterations
itr = parser.getint(selection, "iterations")
# Generate the time axis
time = np.linspace(0, itr + 1, itr + 1)
# Set the starting ecoli count
ecoli_count = Ecoli(parser.getint(selection, "ecoli"))
# Set the starting glucose count
glucose_count = Glucose(parser.getint(selection, "glucose_count"))
# Set the starting galactose count
galactose_count = Galactose(parser.getint(selection, "galactose_count"))
# Set the starting Lactose count
lactose_count = Lactose(parser.getint(selection, "lactose_count"))
# Set the starting Lactase count
beta_galactosidase = LactaseEnzyme(parser.getint(selection, "lactase"))
# --------------------------------------------------------------------------------------
# Set the replenishment values if they are set
# --------------------------------------------------------------------------------------
replenish_glucose = parser.getint(selection, "replenish_glucose") if (
    parser.has_option(selection, "replenish_glucose")) else 0
replenish_galactose = parser.getint(selection, "replenish_galactose") if (
    parser.has_option(selection, "replenish_galactose")) else 0
replenish_lactose = parser.getint(selection, "replenish_lactose") if (
    parser.has_option(selection, "replenish_lactose")) else 0
replenish_interval = parser.getint(selection, "replenish_interval") if (
    parser.has_option(selection, "replenish_interval")) else 0

# ----------------------------------------------------------------------------------------------------------------------
# To make our model realistic we have divided the ecoli into sample frames
# Since 1 unit of time is 10min are each E.Coli replicate at 2 iterations
# We treat E.coli as a batch of 2 sets ( 1st element of array and 2nd element)
# This was done to simulate an environment where an Ecoli cell always divides at any given time like a biological ecoli.
# ----------------------------------------------------------------------------------------------------------------------
sample_frame = 2
# Initialize a variable to keep track of Lactose swap period; at 2 the colony swaps to lactose consumption
lactose_swap_period = LactaseSwapPeriod(0)

# Create a numpy array, generic method
def create_seed_array(initial_seed, dummy, frame, extra=1):
    return np.concatenate([np.full(frame, initial_seed), np.full(itr + extra - sample_frame, dummy)], axis=0)

# ----------------------------------------------------------------------------------------------------------------------
# Create seed for Ecoli, Glucose, Galactose, Lactose, Lactase
# ----------------------------------------------------------------------------------------------------------------------
dedt = create_seed_array(ecoli_count.ecoli_count, 0, sample_frame)
dgdt = create_seed_array(glucose_count.glucose_count, 0, 1, 2)
dgadt = create_seed_array(galactose_count.galactose_count, 0, 1, 2)
dldt = create_seed_array(lactose_count.lactose_count, 0, 1, 2)
lactaseDt = create_seed_array(10 * ecoli_count.ecoli_count, 0, 1, 2)
# Rate at which each E.Coli consumes the resources
resource_consumption = 0.12
# Rate at which the lactase enzyme deplete when the E.Coli uses only Glucose and Galactose
lactase_enzyme_depletion_rate = 0.1

# ----------------------------------------------------------------------------------------------------------------------
# An internal helper method to fetch the parent of the sample we are replicating
# Ex: Sample_frame = 2;
#     at iteration 3 -> the E.Coli colony to replicate would be 1 (itr - sample_frame)
#     at iteration 4 -> The E.Coli colony to replicate would be 2
#     at iteration 5 -> Parent would be iteration 3 output.
# ----------------------------------------------------------------------------------------------------------------------
def get_indx_to_multiply(baseIndx):
    if baseIndx < sample_frame:
        return baseIndx
    else:
        return baseIndx - sample_frame

# ----------------------------------------------------------------------------------------------------------------------
# Get the available Glucose and Galactose available in the system
# ----------------------------------------------------------------------------------------------------------------------
def get_available_glucose_and_galactose():
    return glucose_count.glucose_count + galactose_count.galactose_count

# ----------------------------------------------------------------------------------------------------------------------
# Checks if the E.Coli colony is extinct
# ----------------------------------------------------------------------------------------------------------------------
def check_if_colony_is_dead():
    return ecoli_count.ecolicount <= 0

# ----------------------------------------------------------------------------------------------------------------------
# Calculate the possible lactose metabolism using Lactase count
# ----------------------------------------------------------------------------------------------------------------------
def calc_possible_lactose_metabolism(cellColony, lactoseAvailable):
    possible_metabolism = floor(beta_galactosidase.lactase / 100)
    return possible_metabolism if (lactoseAvailable >= possible_metabolism) else lactoseAvailable

# ----------------------------------------------------------------------------------------------------------------------
# Handles the multiplication and resource consumption of the colony when there is sufficient resources
# ----------------------------------------------------------------------------------------------------------------------
def multiply_under_adundant_resources(expected_resource_requirement,
                                      expected_ecoli_division,
                                      itr):
    # Deduct the require resource count from Glucose and Galactose
    glucose_consumed = glucose_count.glucose_count - expected_resource_requirement
    deduct_from_galactose = (galactose_count.galactose_count - glucose_consumed) if glucose_consumed < 0 else 0
    glucose_count.glucosecount = glucose_consumed if glucose_consumed >= 0 else 0
    galactose_count.galactosecount = (galactose_count.galactose_count - deduct_from_galactose) if glucose_consumed >= 0 else 0

    # Multiply the E.coli * 2 as there is abundant resources
    ecoli_count.ecolicount = expected_ecoli_division
    # Store the E.Coli colony status to the numpyArray to plot later
    dedt[itr] = ecoli_count.ecoli_count

    # Record the status of all the resources for plotting
    dgdt[itr] = glucose_count.glucose_count
    dgadt[itr] = galactose_count.galactose_count
    dldt[itr] = lactose_count.lactose_count

    # For every new E.Coli in the system we add 10 lactase enzymes so as to have a minimum number of lactase in the system.
    # This is done to prevent the model from being punished by lack of Lactase enzyme when the colony grows massively
    curr_lactase_count = lactaseDt[itr:itr + 1]
    new_lactase = curr_lactase_count - curr_lactase_count * lactase_enzyme_depletion_rate
    lactaseDt[itr] = new_lactase if new_lactase >= ecoli_count.ecoli_count * 10 else curr_lactase_count

# ----------------------------------------------------------------------------------------------------------------------
# A method that Orchestrates the replication and resource utilization of the model
# ----------------------------------------------------------------------------------------------------------------------
def consume_resources_and_multiply(itr):
    colony_status = check_if_colony_is_dead()
    if colony_status:
        # Colony is dead, set the resources to the count it was prior to extinction of the colony
        print("The E-coli colony is dead")
        dldt[itr] = lactose_count.lactose_count
        dgdt[itr] = glucose_count.glucose_count
        dgadt[itr] = galactose_count.galactose_count
        lactaseDt[itr] = beta_galactosidase.lactase
    else:
        # Get the parent index to multiply
        multiplying_indx = get_indx_to_multiply(itr)
        # Get the E.coli count from the parent index
        curr_ecoli_count = dedt[multiplying_indx]

        # Get a colony slice to compute the Lactase available in the system
        # The colony is bucketed into sample_frame but the environment they live in is the same
        colony_slice = dedt[itr - sample_frame:itr] if itr >= sample_frame else dedt[0:itr + 1]
        # Expected cell division under Glucose environment
        expected_ecoli_division = curr_ecoli_count * 2
        # Required Glucose and Galactose resources for the cell division to occur
        expected_resource_requirement = curr_ecoli_count * resource_consumption
        # Available Glucose and Galactose in the system
        available_glucose_galactose = get_available_glucose_and_galactose()
        # The amount of Lactose that can be metabolised (LM equation in the report)
        possible_lactose_metabolism = calc_possible_lactose_metabolism(colony_slice, lactose_count.lactose_count)
        # Total resources Glucose, Glactose and Lactose combined
        total_resources_available = available_glucose_galactose + possible_lactose_metabolism
        # State of the system, decides if the colony has sufficient resources or not
        resource_state = total_resources_available - expected_resource_requirement
        # Lactase enzyme present in the system, stored for plotting
        lactaseDt[itr] = beta_galactosidase.lactase

        # If the E.Coli colony is in a Lactose consumption phase then metabolize the Lactose to Glucose and Galactose
        if lactose_swap_period.lactase_swap_period > 2:
            glucose_count.glucosecount += possible_lactose_metabolism
            galactose_count.galactosecount += possible_lactose_metabolism
            lactose_count.lactosecount -= possible_lactose_metabolism
            # Reducing the Lactase enzyme
            beta_galactosidase.lactase = (beta_galactosidase.lactase + ecoli_count.ecoli_count * 50) if(lactose_count.lactose_count > 0) else beta_galactosidase.lactase
            # Reset the Lactase enzyme value again for plotting
            lactaseDt[itr] = beta_galactosidase.lactase

        # if Lactose is the only source, decrease the growth by 20%
        expected_ecoli_division -= 0 if available_glucose_galactose > 0 else expected_ecoli_division * 0.20

        # If there is Sufficient resources, including Lactose
        if resource_state >= 0:
            # Check if Glucose and Galactose is sufficient
            # if insufficient, start the Lactose swap counter to 1, else decrease it by 1
            if available_glucose_galactose >= expected_resource_requirement and lactose_swap_period.lactase_swap_period > 0:
                lactose_swap_period.lactaseSwap = lactose_swap_period.lactase_swap_period - 1
            else:
                lactose_swap_period.lactaseSwap = lactose_swap_period.lactase_swap_period + 1

            # Multiply the Colony and Consume the resources
            multiply_under_adundant_resources(expected_resource_requirement,
                                              expected_ecoli_division,
                                              itr)

        # Colony starvation - Assuming colony survives for 2 iterations and kills 30% of the population
        elif resource_state <= 0:
            # Available Glucose and Galctose
            available_raw_resource = get_available_glucose_and_galactose()
            # Part of the Colony under starvation
            cells_under_starvation = expected_ecoli_division - (available_raw_resource / resource_consumption)
            # Part of the Colony who received sufficient resources
            cells_with_sufficient_resources = expected_ecoli_division - cells_under_starvation
            # Divide the cells which had sufficient resources only
            dedt[itr] = ecoli_count.ecoli_count + cells_with_sufficient_resources
            # Kill 30% of the E.Coli
            dedt[itr] = (dedt[itr] - dedt[itr] * 0.3)
            # Set the Ecoli count for plotting
            ecoli_count.ecolicount = dedt[itr]
            # Set other resource values for plotting
            dgdt[itr] = 0
            glucose_count.glucosecount = 0
            dgadt[itr] = 0
            galactose_count.galactosecount = 0
            lactose_count.lactosecount = dldt[itr] = lactose_count.lactose_count - possible_lactose_metabolism
            lactaseDt[itr] = beta_galactosidase.lactase
        else:
            # This else part never occurs, this is just a placeholder.
            # By the time resources are completely empty the colony would be dead
            dedt[itr] = ecoli_count.ecoli_count
            dgdt[itr] = glucose_count.glucose_count
            dgadt[itr] = galactose_count.galactose_count
            dldt[itr] = lactose_count.lactose_count
            lactaseDt[itr] = beta_galactosidase.lactase

# Iterate the model for itr number of times
for i in range(itr + 1):
    # If Replenish interval has been set, replenish the resources at its interval
    if replenish_interval != 0 and itr % replenish_interval == 0:
        glucose_count.glucosecount += replenish_glucose
        galactose_count.galactosecount += replenish_galactose
        lactose_count.lactosecount += replenish_lactose

    # Start the simulation for the iteration
    consume_resources_and_multiply(i)

# Create subplots for the graphs
style.use("fivethirtyeight")
fig = plot.figure("Ecoli bacterial growth simulation:" + selection)
ax = fig.add_subplot(2, 2, 1)
ax2 = fig.add_subplot(2, 2, 2)
ax3 = fig.add_subplot(2, 2, 3)
ax4 = fig.add_subplot(2, 2, 4)

# Helper function used by animation to animate the simulation
def displayPlot(i):
    ax.cla()
    ax2.cla()
    ax3.cla()
    ax4.cla()
    ax.title.set_text('Ecoli bacterial growth simulation:' + selection)
    if parser.getboolean("display_control", "plot_ecoli"):
        ax.plot(time[0:i], dedt[0:i], 'g', label='Ecoli-Multiplication')
    if parser.getboolean("display_control", "plot_glucose"):
        ax2.plot(time[0:i], dgdt[0:i], 'r', label='Glucose-Consumption')
    if parser.getboolean("display_control", "plot_galactose"):
        ax2.plot(time[0:i], dgadt[0:i], 'y', linestyle="-.", label='GalactoseConsumption')
    if parser.getboolean("display_control", "plot_lactase"):
        ax3.plot(time[0:i], lactaseDt[0:i], 'b', label='Lactase enzyme')
    if parser.getboolean("display_control", "plot_lactose"):
        ax4.plot(time[0:i], dldt[0:i], 'c', linestyle="--", label='Lactose Consumption')
    ax.legend()
    ax.set_xlabel('Time in min')
    ax.set_ylabel('Ecoli Growth')
    ax2.legend()
    ax2.set_xlabel('Time in min')
    ax2.set_ylabel('Glucose + Galactose')
    ax3.legend()
    ax3.set_xlabel('Time in min')
    ax3.set_ylabel('Lactase enzyme')
    ax4.legend()
    ax4.set_xlabel('Time in min')
    ax4.set_ylabel('Lactose Consumption')

# Animation support with interval 500, 0.5sec
ani = animation.FuncAnimation(fig, displayPlot, interval=500)
plot.tight_layout()
plot.show()


import logging

from colorlog import ColoredFormatter

LOGGER = {}
LOG_LEVEL = logging.DEBUG
LOGFORMAT = (
    "[%(log_color)s%(levelname)s%(reset)s] "
    "%(log_color)s%(message)s%(reset)s"
)

def get_logger(name):
    """Returns specific logger instance."""
    try:
        return LOGGER[name]
    except KeyError:
        logging.root.setLevel(LOG_LEVEL)
        formatter = ColoredFormatter(LOGFORMAT)
        stream = logging.StreamHandler()
        stream.setLevel(LOG_LEVEL)
        stream.setFormatter(formatter)
        log = logging.getLogger(name)
        log.setLevel(LOG_LEVEL)
        log.addHandler(stream)

        # Store the logger to avoid duplicate handler which causing duplicate
        # message and return it later when same logger is asked.
        LOGGER[name] = log
        return log


from keras.models import Sequential
from keras.layers import Dense, LSTM
import tensorflow as tf
import numpy as np
import keras
import pandas as pd
from tqdm import tqdm
import os
from tensorflow.keras.callbacks import ModelCheckpoint

class LstmPredictor:

    def __init__(self, n_input, n_output, n_feature, file=None):
        model = Sequential()
        model.add(LSTM(n_input, return_sequences=True))
        model.add(LSTM(n_input * n_feature, return_sequences=False))
        model.add(Dense(n_input * n_feature * 10))
        model.add(Dense(n_input * n_feature * 5))
        model.add(Dense(n_output * n_feature))
        model.add(tf.keras.layers.Reshape((n_output, n_feature)))
        self.model: Sequential = model
        if file is not None:
            self.model: Sequential = keras.models.load_model(file)
        self.n_output = n_output
        self.n_feature = n_feature

    def train(self, train_seq, val_data, epochs, model_path):
        ck_pt = ModelCheckpoint(model_path, monitor='val_loss', verbose=1, save_best_only=True, mode='min')
        self.model.compile(optimizer="adam", loss="mean_squared_error")
        self.model.fit(x=train_seq, validation_data=val_data, epochs=epochs, callbacks=[ck_pt], verbose=1)

    def predict(self, x, chunk_size=None):
        """
        x.shape : [batch_size, time_steps, features]
        """
        scale_max = np.max(x, axis=1)[:, np.newaxis, :]
        scale_min = np.min(x, axis=1)[:, np.newaxis, :]
        scale_range = np.maximum(scale_max - scale_min, np.finfo(float).eps)
        x_scaled = (x - scale_min) / scale_range
        total_size = len(x)
        if chunk_size is None:
            chunk_size = total_size
        indices = list(np.arange(0, total_size, chunk_size))
        if total_size != indices[-1]:
            indices = indices + [total_size]
        y_scaled = np.zeros((0, self.n_output, self.n_feature))
        print("Predicting per chunk:")
        for i in tqdm(range(len(indices) - 1)):
            n1, n2 = indices[i], indices[i+1]
            _y = self.model.predict(x_scaled[n1:n2, :, :])
            y_scaled = np.concatenate((y_scaled, _y), axis=0)
        y = y_scaled * scale_range + scale_min
        return y

class DataSequence(keras.utils.Sequence):

    def __init__(self, batch_dir, batch_size, n_input, n_output, n_feature):
        self.length = len(os.listdir(batch_dir))
        self.paths = []
        for f in os.listdir(batch_dir):
            self.paths.append(os.path.join(batch_dir, f))
        self.n_feature = n_feature
        self.n_input = n_input
        self.n_output = n_output
        self.batch_size = batch_size

    def __len__(self):
        return self.length

    def __getitem__(self, batch_index):
        df = pd.read_csv(self.paths[batch_index], header=None)
        arr = df.to_numpy()
        nfe = self.n_feature
        nou = self.n_output
        nin = self.n_input
        batch_x = np.reshape(arr[:, :nin * nfe], (self.batch_size, nin, nfe))
        batch_y = np.reshape(arr[:, nin * nfe:], (self.batch_size, nou, nfe))
        scale_max = np.max(batch_x, axis=1)[:, np.newaxis, :]
        scale_min = np.min(batch_x, axis=1)[:, np.newaxis, :]
        batch_x = (batch_x - scale_min) / (scale_max - scale_min)
        batch_y = (batch_y - scale_min) / (scale_max - scale_min)
        return batch_x, batch_y

    def convert_to_data(self):
        print("Sequence: converting to data")
        x, y = [], []
        for i in tqdm(range(self.__len__())):
            batch_x, batch_y = self.__getitem__(i)
            x.extend(batch_x)
            y.extend(batch_y)
        x, y = np.asarray(x), np.asarray(y)
        return x, y


from fastsklearnfeature.candidates.CandidateFeature import CandidateFeature
from fastsklearnfeature.transformations.Transformation import Transformation
from typing import List
import numpy as np
from fastsklearnfeature.reader.Reader import Reader
from fastsklearnfeature.splitting.Splitter import Splitter
import time
from fastsklearnfeature.candidate_generation.explorekit.Generator import Generator
from fastsklearnfeature.candidates.RawFeature import RawFeature
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import make_scorer
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import LabelEncoder
import numpy as np
from fastfeature.plotting.plotter import cool_plotting
import pickle
from sklearn.model_selection import GridSearchCV
import multiprocessing as mp
from sklearn.pipeline import Pipeline
from sklearn.model_selection import StratifiedKFold
from fastsklearnfeature.configuration.Config import Config

class ExploreKitSelection:
    def __init__(self, dataset_config, classifier=LogisticRegression(), grid_search_parameters={'classifier__penalty': ['l2'],
                                                                                'classifier__C': [0.001, 0.01, 0.1, 1, 10, 100, 1000],
                                                                                'classifier__solver': ['lbfgs']}):
        self.dataset_config = dataset_config
        self.classifier = classifier
        self.grid_search_parameters = grid_search_parameters

    #generate all possible combinations of features
    def generate(self):

        s = Splitter(train_fraction=[0.6, 10000000], seed=42)
        #s = Splitter(train_fraction=[0.1, 10000000], seed=42)

        self.dataset = Reader(self.dataset_config[0], self.dataset_config[1], s)
        raw_features = self.dataset.read()

        g = Generator(raw_features)
        self.candidates = g.generate_all_candidates()
        print("Number candidates: " + str(len(self.candidates)))

    #rank and select features
    def random_select(self, k: int):
        arr = np.arange(len(self.candidates))
        np.random.shuffle(arr)
        return arr[0:k]

    def select_interpretable(self, k: int):
        inv_map = {v: k for k, v in self.candidate_id_to_ranked_id.items()}
        selected = []
        for i in range(k):
            selected.append(inv_map[i])
        return selected

    def generate_target(self):
        current_target = self.dataset.splitted_target['train']
        self.current_target = LabelEncoder().fit_transform(current_target)

    def evaluate(self, candidate: CandidateFeature, runs=10):
        for i in range(runs):
            candidate.pipeline.fit(self.dataset.splitted_values['train'])
            candidate.pipeline.transform(self.dataset.splitted_values['train'])

    def create_starting_features(self):
        Fi: List[RawFeature]= self.dataset.raw_features

        #materialize and numpyfy the features
        starting_feature_matrix = np.zeros((Fi[0].materialize()['train'].shape[0], len(Fi)))
        for f_index in range(len(Fi)):
            starting_feature_matrix[:, f_index] = Fi[f_index].materialize()['train']
        return starting_feature_matrix

    def my_arg_sort(self, seq):
        # http://stackoverflow.com/questions/3382352/equivalent-of-numpy-argsort-in-basic-python/3383106#3383106
        # non-lambda version by Tony Veijalainen
        return [i for (v, i) in sorted((v, i) for (i, v) in enumerate(seq))]

    def get_interpretability_ranking(self):
        #high interpretability -> low interpretability
        interpretability_ids = self.my_arg_sort(self.candidates)

        self.candidate_id_to_ranked_id = {}
        for i in range(len(interpretability_ids)):
            self.candidate_id_to_ranked_id[interpretability_ids[i]] = i

    def get_traceability_ranking(self):
        # high traceability -> low traceability
        self.traceability: List[float] = []
        for c_i in range(len(self.candidates)):
            self.traceability.append(self.candidates[c_i].calculate_traceability())
        ids = np.argsort(np.array(self.traceability)*-1)

        self.candidate_id_to_ranked_id = {}
        for i in range(len(ids)):
            self.candidate_id_to_ranked_id[ids[i]] = i

        all_data = {}
        all_data['my_dict'] = self.candidate_id_to_ranked_id
        all_data['traceability'] = self.traceability
        pickle.dump(all_data, open("/tmp/traceability.p", "wb"))

    def get_interpretability(self, candidate_id):
        return 1.0 - ((self.candidate_id_to_ranked_id[candidate_id] + 1) / float(len(self.candidate_id_to_ranked_id)))

    def evaluate_candidates(self, candidates):
        self.preprocessed_folds = []
        for train, test in StratifiedKFold(n_splits=10, random_state=42).split(self.dataset.splitted_values['train'], self.current_target):
            self.preprocessed_folds.append((train, test))

        pool = mp.Pool(processes=int(Config.get("parallelism")))
        results = pool.map(self.evaluate_single_candidate, candidates)
        return results

    '''
    def evaluate_candidates(self, candidates):
        self.preprocessed_folds = []
        for train, test in StratifiedKFold(n_splits=10, random_state=42).split(self.dataset.splitted_values['train'],
                                                                               self.current_target):
            self.preprocessed_folds.append((train, test))

        results = []
        for c in candidates:
            results.append(self.evaluate_single_candidate(c))
        return results

    '''

    def evaluate_single_candidate(self, candidate):
        result = {}
        time_start_gs = time.time()
        runs = 10
        try:
            self.evaluate(candidate, runs)
            #print("feature: " + str(candidate) + " -> " + str(new_score))
        except Exception as e:
            print(str(candidate) + " -> " + str(e))
            pass
        result['candidate'] = candidate
        result['time'] = (time.time() - time_start_gs) / float(runs)
        return result

    '''
    def evaluate_single_candidate(self, candidate):
        new_score = -1.0
        new_score = self.evaluate(candidate)
        return new_score
    '''

    def run(self):
        # generate all candidates
        self.generate()
        #starting_feature_matrix = self.create_starting_features()
        self.generate_target()

        candidate_name_to_id = {}
        for c_i in range(len(self.candidates)):
            candidate_name_to_id[self.candidates[c_i].get_name()] = c_i

        pickle.dump(candidate_name_to_id, open("/tmp/name2id.p", "wb"))

        pickle.dump(self.candidates, open("/tmp/all_candiates.p", "wb"))

        self.get_interpretability_ranking()
        #self.get_traceability_ranking()

        #evaluate starting matrix
        #start_score = self.evaluate(starting_feature_matrix)
        start_score = -1
        print("start score: " + str(start_score))

        #get candidates that should be evaluated
        ranked_selected_candidate_ids = self.select_interpretable(len(self.candidates))

        start_time = time.time()

        results = self.evaluate_candidates(np.array(self.candidates)[ranked_selected_candidate_ids])

        print("evaluation time: " + str((time.time()-start_time) / 60) + " min")

        return start_score, results, ranked_selected_candidate_ids

#statlog_heart.csv=/home/felix/datasets/ExploreKit/csv/dataset_53_heart-statlog_heart.csv
#statlog_heart.target=13

if __name__ == '__main__':
    dataset = (Config.get('statlog_heart.csv'), int(Config.get('statlog_heart.target')))
    #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_27_colic_horse.csv", 22)
    #dataset = ("/home/felix/datasets/ExploreKit/csv/phpAmSP4g_cancer.csv", 30)
    # dataset = ("/home/felix/datasets/ExploreKit/csv/phpOJxGL9_indianliver.csv", 10)
    # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_29_credit-a_credit.csv", 15)
    #dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_37_diabetes_diabetes.csv", 8)
    # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_31_credit-g_german_credit.csv", 20)
    # dataset = ("/home/felix/datasets/ExploreKit/csv/dataset_23_cmc_contraceptive.csv", 9)
    # dataset = ("/home/felix/datasets/ExploreKit/csv/phpn1jVwe_mammography.csv", 6)

    selector = ExploreKitSelection(dataset)
    #selector = ExploreKitSelection(dataset, KNeighborsClassifier(), {'n_neighbors': np.arange(3,10), 'weights': ['uniform','distance'], 'metric': ['minkowski','euclidean','manhattan']})

    start_score, results, ids = selector.run()

    pickle.dump(results, open("/tmp/all_data_runtime.p", "wb"))


# coding=utf-8
import json
import os
import pandas as pd
import requests
from bs4 import BeautifulSoup
import time

def get_info_from_zhilian():
    print('reading begin...')

    df = pd.DataFrame([])

    output_file_name = 'output-zhilian.xlsx'
    keyword = 'java'

    for page in range(0, 5):
        df_temp = get_info_by_page(page, keyword)
        df = pd.concat([df, df_temp], ignore_index=True)

    df = df.drop_duplicates(subset='SOU_POSITION_ID', keep='first', inplace=False)

    if os.path.exists(output_file_name):
        os.remove(output_file_name)

    df.to_excel(output_file_name, index=False)

    print('read success...')

def get_info_by_page(page, keyword):
    print('reading ' + str(page + 1) + ' page begin...')
    url = 'https://fe-api.zhaopin.com/c/i/sou'

    params = {
        'start': 90 * page,
        'pageSize': '90',
        'cityId': '765',
        'salary': '10001,15000',
        'workExperience': '0103',
        'education': '-1',
        'companyType': '-1',
        'employmentType': '-1',
        'jobWelfareTag': '-1',
        'kw': keyword,
        'kt': '3',
        'at': '614336692cf84336b01268b3b844153f',
        'rt': 'd93899c208764ebe83c99293b94e1248',
        '_v': '0.49083468',
        'userCode': '692270980',
        'x-zp-page-request-id': 'be6dd0fa12bd481a9b96c6550052278c-1551151427718-627218',
    }

    content = requests.get(url, params=params)

    json1 = json.loads(content.text)

    df = pd.DataFrame(json1['data']['results'])

    required_columns = ['SOU_POSITION_ID', 'jobName', 'company', 'salary', 'businessArea', 'jobName', 'positionURL']

    df = df[required_columns]

    df['company_name'] = df.apply(lambda x: x['company']['name'], axis=1)
    df['company_type'] = df.apply(lambda x: x['company']['type']['name'], axis=1)
    df['company_size'] = df.apply(lambda x: x['company']['size']['name'], axis=1)
    df['detail'] = df.apply(lambda x: get_detail_by_url(x['positionURL']), axis=1)

    df = df.drop(columns=['company'])

    print('reading ' + str(page + 1) + ' page success...')

    return df

def get_detail_by_url(url):
    page = requests.get(url)
    soup = BeautifulSoup(page.text, "lxml")

    # time.sleep()

    if soup.select('div.pos-ul')[0]:
        return soup.select('div.pos-ul')[0].get_text()
    else:
        return '暂未查到信息'

if __name__ == '__main__':
    get_info_from_zhilian()


# coding: utf-8

# ## 探索电影数据集
#
# 在这个项目中，你将尝试使用所学的知识，使用 `NumPy`、`Pandas`、`matplotlib`、`seaborn` 库中的函数，来对电影数据集进行探索。
#
# 下载数据集：
# [TMDb电影数据](https://s3.cn-north-1.amazonaws.com.cn/static-documents/nd101/explore+dataset/tmdb-movies.csv)
#

#
# 数据集各列名称的含义：
#

#

列名称	id	imdb_id	popularity	budget	revenue	original_title	cast	homepage	director	tagline	keywords	overview	runtime	genres	production_companies	release_date	vote_count	vote_average	release_year	budget_adj	revenue_adj

#
含义	编号	IMDB 编号	知名度	预算	票房	名称	主演	网站	导演	宣传词	关键词	简介	时常	类别	发行公司	发行日期	投票总数	投票均值	发行年份	预算（调整后）	票房（调整后）

#

#

# **请注意，你需要提交该报告导出的 `.html`、`.ipynb` 以及 `.py` 文件。**

#
#
# ---
#
# ---
#
# ## 第一节 数据的导入与处理
#
# 在这一部分，你需要编写代码，使用 Pandas 读取数据，并进行预处理。

#
# **任务1.1：** 导入库以及数据
#
# 1. 载入需要的库 `NumPy`、`Pandas`、`matplotlib`、`seaborn`。
# 2. 利用 `Pandas` 库，读取 `tmdb-movies.csv` 中的数据，保存为 `movie_data`。
#
# 提示：记得使用 notebook 中的魔法指令 `%matplotlib inline`，否则会导致你接下来无法打印出图像。

# In[2]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sb
get_ipython().run_line_magic('matplotlib', 'inline')

movie_data = pd.read_csv('./tmdb-movies.csv')

# ---
#
# **任务1.2: ** 了解数据
#
# 你会接触到各种各样的数据表，因此在读取之后，我们有必要通过一些简单的方法，来了解我们数据表是什么样子的。
#
# 1. 获取数据表的行列，并打印。
# 2. 使用 `.head()`、`.tail()`、`.sample()` 方法，观察、了解数据表的情况。
# 3. 使用 `.dtypes` 属性，来查看各列数据的数据类型。
# 4. 使用 `isnull()` 配合 `.any()` 等方法，来查看各列是否存在空值。
# 5. 使用 `.describe()` 方法，看看数据表中数值型的数据是怎么分布的。
#
#

# In[3]:

#print(movie_data.head())
#print(movie_data.tail())
#print(movie_data.sample())
#print(movie_data['cast'].dtypes)
#print(movie_data['cast'].isnull())
#print(movie_data.describe())
print(movie_data.shape)

# ---
#
# **任务1.3: ** 清理数据
#
# 在真实的工作场景中，数据处理往往是最为费时费力的环节。但是幸运的是，我们提供给大家的 tmdb 数据集非常的“干净”，不需要大家做特别多的数据清洗以及处理工作。在这一步中，你的核心的工作主要是对数据表中的空值进行处理。你可以使用 `.fillna()` 来填补空值，当然也可以使用 `.dropna()` 来丢弃数据表中包含空值的某些行或者列。
#
# 任务：使用适当的方法来清理空值，并将得到的数据保存。

# In[8]:

c_data = movie_data.fillna(0)
print(c_data.shape)

# ---
#
# ---
#
# ## 第二节 根据指定要求读取数据
#
#
# 相比 Excel 等数据分析软件，Pandas 的一大特长在于，能够轻松地基于复杂的逻辑选择合适的数据。因此，如何根据指定的要求，从数据表当获取适当的数据，是使用 Pandas 中非常重要的技能，也是本节重点考察大家的内容。
#
#

# ---
#
# **任务2.1: ** 简单读取
#
# 1. 读取数据表中名为 `id`、`popularity`、`budget`、`runtime`、`vote_average` 列的数据。
# 2. 读取数据表中前1～20行以及48、49行的数据。
# 3. 读取数据表中第50～60行的 `popularity` 那一列的数据。
#
# 要求：每一个语句只能用一行代码实现。

# In[16]:

print(c_data[['id','popularity','budget','runtime','vote_average']])
print(c_data.loc[0:19].append(c_data.loc[47:48]))
print(c_data['popularity'][49:60])

# ---
#
# **任务2.2: **逻辑读取（Logical Indexing）
#
# 1. 读取数据表中 **`popularity` 大于5** 的所有数据。
# 2. 读取数据表中 **`popularity` 大于5** 的所有数据且**发行年份在1996年之后**的所有数据。
#
# 提示：Pandas 中的逻辑运算符如 `&`、`|`，分别代表`且`以及`或`。
#
# 要求：请使用 Logical Indexing实现。

# In[6]:

#print(c_data[c_data['popularity'] > 5]['popularity'])
print(c_data[(c_data['popularity'] > 5) & (c_data['release_year'] > 1996)])

# ---
#
# **任务2.3: **分组读取
#
# 1. 对 `release_year` 进行分组，使用 [`.agg`](http://pandas.pydata.org/pandas-docs/version/0.22/generated/pandas.core.groupby.DataFrameGroupBy.agg.html) 获得 `revenue` 的均值。
# 2. 对 `director` 进行分组，使用 [`.agg`](http://pandas.pydata.org/pandas-docs/version/0.22/generated/pandas.core.groupby.DataFrameGroupBy.agg.html) 获得 `popularity` 的均值，从高到低排列。
#
# 要求：使用 `Groupby` 命令实现。

# In[7]:

#c_data.groupby('release_year').agg({'revenue' : 'mean'})
c_data.groupby('director').agg({'popularity' : 'mean'}).sort_values(by = 'popularity', ascending = False)

# ---
#
# ---
#
# ## 第三节 绘图与可视化
#
# 接着你要尝试对你的数据进行图像的绘制以及可视化。这一节最重要的是，你能够选择合适的图像，对特定的可视化目标进行可视化。所谓可视化的目标，是你希望从可视化的过程中，观察到怎样的信息以及变化。例如，观察票房随着时间的变化、哪个导演最受欢迎等。
#
#

#

可视化的目标	可以使用的图像

#
表示某一属性数据的分布	饼图、直方图、散点图

#
表示某一属性数据随着某一个变量变化	条形图、折线图、热力图

#
比较多个属性的数据之间的关系	散点图、小提琴图、堆积条形图、堆积折线图

#

#
# 在这个部分，你需要根据题目中问题，选择适当的可视化图像进行绘制，并进行相应的分析。对于选做题，他们具有一定的难度，你可以尝试挑战一下～

# **任务3.1：**对 `popularity` 最高的20名电影绘制其 `popularity` 值。

# In[28]:

c_data.sort_values('popularity', ascending=False).head(20).plot(kind='barh', y='popularity', x='original_title', legend=False)
plt.xlabel('popularity')

# ---
# **任务3.2：**分析电影净利润（票房-成本）随着年份变化的情况，并简单进行分析。

# In[30]:

plt.style.use('ggplot')
_,axes = plt.subplots(2,1,figsize = (10,10))
movie_data['profit'] = movie_data['revenue'] - movie_data['budget']
target_data = movie_data.groupby('release_year')['profit'].agg(['mean','sum'])

#print(target_data)
axes[0].errorbar(target_data.index, target_data['mean'])
axes[1].errorbar(target_data.index, target_data['sum'])

axes[0].set_ylabel('profit_mean')
axes[1].set_ylabel('profit_sum')
axes[1].set_xlabel('release year')
#纯利润在1960-1975年逐渐上升，并在1975左右达到顶峰，然后一直呈下降趋势

# ---
#
# **[选做]任务3.3：**选择最多产的10位导演（电影数量最多的），绘制他们排行前3的三部电影的票房情况，并简要进行分析。

# In[47]:

df = movie_data.dropna(subset=['director'])
top10_directors = df.director.value_counts()[0:10]
print(top10_directors)

df = df.loc[df.director.isin(top10_directors.index)]
df = df.sort_values('revenue', ascending = False).groupby('director').head(3)
df = df.sort_values('director')

fig = plt.figure(figsize = (15,10))
ax = sb.barplot(data = df,x = 'original_title', y = 'revenue_adj', hue = 'director', dodge=False)
plt.xticks(rotation = 90)
plt.xlabel('original_title')
plt.ylabel('revenue')

# ---
#
# **[选做]任务3.4：**分析1968年~2015年六月电影的数量的变化。

# In[29]:

df = movie_data.dropna(subset = ['release_date'])
df['release_date'] = pd.to_datetime(df['release_date'])
df[df['release_date'].dt.month == 6].groupby(df['release_year']).count()   .plot(kind = 'bar', y = 'id',figsize = (15,10),legend = False)
plt.ylabel('movie counts')

# ---
#
# **[选做]任务3.5：**分析1968年~2015年六月电影 `Comedy` 和 `Drama` 两类电影的数量的变化。

# In[44]:

# 获取所有的电影类型
df = movie_data
all_genre = set()
for i in df['genres'].unique():
  if isinstance(i,str):
      all_genre = all_genre.union([g for g in i.split('|')])
disp = pd.DataFrame(columns=['genres'])

# 分析时间戳
df['release_date'] = (pd.to_datetime(df['release_date']))
#统计对应的类别
for i,j in df[df['release_date'].dt.month==6].groupby(df['release_date'].dt.year):
  tmp_genre = {}
  for i in j['genres']:
      if not isinstance(i,str):
          continue
      for g in i.split('|'):
          tmp_genre[g] = tmp_genre.get(g, 0)+1
  cat = pd.DataFrame.from_dict(tmp_genre,orient='index').T
  cat.index = [j['release_date'].dt.year.values[0]]
  disp = disp.append(cat)

# 展示变化
disp = disp.fillna(0)
disp.loc[1968:2016,['Comedy', 'Drama']].plot()
plt.xlabel('year')
plt.ylabel('numer of moives')
plt.title('Change in number of moives in June since 1968 to 2015')

# > 注意: 当你写完了所有的代码，并且回答了所有的问题。你就可以把你的 iPython Notebook 导出成 HTML 文件。你可以在菜单栏，这样导出**File -> Download as -> HTML (.html)、Python (.py)** 把导出的 HTML、python文件 和这个 iPython notebook 一起提交给审阅者。


import os
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
import pandas as pd
import tweepy
import numpy as np
from transformers import BertTokenizer
import torch
from torch import nn
from transformers import BertModel
from torch.optim import Adam
from tqdm import tqdm
from collections import Counter
import random
import joblib
from sklearn.model_selection import train_test_split
from pathlib import Path

def split_into_512_approx(users, tweet_chunks_per_user, all_info, by_tweet=False):
    """This function will take in a list of users and construct either test,
    validation, or training dataframe.  tweet_chunks_per_user per user specifies how many chunks of tweets
    we wish to include for each user.  Each tweet chunk is a concatenation of tweets by a particular user.
    This function will ensure that no tweet chunk contains more than 500 words, as BERT can only handle
    512 tokens at a time. The chunks will be constructed with the longest tweets first, on the
    assumption that longer tweets would be more likely to contain political messaging.

    We could have run BERT with individual tweets as opposed to tweet chunks, and we did try that
    many times.  We aren't exactly sure why, but chunking the tweets together seemed to produce
    better results and made computation faster.

    This function was constructed so as to ensure that tweets from any particular
    politician only appear in only the test, validation or training set, not some
    combination of the three.
    """
    df_out = pd.DataFrame(columns=['Party', 'Text', 'UserID'])
    num_dem = 0
    num_rep = 0
    counter2 = 0
    for user in users:
        temp_df = all_info.loc[all_info['UserID'] == user]
        if temp_df['Party'].iloc[0] == 'Republican Party':
            num_rep += 1
        if temp_df['Party'].iloc[0] == 'Democratic Party':
            num_dem += 1
        max_indexes = []
        temp_texts = list(temp_df['Text'])
        lengths = np.zeros(len(temp_texts))
        for i in range(0, len(temp_texts)):
            lengths[i] = len(temp_texts[i].split(' '))
        for i in range(0, len(temp_texts)):
            temp_max_indi = lengths.argmax()
            lengths[temp_max_indi] = 0
            max_indexes.append(temp_max_indi)
        counter = 0
        big_counter = 0
        working_string = ''
        for index in max_indexes:
            data = temp_df['Text'].iloc[index]
            if counter >= tweet_chunks_per_user:
                break
            if by_tweet is True:
                df_out = df_out.append({'Party': temp_df['Party'].iloc[0], 'Text': data, 'UserID': user}, ignore_index=True)
                working_string = ''
                counter += 1
            if (len((working_string + ' ' + data).split(' ')) > 500):
                df_out = df_out.append({'Party': temp_df['Party'].iloc[0], 'Text': working_string, 'UserID': user}, ignore_index=True)
                working_string = ''
                counter += 1
            else:
                working_string = working_string + ' ' + data
                counter2 += 1
    return df_out

tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
labels = {'Democratic Party':0,'Republican Party':1}

# Dataset class
class Dataset(torch.utils.data.Dataset):

    def __init__(self, df):

        self.labels = [labels[label] for label in df['Party']]
        # I changed max_length to 70 here. Was 512.
        self.texts = [tokenizer(text,
                               padding='max_length', max_length = 70, truncation=True,
                                return_tensors="pt") for text in df['Text']]

    def classes(self):
        return self.labels

    def __len__(self):
        return len(self.labels)

    def get_batch_labels(self, idx):
        # Fetch a batch of labels
        return np.array(self.labels[idx])

    def get_batch_texts(self, idx):
        # Fetch a batch of inputs
        return self.texts[idx]

    def __getitem__(self, idx):

        batch_texts = self.get_batch_texts(idx)
        batch_y = self.get_batch_labels(idx)

        return batch_texts, batch_y

class BertClassifier(nn.Module):

    def __init__(self, dropout=0.5):

        super(BertClassifier, self).__init__()

        self.bert = BertModel.from_pretrained('bert-base-cased')
        self.dropout = nn.Dropout(dropout)
        self.linear = nn.Linear(768, 2)
        self.relu = nn.ReLU()

    def forward(self, input_id, mask):

        _, pooled_output = self.bert(input_ids= input_id, attention_mask=mask,return_dict=False)
        dropout_output = self.dropout(pooled_output)
        linear_output = self.linear(dropout_output)
        final_layer = self.relu(linear_output)

        return final_layer

def train(model, train_data, val_data, learning_rate, epochs):

    train, val = Dataset(train_data), Dataset(val_data)
    # batch_size = 2 is what produced the best results for us.
    # Increasing batch_size i.e. how many attention head calculations
    # before recalculating embeddings did make the code go faster,
    # but tended to affect performance.
    train_dataloader = torch.utils.data.DataLoader(train, batch_size=2, shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(val, batch_size=2)

    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")

    criterion = nn.CrossEntropyLoss()
    optimizer = Adam(model.parameters(), lr= learning_rate)

    if use_cuda:

            model = model.cuda()
            criterion = criterion.cuda()

    for epoch_num in range(epochs):

            total_acc_train = 0
            total_loss_train = 0

            for train_input, train_label in tqdm(train_dataloader):

                train_label = train_label.to(device)
                mask = train_input['attention_mask'].to(device)
                input_id = train_input['input_ids'].squeeze(1).to(device)

                output = model(input_id, mask)

                batch_loss = criterion(output, train_label.long())
                total_loss_train += batch_loss.item()

                acc = (output.argmax(dim=1) == train_label).sum().item()
                total_acc_train += acc

                model.zero_grad()
                batch_loss.backward()
                optimizer.step()

            total_acc_val = 0
            total_loss_val = 0

            with torch.no_grad():

                for val_input, val_label in val_dataloader:

                    val_label = val_label.to(device)
                    mask = val_input['attention_mask'].to(device)
                    input_id = val_input['input_ids'].squeeze(1).to(device)

                    output = model(input_id, mask)

                    batch_loss = criterion(output, val_label.long())
                    total_loss_val += batch_loss.item()

                    acc = (output.argmax(dim=1) == val_label).sum().item()
                    total_acc_val += acc

            print(
                f'Epochs: {epoch_num + 1} | Train Loss: {total_loss_train / len(train_data): .3f} \
                | Train Accuracy: {total_acc_train / len(train_data): .3f} \
                | Val Loss: {total_loss_val / len(val_data): .3f} \
                | Val Accuracy: {total_acc_val / len(val_data): .3f}')

# Note on hyperparameters: EPOCHS=5 and LR (learning rate) = 1e-6
# seems to be about right.  More EPOCHS results in overfitting with the
# size of data we were running with.  Increasing learning rate
# makes the results unstable.
EPOCHS = 5
model = BertClassifier()
LR = 1e-6

# Evaluate the model using testing data
def evaluate(model, test_data, user_IDs, save_results=False):

    test = Dataset(test_data)

    test_dataloader = torch.utils.data.DataLoader(test, batch_size=2)

    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")

    if use_cuda:

        model = model.cuda()

    total_acc_test = 0
    test_labels = []
    test_prediction = []
    correct_or_no = []
    counter = 0
    with torch.no_grad():

        for test_input, test_label in test_dataloader:
            test_label = test_label.to(device)
            test_labels.append(test_label)
            mask = test_input['attention_mask'].to(device)

            input_id = test_input['input_ids'].squeeze(1).to(device)
            output = model(input_id, mask)
            acc = (output.argmax(dim=1) == test_label).sum().item()
            if acc == 0:
                correct_or_no.append('no')
            else:
                correct_or_no.append('yes')
            for prediction in output.argmax(dim=1).flatten().tolist():
                test_prediction.append(prediction)
            total_acc_test += acc
            counter += 1
    test_data['UserID'] = user_IDs
    test_data['Predictions'] = test_prediction
    print(f'Test Accuracy: {total_acc_test / len(test_data): .3f}')

    if not save_results:
        return test_data
    else:
        test_data.to_csv(Path('Data/Raw_BERT_results/no_chunk_2_13_23.csv'))

def main():

    # First we balance the data set.
    df = pd.read_csv(Path('Data/text_party_IDs.csv'))
    temp_df_dem = df.loc[df['Party'] == 'Democratic Party']
    temp_df_rep = df.loc[df['Party'] == 'Republican Party']
    dem_list = list(set(temp_df_dem['UserID']))
    rep_list = list(set(temp_df_rep['UserID']))
    dem_list = random.sample(dem_list, len(rep_list))

    # Divide training/validation/test sets s.t. each has an equal number of dems and reps. 50/30/20 train/val/test split
    train_dems, test_dems = train_test_split(dem_list, test_size=int(.5*len(dem_list)))
    validation_dems, test_dems = train_test_split(test_dems, test_size=int(.4*len(test_dems)))
    train_reps, test_reps = train_test_split(rep_list, test_size=int(.5*len(rep_list)))
    validation_reps, test_reps = train_test_split(test_reps, test_size=int(.4*len(test_reps)))

    train_users = train_dems + train_reps
    validation_users = validation_dems + validation_reps
    test_users = test_dems + test_reps
    random.shuffle(train_users)
    random.shuffle(validation_users)
    random.shuffle(test_users)

    # call the function to construct training/val/test sets
    # on our lists of User IDs corresponding to each.
    df_train = split_into_512_approx(train_users, 73, df, by_tweet=True)
    df_val = split_into_512_approx(validation_users, 73, df, by_tweet=True)
    df_test = split_into_512_approx(test_users, 73, df, by_tweet=True)
    df_train.to_csv(Path('see_running_data/no_chunk_2_16_train.csv'))
    df_val.to_csv(Path('see_running_data/no_chunk_2_16_val.csv'))
    df_test.to_csv(Path('see_running_data/no_chunk_2_16_test.csv'))

    # Drop User IDs, save the test set IDs for evaluation.
    df_train.drop('UserID', axis=1)
    df_val.drop('UserID', axis=1)
    user_ID_add_later = list(df_test['UserID'])
    df_test.drop('UserID', axis=1)

    train(model, df_train, df_val, LR, EPOCHS)
    filename = 'trained_model_no_chunk_2_13_23.joblib'
    # serialize and save model
    joblib.dump(model, filename)
    evaluate(model, df_test, user_ID_add_later, save_results=True)
    print(LR, EPOCHS)

if __name__ == "__main__":
    main()


import logging
from typing import Dict, List, Tuple, Union

import numpy
import torch

logger = logging.getLogger(__name__)

PRETRAINED_LONGFORMER_MODELS = {
    "longformer-base-4096": "allenai/longformer-base-4096",
    "longformer-large-4096": "allenai/longformer-large-4096",
    "longformer-random-tiny": "patrickvonplaten/longformer-random-tiny",  # A tiny model for debugging
}

class LongformerInputs:
    def __init__(self, input_ids, attention_mask, global_attention_mask):
        self.input_ids: torch.LongTensor = input_ids
        self.attention_mask: Union[torch.FloatTensor, torch.HalfTensor] = attention_mask
        self.global_attention_mask: Union[torch.FloatTensor, torch.HalfTensor] = global_attention_mask

    def to_list(self) -> List:
        return [v for v in [self.input_ids, self.attention_mask, self.global_attention_mask] if v is not None]

    def to_tuple(self) -> Tuple:
        return tuple(v for v in self.to_list())

    def get_ort_inputs(self) -> Dict:
        return {
            "input_ids": numpy.ascontiguousarray(self.input_ids.cpu().numpy()),
            "attention_mask": numpy.ascontiguousarray(self.attention_mask.cpu().numpy()),
            "global_attention_mask": numpy.ascontiguousarray(self.global_attention_mask.cpu().numpy()),
        }

class LongformerHelper:
    """A helper class for Longformer model conversion, inference and verification."""

    @staticmethod
    def get_dummy_inputs(
        batch_size: int,
        sequence_length: int,
        num_global_tokens: int,
        device: torch.device,
        vocab_size: int = 100,
    ) -> LongformerInputs:
        """Create random inputs for Longformer model.
        Returns torch tensors of input_ids, attention_mask and global_attention_mask tensors.
        """

        input_ids = torch.randint(
            low=0,
            high=vocab_size - 1,
            size=(batch_size, sequence_length),
            dtype=torch.long,
            device=device,
        )
        attention_mask = torch.ones(input_ids.shape, dtype=torch.long, device=device)
        global_attention_mask = torch.zeros(input_ids.shape, dtype=torch.long, device=device)
        global_token_index = list(range(num_global_tokens))
        global_attention_mask[:, global_token_index] = 1
        return LongformerInputs(input_ids, attention_mask, global_attention_mask)

    @staticmethod
    def get_output_shapes(batch_size: int, sequence_length: int, hidden_size: int) -> Dict[str, List[int]]:
        """Returns a dictionary with output name as key, and shape as value."""
        return {
            "last_state": [batch_size, sequence_length, hidden_size],
            "pooler": [batch_size, sequence_length],
        }


import random
import speech_recognition as sr
import features.constant as c
from features.speaker import Speaks as sp

# INITIALIZING THE VOICE RECOGNIZER
record = sr.Recognizer ()

#  DEFINING THE CLASS TO LISTEN THE VOICE AND RECOGNIZE THE ERROR SOUND
class Listen:

    def record_audio():
        with sr.Microphone() as source:
            audio = record.listen(source)

            try:
                voice_data = record.recognize_google(audio)
                print(f'Yukesh: {voice_data}')

            except sr.UnknownValueError:
                sp(random.choice(c.error_msg)).speak()
                return "None"

            except sr.RequestError:
                sp('Sorry,my speech service is down').speak()
            return voice_data


from env_setup import setup_test_paths

setup_test_paths()

from models import Tenant, ChromeOsDevice, Distributor, Domain
from agar.test import BaseTest
from integrations.directory_api.chrome_os_devices_api import ChromeOsDevicesApi
from workflow.refresh_device_by_mac_address import refresh_device_by_mac_address

class TestRefreshDeviceByMacAddress(BaseTest):
    SKYKIT_COM_CUSTOMER_ID = 'C04c2u0lg'
    ADMIN_ACCOUNT_TO_IMPERSONATE = 'administrator@skykit.com'
    TESTING_DEVICE_ID = 'bca3c43b-0e9f-43b6-9028-c90ad432b80f'
    ORG_UNIT_DEPLOYED = '/SKD Automated Test/SKD Automated Deployed'
    ORG_UNIT_DISTRIBUTOR = '/SKD Automated Test/SKD Automated Distributor'
    CHROME_DEVICE_DOMAIN = 'dev.agosto.com'
    DISTRIBUTOR_NAME = 'agosto'
    IMPERSONATION_EMAIL = 'skykit.api@dev.agosto.com'

    def setUp(self):
        super(TestRefreshDeviceByMacAddress, self).setUp()
        self.chrome_os_devices_api = ChromeOsDevicesApi(self.ADMIN_ACCOUNT_TO_IMPERSONATE)
        self.distributor = Distributor.create(name=self.DISTRIBUTOR_NAME,
                                              active=True)
        self.distributor_key = self.distributor.put()
        self.domain = Domain.create(name=self.CHROME_DEVICE_DOMAIN,
                                    distributor_key=self.distributor_key,
                                    impersonation_admin_email_address=self.IMPERSONATION_EMAIL,
                                    active=True)
        self.domain_key = self.domain.put()
        self.tenant = Tenant.create(name='Foobar, Inc',
                                    tenant_code='foobar_inc',
                                    admin_email='admin@foobar.com',
                                    content_server_url='https://skykit-contentmanager-int.appspot.com/content',
                                    content_manager_base_url='https://skykit-contentmanager-int.appspot.com',
                                    domain_key=self.domain_key,
                                    active=True)
        self.tenant_key = self.tenant.put()
        self.mac_address = 'd0df9a904b99'
        self.device = ChromeOsDevice.create_managed(tenant_key=self.tenant_key,
                                                    gcm_registration_id='8d70a8d78a6dfa6df76dfasd',
                                                    mac_address=self.mac_address)
        self.device_key = self.device.put()

    def test_refresh_device_by_mac_address(self):
        """ Tests the live connection to Admin SDK Directory API. """
        result = refresh_device_by_mac_address(
            device_urlsafe_key=self.device_key.urlsafe(),
            device_mac_address=self.mac_address,
            device_has_previous_directory_api_info=False)
        self.assertEqual(result.device_id, self.TESTING_DEVICE_ID)


from discord.ext.commands import *
from discord import *
from aiohttp import web
import random

from discord_backups import Backup

class BackupConv(Converter):
    async def convert(self, ctx, argument):
        backup =  await ctx.bot.database.save_backup(argument.lower())
        if backup is None:
            raise BadArgument("Cant find a backup with that id.")

        return backup

class WebServer:
    routes = web.RouteTableDef()
    def __init__(self):
        self.app = web.Application()

    async def authorization(self, request):
        return web.Response(text="Authorization")

    async def setup(self):
        self.app.add_routes([web.get('/', self.authorization)])

        runner = web.AppRunner(self.app)
        await runner.setup()
        site = web.TCPSite(runner, '0.0.0.0', 4321)
        await site.start()

class Backups:
    def __init__(self, bot):
        self.bot = bot
        self.webserver = WebServer()
        self.temp_backup = None

    async def on_ready(self):
        await self.webserver.setup()

    @group(invoke_without_command=True, aliases=["bu"])
    async def backup(self, ctx):
        """Create or load personal server backups"""
        await ctx.invoke(self.bot.get_command("help"), "backup")

    @backup.command(aliases=["c"])
    @guild_only()
    @has_permissions(administrator=True)
    @bot_has_permissions(administrator=True)
    #@cooldown(1, 1 * 60, BucketType.guild)
    async def create(self, ctx):
        "Create a backup of your server"
        status_message = await ctx.send("Creating backup, thais could take a while.")

        backup = await Backup.from_guild(self.bot, ctx.guild, ctx.author)
        self.temp_backup = backup

        try:
            await ctx.author.send()
        except Exception as e:
            await status_message.edit(content="I was **unable to send you the backup-id**. Please enable private messages on this server!")
            return

        await status_message.edit(content="Successfully **created backup**. Please **check your dm's** to see the backup-id.")

    @backup.command(aliases=["l"])
    @guild_only()
    @has_permissions(administrator=True)
    @bot_has_permissions(administrator=True)
    #@cooldown(1, 5 * 60, BucketType.guild)
    async def load(self, ctx, backup):
        await ctx.send("Loading")
        await self.temp_backup.load(ctx.guild)
        await ctx.send("Finished")

def setup(bot):
    bot.add_cog(Backups(bot))


import json
import logging
import os

import redis

from onyx.util.StructMessage import StructMessage

class StorageEntity:
    def __init__(self, db=0):
        self.host = os.getenv('ONYX_REDIS_HOST', 'localhost')
        self.port = int(os.getenv('ONYX_REDIS_PORT', '6379'))
        self.db = db

    @staticmethod
    def exists(identifier):
        logging.info(StructMessage(message='Checking existence of identifier', identifier=identifier))

        storage = StorageEntity()
        return redis.Redis(decode_responses=True, host=storage.host, port=storage.port, db=storage.db).exists(identifier) == 1

    @staticmethod
    def load(identifier):
        logging.info(StructMessage(message='Fetching qr content', identifier=identifier))

        storage = StorageEntity()
        return json.loads(redis.Redis(decode_responses=True, host=storage.host, port=storage.port, db=storage.db).get(identifier))

    @staticmethod
    def save(identifier, value):
        logging.info(StructMessage(message='Saving qr content', identifier=identifier, value=value))

        storage = StorageEntity()
        return redis.Redis(host=storage.host, port=storage.port, db=storage.db).set(identifier, json.dumps(value))

    @staticmethod
    def delete(identifier):
        logging.info(StructMessage(message='Fetching qr content', identifier=identifier))

        storage = StorageEntity()
        return redis.Redis(host=storage.host, port=storage.port, db=storage.db).delete(identifier)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue May 14 10:35:12 2019

@author: AZROUMAHLI Chaimae
"""

# Libraries
from glove import Glove
import numpy as np
import io
import os
#from dask import array as da

#For Glove: reading it from a model
def get_WE_Glove(model_file):
    A=os.getcwd()
    os.chdir("/home/ubuntu/NLP_and_Irace/Tuning_run/Instances")
    try:
        model=Glove.load(model_file)
        Glove_dict={}
        for key,val in model.dictionary.items():
            Glove_dict.update({key:model.word_vectors[val]})
    except:
        model=Glove.load("Glove_Joker")
        Glove_dict={}
        for key,val in model.dictionary.items():
            Glove_dict.update({key:model.word_vectors[val]})
    os.chdir(A)
    return Glove_dict

#For Word2vec: reading it directly from a npy dicrtionnary
def get_WE_Word2Vec(dictionnary_file):
    A=os.getcwd()
    os.chdir("/home/ubuntu/NLP_and_Irace/Tuning_run/Instances")
    W2Vec=np.load(dictionnary_file,allow_pickle=True).item()
    os.chdir(A)
    return W2Vec

#For Fast text
def get_WE_fasttext(fname):
    A=os.getcwd()
    os.chdir("/home/ubuntu/NLP_and_Irace/Tuning_run/Instances")
    fin = io.open(fname, 'r', encoding='utf-8-sig', newline='\n', errors='ignore')
    n, d = map(int, fin.readline().split())
    data = {}
    for line in fin:
        tokens = line.rstrip().split(' ')
        try:
            data[tokens[0]] = list(map(float,tokens[1:]))
        except:
            pass
    os.chdir(A)
    return data