import os, sys
import numpy
import math

try:
    from pyspatialite import dbapi2 as db
    import sqlite3
except:
    import sqlite3

try:
    from osgeo import ogr
except:
    import osr
try:
    from osgeo import osr
except:
    import ogr

import matplotlib.pyplot as plt
from matplotlib.pylab import *
from matplotlib.font_manager import FontProperties
import csv

def DamBreakHyrograph(VolumeInvaso,AltezzaDiga,LarghezzaSezioneRottura,NomeDiga):
    """
    Evaluation of the dambreak hydrograph:
    -------------------------------------
    VolumeInvaso            : Reservoir volume in millions of cubic meters
    AltezzaDiga             : height of the dam in meters
    LarghezzaSezioneRottura : Width of the breach in meters
    NomeDiga                : name of the dam
    """
    # form Mmc to mc
    VolumeInvaso=VolumeInvaso*10**6

    # reservoir volume curve : V=k*h^alfa
    # ====================================
    alfa=1.5
    kinvaso=VolumeInvaso/(AltezzaDiga**alfa)

    Vcurr=VolumeInvaso
    Vfin=VolumeInvaso*0.001

    # flow calculation for instant collapse
	# ====================================
	# gravity acceleration
    G=9.81

    Yc=4.0/9.0*AltezzaDiga

    Qmax=8.0/27.0*LarghezzaSezioneRottura*AltezzaDiga*(G*AltezzaDiga)**0.5

    Vmax=Qmax/(Yc*LarghezzaSezioneRottura)

    # Total theoretical energy height in the hypothesis of permanent motion
    Htot=Yc+Vmax**2/2./G
    # volume
    Vol_Htot=kinvaso*Htot**alfa

    dt=60.0
    Idrogramma=[]
    Time=[]
    T=0.0
    Time.append(T)
    Idrogramma.append(Qmax)

    while Vcurr>=Vol_Htot:
        dv=-Qmax*dt
        # update volume
        Vcurr+=dv
        if Vcurr<0.0:
            while Vcurr<0.0:
                dt=dt/2.0
                Vcurr+=-dv
                dv=-Qmax*dt
                Vcurr+=dv
        Idrogramma.append(Qmax)
        T+=dt
        Time.append(T)

    alfa_1=1.0/alfa
    # water depth
    hcurr=math.pow((Vcurr/kinvaso),alfa_1)

    hmin=0.5

    while hcurr>hmin:
        # critical depth for rectangular section
        Yc=2.0/3.0*hcurr
        # critical speed for rectangular section
        Uc=math.sqrt(G*Yc)
        # critical flow rate for rectangular section
        Qc=Uc*Yc*LarghezzaSezioneRottura
        # volume released in time unit
        dv=-Qc*dt
        # update the volume at the end of dt
        Vcurr+=dv
        if Vcurr<0.0:
            while Vcurr<0.0:
                dt=dt/2.0
                Vcurr+=-dv
                dv=-Qc*dt
                Vcurr+=dv
        Idrogramma.append(Qc)
        T+=dt
        Time.append(T)
        # update the water depth at the end of dt
        hcurr=math.pow((Vcurr/kinvaso),alfa_1)

    # calculate hydrogram volume
    Vol=0.0
    nn=len(Idrogramma)
    for i in range(1,nn):
        Vol+=(Time[i]-Time[i-1])*(Idrogramma[i]+Idrogramma[i-1])/2.0

    grafico=0
    if grafico>0:
        try:
            plt.subplot(1, 1, 1)

            plt.plot(Time,Idrogramma)

            plt.grid(True)
            txt='Dam: %s  - Dam-Break discharge' % NomeDiga
            plt.title(txt)
            plt.xlabel('Time (sec)')
            plt.ylabel('discharge (cm/s)')
            plt.legend()
            plt.show()
        except:
            pass

    return Idrogramma, Time

def ValoreInX(x,n,datix,datiy):

    nx=len(datix)
    ny=len(datiy)
    if nx==ny:
        if x>datix[0]:
            if x<datix[-1]:
                for i in range(1,nx):
                    if datix[i]>=x:
                        y=datiy[i-1]+(datiy[i]-datiy[i-1])*(x-datix[i-1])/(datix[i]-datix[i-1])
                        break
            else:
                y=datiy[-1]
        else:
            y=datiy[0]
    else:
        sys.exit()

    return y

def RunCinemat(mydb_path_user,DamID,PathFiles,grafico2):

    """
    Run Cinemat model for routing flood downstream
    """

    pythonver_log=sys.version
    ppp=pythonver_log.split(' ')
    py_ver=ppp[0][0:1]

    NotErr=bool('True')
    errMsg='OK'

    PathFiles=os.path.realpath(PathFiles)

    if not os.path.exists(PathFiles):
        errMsg = "Non ci sono dati per la diga num =%s \nEffettuare prima il calcolo delle sezioni a valle !" % (DamID)
        NotErr= bool()
        return NotErr, errMsg

    FileOutput=PathFiles+os.sep+'Q_H_max.csv'

    # =========================================
    # apertura del database sqlite
    # =========================================

    conn = sqlite3.connect(mydb_path_user, detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
   # import extention
    conn.enable_load_extension(True)
    conn.execute('SELECT load_extension("mod_spatialite")')

    # creating a Cursor
    cur = conn.cursor()

    NomeTabella='DAMS'

    sql='SELECT ResVolMcm,Height_m,BreachWidth,Name FROM %s WHERE DamID=%d' % (NomeTabella,DamID)
    cur.execute(sql)
    DatiDiga=cur.fetchone()

    if DatiDiga!=None:

        VolumeInvaso=float(DatiDiga[0])

        # height of the dam
        AltezzaDiga = float(DatiDiga[1])

        # Width of the breach
        LarghezzaSezioneRottura = float(DatiDiga[2])

        # Name of the dam
        NomeDiga=DatiDiga[3]

    else:
        errMsg = 'In the table= %s there is no data for the dam num =%s \nPerform data saving first !' % (NomeTabellaTest,DamID)
        NotErr= bool()
        return NotErr, errMsg

    # reading the data for the calculation
    NomeTabellaMatrice='MatriceAexp'

    sql='SELECT '
    sql+=' PixDist'
    sql+=', progr_lungo_fiume'
    sql+=', distanza_fiume'
    sql+=', distanza_linea_retta'
    sql+=', pend'
    sql+=', ka'
    sql+=', ma'
    sql+=', kq'
    sql+=', mq'
    sql+=', kcel'
    sql+=', mcel'
    sql+=' FROM %s' % NomeTabellaMatrice
    sql+=' WHERE DamID=%d' % (DamID)
    sql+=' ORDER BY PixDist;'
    cur.execute(sql)
    MatriceDati=cur.fetchall()

    ListaTratti=[]
    Progressive_fiume=[]
    Distanze_fiume=[]
    # coefficient matrix: ka;ma;kq;mq;kcel;mcel for every reach
    MatriceCoeff=[]
    for row in MatriceDati:
        ListaTratti.append(int(row[0]))
        Progressive_fiume.append(float(row[1]))
        Distanze_fiume.append(float(row[2]))
        MatriceCoeff.append(row[3:])

    # Calculation of the flood hydrograph
    # ................................

    Idrogramma, T = DamBreakHyrograph(VolumeInvaso,AltezzaDiga,LarghezzaSezioneRottura,NomeDiga)

    # the first input corresponds to the hydrograph at the initial section
    Qin=numpy.array(Idrogramma,dtype =numpy.float)
    maxQ=Qin.max()

    nn=len(MatriceCoeff)
    x=0.0

    numtime=len(Qin)

    # lists of times, distances and the envelope of the maximum flow along the river
    TQX=[]
    XQX=[]
    QX=[]
    # Water depth
    HX=[]
    # water width
    BX=[]
    # water velocity
    VX=[]

    TrattoX=[]
    PendX=[]
    TQX.append(T[0])
    XQX.append(0.0)
    Progr_fiume=[]
    Progr_fiume.append(0.0)

    QX.append(Qin[0])
    TrattoX.append(0)
    PendX.append(MatriceCoeff[0][1])

    # coeff. and exponent of the formula monomer of the flow Q=kq*h^mq
    kq=float(MatriceCoeff[0][4])
    mq=float(MatriceCoeff[0][5])
    invmq=1.0/mq
    h_0=math.pow(Qin[0]/kq,invmq)
    HX.append(h_0)

    # water width = ka*ma*h^(ma-1)
    # ----------------------------------------------------
    ka=float(MatriceCoeff[0][2])
    ma=float(MatriceCoeff[0][3])

    # water width calculation
    mb=ma-1.0
    b_0=ka*ma*math.pow(h_0,mb)
    BX.append(b_0)

    # velocity
    # ------------------
    A_0=ka*math.pow(h_0,ma)
    if A_0>0:
        V_0=Qin[0]/A_0
    else:
        V_0=0.0

    VX.append(V_0)

    # settings for the chart
    #============================
    fontP = FontProperties()
    fontP.set_size('small')

    Progr_fiume=[]
    Progr_fiume.append(0.0)

    # cycle along the river reaches
    # -------------------------------
    for i in range(nn):

        # abscissa and instants of the initial point of the previous reach
        xini=XQX[-1]
        xini_fiume=Progr_fiume[-1]

        tini=TQX[-1]
        # abscissa from the beginning of reach
        x=0.0
        xfronte=0.0

        # kx and mx coefficients of geometric and hydraulic reach parameters
        tratto=ListaTratti[i]

        # distance in a straight line: used for calculating the propagation
        distanza=float(MatriceCoeff[i][0])

        # reading distance on the river
        Progr_curr=Progressive_fiume[i]
        Parz_curr=Distanze_fiume[i]
        # relationship between the distance along the river axis and the distance in a straight line
        Amplificazione=Parz_curr/distanza

        # slope
        pend=float(MatriceCoeff[i][1])
        # coeff. and exponent of the area monomular formula
        ka=float(MatriceCoeff[i][2])
        ma=float(MatriceCoeff[i][3])
        # coeff. and exponent of the monomer flow rate formula Q=kq*h^mq
        kq=float(MatriceCoeff[i][4])
        mq=float(MatriceCoeff[i][5])
        # coeff. and exponent of the monomy formula of celerity
        kcel=float(MatriceCoeff[i][6])
        mcel=float(MatriceCoeff[i][7])

        # parameters for calculating the water width B=dA/dh
        # ----------------------------------------------------------------
        # B=ka*ma*h^(m-1) = kb*h^mb
        kb=ka*ma
        mb=ma-1.0

        # calculation of the  Dt
        # ......................
        t=T[0]
        tau=T[0]
        # calculate the corresponding water depth
        invmq=1.0/mq

        for k in range(1,numtime):

            dtau=float(T[k]-T[k-1])

            Qtau=Qin[k-1]
            Qtau1=Qin[k]

            h_tau=math.pow(Qtau/kq,invmq)
            h_tau1=math.pow(Qtau1/kq,invmq)
            #  dQ/dA for tau cost = celerity
            Qprimo=kcel*math.pow(h_tau,mcel)
            # dQ/dA^2 for tau cost = derivative of celerity
            mcel2=mcel-1.0
            Qseconda=kcel*mcel*math.pow(h_tau,mcel2)
            # computation A(tau)
            Atau=ka*math.pow(h_tau,ma)
            Atau1=ka*math.pow(h_tau1,ma)
            # computation dA/dtau
            dA=(Atau1-Atau)/dtau
            # Vfronte calculation that on dry riverbed (ie Qo = 0) is Vfronte = Qtau / Atau
            Vfronte=Qtau/Atau
            # then dt calculation
            numerat=Qseconda*dA*x/Qprimo-Qprimo
            denom=-Qprimo+Vfronte
            dt=numerat/denom*dtau
            # dx calculation
            dx=dt*Vfronte

            # check the distance traveled
            if (x+dx)>distanza:
                # calculates the times to reach the end of the section
                dx1=distanza-x
                dt1=dx1/Vfronte
                dtau1=denom/numerat*dt1
                t1=t+dt1
                tau1=tau+dtau1
                # flow at instant tau1
                Q1=ValoreInX(tau1,numtime,T,Qin)
                h_1=math.pow(Q1/kq,invmq)
                # calculating the water width
                b_1=kb*math.pow(h_1,mb)

                # calculating celerity
                c1=kcel*math.pow(h_1,mcel)

                # calculates the hydrograph leaving the reach
                Tout=[]
                Qout=[]
                Tout.append(t1)
                Qout.append(Q1)

                # inserts a new point in the TQX curve
                TQX.append(t1)
                x1=xini+distanza
                XQX.append(x1)

                # progressive distance upgrade along the river axis
                x1_fiume=xini_fiume+distanza*Amplificazione
                Progr_fiume.append(x1_fiume)

                # save the results in the respective lists
                QX.append(Q1)
                HX.append(h_1)
                BX.append(b_1)
                A_1=ka*math.pow(h_1,ma)
                if A_1>0:
                    V_1=Q1/A_1
                else:
                    V_1=0.0
                VX.append(V_1)

                TrattoX.append(tratto)
                PendX.append(pend)

                # look for the next point in tau1 in the hydrograph
                for kk in range(1,numtime):
                    if T[kk]>tau1:
                        k1=kk
                        break
                for kk in range(k1,numtime):
                    dttau=T[kk]-tau1
                    tau1=tau1+dttau
                    Q2=ValoreInX(tau1,numtime,T,Qin)
                    h_2=math.pow(Q2/kq,invmq)
                    # calculating celerity
                    c2=kcel*math.pow(h_2,mcel)
                    # time difference of the two points of the hydrograph at the end of the section
                    dtt=(1.0/c2-1.0/c1)*dx1+dttau

                    t1=t1+dtt
                    Q1=Q2*1.0
                    c1=c2*1.0
                    Tout.append(t1)
                    Qout.append(Q1)
                break

            else:
                # case in which the end of reach was not reached
                x=x+dx
                tau=tau+dtau
                t=t+dt
                # inserts a new point in the TQX curve
                TQX.append(t)
                x1=xini+x
                XQX.append(x1)
                # progressive distance upgrade along the river axis
                x1_fiume=xini_fiume+x*Amplificazione
                Progr_fiume.append(x1_fiume)

                Q1=ValoreInX(tau,numtime,T,Qin)
                h_1=math.pow(Q1/kq,invmq)
                # water width calculation
                b_1=kb*math.pow(h_1,mb)

                # save results
                QX.append(Q1)
                HX.append(h_1)
                BX.append(b_1)
                A_1=ka*math.pow(h_1,ma)
                if A_1>0:
                    V_1=Q1/A_1
                else:
                    V_1=0.0
                VX.append(V_1)

                TrattoX.append(tratto)
                PendX.append(pend)

                # -----------------------------------
                # check if the hydrograph has ended
                # -----------------------------------
                if k> (numtime-2):

                    # calculates the times to reach the end of reach
                    dx1=distanza-x
                    dt1=dx1/Vfronte
                    dtau1=denom/numerat*dt1
                    t1=t+dt1
                    tau1=tau+dtau1
                    # flow at instant tau1
                    Q1=ValoreInX(tau1,numtime,T,Qin)
                    h_1=math.pow(Q1/kq,invmq)
                    # water width calculation
                    b_1=kb*math.pow(h_1,mb)

                    # calculate the celerity
                    c1=kcel*math.pow(h_1,mcel)

                    # calculates the hydrograph leaving the reach
                    Tout=[]
                    Qout=[]
                    Tout.append(t1)
                    Qout.append(Q1)

                    # inserts a new point in the TQX curve
                    TQX.append(t1)
                    x1=xini+distanza
                    XQX.append(x1)

                    # progressive distance upgrade along the river axis
                    x1_fiume=xini_fiume+distanza*Amplificazione
                    Progr_fiume.append(x1_fiume)

                    # save the results in the respective lists
                    QX.append(Q1)
                    HX.append(h_1)
                    BX.append(b_1)
                    A_1=ka*math.pow(h_1,ma)
                    if A_1>0:
                        V_1=Q1/A_1
                    else:
                        V_1=0.0
                    VX.append(V_1)

                    TrattoX.append(tratto)
                    PendX.append(pend)

                    # extends the length of the hydrograph
                    Tout.append(t1+1000.0)
                    Qout.append(Q1)

        grafico1=0
        if grafico1>0:
            # praph
            fig1 = figure()

            ax1 = fig1.add_subplot(111)
            ax1.plot(T,Qin,'-')
            ax1.plot(Tout,Qout,'-')
            nomi=[]
            nomi.append('Inflow')
            nomi.append('Outflow')

            PosizoneLegenda = 'best'
            leg = ax1.legend((nomi),
                       PosizoneLegenda, shadow=True,prop = fontP)
            ax1.grid(True)
            ax1.set_ylabel('Q (mc/s)')
            ax1.set_xlabel('Time (hrs)')
            Titolo='CINE method applied to reach %d distance pixel n: %d  x=%.3f' % (i+1,tratto,x)

            suptitle(Titolo, fontsize=16,fontstyle='italic')

            show()

        #assigns the outgoing idrograph to input to the next reach
        Qin=numpy.array(Qout,dtype =numpy.float)
        T=numpy.array(Tout,dtype =numpy.float)
        numtime=len(T)

    if grafico2>0:

        # graph QX
        fig1 = figure()

        ax1 = fig1.add_subplot(111)
        ax1.plot(Progr_fiume,QX,'o-')
        nomi=[]
        nomi.append('QX')

        ax2 = ax1.twinx()
        ax2.plot(Progr_fiume,HX,'-r')

        nomi2=[]
        nomi2.append('HX')

        PosizoneLegenda = 'upper left'
        leg = ax1.legend((nomi), loc=PosizoneLegenda,
         shadow=True,prop = fontP)
        PosizoneLegenda2 = 'upper right'
        leg2 = ax2.legend((nomi2), loc=PosizoneLegenda2,
         shadow=True,prop = fontP)

        ax1.grid(True)
        ax1.set_ylabel('Qmax (mc/s)')
        ax2.set_ylabel('Hmax (m)')
        ax1.set_xlabel('x (m)')
        Titolo='Peak flow rate reduction with distance'

        suptitle(Titolo, fontsize=16,fontstyle='italic')

        show()

    # save the results
    # --------------------------

    NomeTabella='Q_H_max'

    # cancel any previous data
    # ---------------------------------
    sql='DELETE FROM %s WHERE DamID=%d' % (NomeTabella,DamID)
    cur.execute(sql)
    conn.commit()

    fout=open(FileOutput,'w')
    txt='PixDist;Progr_fiume;Progr_retta;pend;Qmax;Hmax;Bmax;Vmax;Time\n'
    fout.write(txt)
    n2=len(QX)
    for i in range(n2):
        # salva i risultati
        txt='%d;%.2f;%.2f;%s;%.2f;%.2f;%.2f;%.2f;%.2f\n' %(TrattoX[i],Progr_fiume[i],XQX[i],PendX[i],QX[i],HX[i],BX[i],VX[i],TQX[i])
        fout.write(txt)

        sql='INSERT INTO %s (DamID' % (NomeTabella)
        sql_value=') VALUES (%d' % DamID
        sql_value+=',%d,%.2f,%.2f,%s,%.2f,%.2f,%.2f,%.2f,%.2f' %(TrattoX[i],Progr_fiume[i],XQX[i],PendX[i],QX[i],HX[i],BX[i],VX[i],TQX[i])
        sql+=', PixDist'
        sql+=', Progr_fiume'
        sql+=', Progr_retta'
        sql+=', pend'
        sql+=', Qmax'
        sql+=', Hmax'
        sql+=', Bmax'
        sql+=', Vmax'
        sql+=', Time'
        sql+='%s' % sql_value
        sql+=');'
        cur.execute(sql)

    fout.close()

    conn.commit()

    # Close communication with the database
    cur.close()
    conn.close()

    return NotErr, errMsg

if __name__ == '__main__':

    mydb_path_user='..'+ os.sep+'db'+os.sep+'USER_GeoDB.sqlite'

    # San Giuliano
    DamID=449
    PathFiles='..'+ os.sep+ str(DamID)

    grafico2=1
    NotErr, errMsg= RunCinemat(mydb_path_user,DamID,PathFiles,grafico2)

    print(NotErr,errMsg)


import torch
import torch.nn as nn
from torch.autograd import Function

from ...utils import box_utils
from . import roipoint_pool3d_cuda

def points_in_boxes_gpu(points, boxes):
    """
    :param points: (B, N, 3)
    :param boxes: (B, M, 7), num_valid_boxes <= T
    :return box_idxs_of_pts: (B, N, M), default background = -1
    """
    assert boxes.shape[0] == points.shape[0]
    assert boxes.shape[2] == 7 and points.shape[2] == 3
    batch_size, num_points, _ = points.shape
    _, num_box, _ = boxes.shape

    pts_assign = points.new_zeros((batch_size, num_points, num_box), dtype=torch.int).fill_(-1)
    roipoint_pool3d_cuda.assign_points(points.contiguous(), boxes.contiguous(), pts_assign)

    return pts_assign

class RoIPointPool3d(nn.Module):
    def __init__(self, num_sampled_points=512, pool_extra_width=1.0):
        super().__init__()
        self.num_sampled_points = num_sampled_points
        self.pool_extra_width = pool_extra_width

    def forward(self, points, point_features, boxes3d):
        """
        Args:
            points: (B, N, 3)
            point_features: (B, N, C)
            boxes3d: (B, M, 7), [x, y, z, dx, dy, dz, heading]

        Returns:
            pooled_features: (B, M, 512, 3 + C)
            pooled_empty_flag: (B, M)
        """
        return RoIPointPool3dFunction.apply(
            points, point_features, boxes3d, self.pool_extra_width, self.num_sampled_points
        )

class RoIPointPool3dFunction(Function):
    @staticmethod
    def forward(ctx, points, point_features, boxes3d, pool_extra_width, num_sampled_points=512):
        """
        Args:
            ctx:
            points: (B, N, 3)
            point_features: (B, N, C)
            boxes3d: (B, num_boxes, 7), [x, y, z, dx, dy, dz, heading]
            pool_extra_width:
            num_sampled_points:

        Returns:
            pooled_features: (B, num_boxes, 512, 3 + C)
            pooled_empty_flag: (B, num_boxes)
        """
        assert points.shape.__len__() == 3 and points.shape[2] == 3
        batch_size, boxes_num, feature_len = points.shape[0], boxes3d.shape[1], point_features.shape[2]
        pooled_boxes3d = box_utils.enlarge_box3d(boxes3d.view(-1, 7), pool_extra_width).view(batch_size, -1, 7)

        pooled_features = point_features.new_zeros((batch_size, boxes_num, num_sampled_points, 3 + feature_len))
        pooled_empty_flag = point_features.new_zeros((batch_size, boxes_num)).int()

        roipoint_pool3d_cuda.forward(
            points.contiguous(), pooled_boxes3d.contiguous(),
            point_features.contiguous(), pooled_features, pooled_empty_flag
        )

        return pooled_features, pooled_empty_flag

    @staticmethod
    def backward(ctx, grad_out):
        raise NotImplementedError

def points_in_boxes_gpu(points, boxes):
    """
    :param points: (B, N, 3)
    :param boxes: (B, M, 7), num_valid_boxes <= T
    :return box_idxs_of_pts: (B, N, M), default background = -1
    """
    assert boxes.shape[0] == points.shape[0]
    assert boxes.shape[2] == 7 and points.shape[2] == 3
    batch_size, num_points, _ = points.shape
    _, num_box, _ = boxes.shape

    pts_assign = points.new_zeros((batch_size, num_points, num_box), dtype=torch.int).fill_(-1)
    roipoint_pool3d_cuda.assign_points(points.contiguous(), boxes.contiguous(), pts_assign)

    return pts_assign

if __name__ == '__main__':
    pass


from django.db import models

class NistControl(models.Model):
    objects = models.Manager()

    control_type = models.CharField(max_length=20)
    baseline = models.CharField(max_length=50, null=True)
    status = models.CharField(max_length=50, null=True)
    # this may just be the statement.description
    number = models.CharField(max_length=20, db_index=True)
    family = models.CharField(max_length=255)
    title = models.CharField(max_length=250, null=True)
    revision = models.IntegerField()
    privacy_impact = models.CharField(max_length=5, null=True, blank=True)
    priority = models.CharField(max_length=20, choices=(('P0', 'P0'), ('P1', 'P1'), ('P2', 'P2'), ('P3', 'P3')), null=True, blank=True)
    related = models.ManyToManyField('NistRelated', related_name='related_controls')
    parent_control = models.ForeignKey('self', related_name='enhancements', null=True, on_delete=models.SET_NULL)

    def __str__(self):
        return str(self.get_family()) + ': ' + str(self.get_number())

    def get_number(self):
        return self.number

    def get_family(self):
        return self.family

    def get_title(self):
        return self.title

class NistReference(models.Model):
    objects = models.Manager()

    href = models.TextField( null=True)
    text = models.TextField(null=True)
    short_name = models.TextField(null=True)
    control = models.ForeignKey(NistControl, related_name='references', on_delete=models.CASCADE)

class NistRelated(models.Model):
    objects = models.Manager()

    text = models.CharField(max_length=20)

class NistSupplementGuidanceRelatedItems(models.Model):
    objects = models.Manager()

    related = models.CharField(max_length=50)

class NistDiscussion(models.Model):
    objects = models.Manager()

    description = models.TextField()
    control = models.OneToOneField(NistControl, related_name='discussion', on_delete=models.CASCADE)

class NistSupplementalGuidance(models.Model):
    objects = models.Manager()

    description = models.TextField()
    control = models.OneToOneField(NistControl, related_name='supplemental_guidance', on_delete=models.CASCADE)
    related = models.ManyToManyField(NistSupplementGuidanceRelatedItems, related_name='related_items')

class NistStatement(models.Model):
    objects = models.Manager()

    number = models.CharField(max_length=25, null=True, db_index=True)
    description = models.TextField(null=True)
    statement = models \
        .ForeignKey('self', related_name='statements', null=True, on_delete=models.CASCADE)
    control = models \
        .OneToOneField(NistControl, related_name='statement', null=True, on_delete=models.CASCADE)
    parent_control = models \
        .ForeignKey(NistControl, related_name='nested_statements', null=True, on_delete=models.CASCADE)

    def __str__(self):
        return self.number

    def statement_control(self):
        if self.control:
            return self.control
        else:
            return self.parent_control

    def get_revision(self):
        if self.control:
            return self.control.revision
        else:
            return self.parent_control.revision

class ControlCorrelationIdentifier(models.Model):
    objects = models.Manager()

    name = models.TextField(null=True, unique=True)
    contributor = models.CharField(max_length=255, null=True, blank=True)
    status = models.CharField(max_length=255, null=True, blank=True)
    type = models.CharField(max_length=255, null=True, blank=True)
    parameter = models.CharField(max_length=255, null=True, blank=True)
    publishdate = models.DateField(null=True, blank=True)
    note = models.TextField(null=True, blank=True)
    definition = models.TextField(null=True, blank=True)
    location = models.TextField(null=True, blank=True)
    statements = models.ManyToManyField(NistStatement, related_name='ccis')
    implementation_guidance = models.TextField(null=True, blank=True)
    assessment_procedures = models.TextField(null=True, blank=True)
    controls = models.ManyToManyField(NistControl, related_name='ccis', )

    def get_number(self):
        return int(str(self.name).replace('CCI-', ''))


from typing import Callable, Protocol, Type, TypeVar

from antidote import implements, interface, world, inject

from antidom.viewdom import VDOM
from antidom.resource import Resource

@interface
class View(Protocol):

    def __vdom__(self) -> VDOM:
        ...

V = TypeVar('V', bound=Type[View])

def view(context: Type[Resource]) -> Callable[[V], V]:
    def decorate(cls: V) -> V:
        implements(View).when(qualified_by=context)(cls)
        return cls

    return decorate

@inject
def get_view(this_resource: Resource = inject.me()) -> View:
    """Find the correct view for the context."""
    context_type = this_resource.__class__
    views = world.get[View]  # type: ignore
    this_view = views.single(qualified_by=context_type)
    return this_view


from http import client
import socket

PORT    = 5050
HEADER  = 64 #Will be sent at first by the client indicating the length of the actual message
FORMAT  = 'utf-8'
DISCONNECT_MESSAGE  = "!DISCONNECT" #Client sends this to close the conn with server
SERVER  = socket.gethostbyname(socket.gethostname())
ADDR    = (SERVER, PORT)

client  = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)

def send(msg):
    message = msg.encode(FORMAT)
    msg_len = len(message)
    send_length = str(msg_len).encode(FORMAT)
    send_length += b' ' * (HEADER - len(send_length))
    client.send(send_length)
    client.send(message)
    print(client.recv(2048).decode(FORMAT))

send("Hello World!")
send(DISCONNECT_MESSAGE)


#!/usr/bin/env python
#
# This example demonstrates the creation of multiple actors and the
# manipulation of their properties and transformations. It is a
# derivative of Cone.py, see that example for more information.
#

import vtk

cone = vtk.vtkConeSource()
cone.SetHeight(50.0)
cone.SetRadius(10.0)
cone.SetResolution(10)

#  RTLreader
reader = vtk.vtkSTLReader()
reader.SetFileName("arm.stl")
rtlMapper = vtk.vtkPolyDataMapper()
rtlMapper.SetInputConnection(reader.GetOutputPort())

coneMapper = vtk.vtkPolyDataMapper()
coneMapper.SetInputConnection(cone.GetOutputPort())

coneActor = vtk.vtkActor()
coneActor.SetMapper(coneMapper)
coneActor.GetProperty().SetColor(0.2, 0.63, 0.79)
coneActor.GetProperty().SetDiffuse(0.7)
coneActor.GetProperty().SetSpecular(0.4)
coneActor.GetProperty().SetSpecularPower(20)

property = vtk.vtkProperty()
property.SetColor(0, 1, 0)
property.SetDiffuse(0.7)
property.SetSpecular(0.4)
property.SetSpecularPower(20)

coneActor2 = vtk.vtkActor()
coneActor2.SetMapper(rtlMapper)
coneActor2.SetProperty(property)
coneActor2.SetPosition(20, 17, -122)
coneActor2.SetOrigin(-20, -17, 122)

ren1 = vtk.vtkRenderer()
ren1.AddActor(coneActor)
ren1.AddActor(coneActor2)
ren1.SetBackground(0.1, 0.2, 0.4)

renWin = vtk.vtkRenderWindow()
renWin.AddRenderer(ren1)
renWin.SetSize(600, 600)

coneActor2.RotateX(-90)
coneActor2.RotateZ(90)

coneActor2.RotateX(30)
'''for i in range(0,360):
    time.sleep(0.1)
    coneActor2.RotateX(2)
    renWin.Render()
    #ren1.GetActiveCamera().Azimuth( 1 )'''


import numpy as np
from PIL import Image
import tensorflow as tf
from torchvision.transforms import transforms

def resize(frame, sz=(1280, 1280), backend="tf") -> np.ndarray:
    # return -> np.ndarray
    H, W = sz
    if backend == "tf":
        return tf.image.resize_with_pad(frame, H, W).numpy().astype(np.uint8)
    elif backend == "pt":
        img = Image.fromarray(frame)
        img = transforms.Compose(
            [transforms.Resize((256, 192)), transforms.ToTensor()]
        )(img)
        return img


# --------------------------------------------------------
# Deep Iterative Matching Network
# Licensed under The Apache-2.0 License [see LICENSE for details]
# Written by Gu Wang, Yi Li
# --------------------------------------------------------
from __future__ import print_function, division
import sys
import os

cur_dir = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(0, os.path.join(cur_dir, "../../external/mxnet/mxnet_v00_origin"))
"""
zoom image with zoom factor
"""

import cv2
import mxnet as mx
import numpy as np

class ZoomImageWithFactorOperator(mx.operator.CustomOp):
    def __init__(self, height, width, pixel_means, high_light_center):
        super(ZoomImageWithFactorOperator, self).__init__()
        self.height = height
        self.width = width
        self.pixel_means = pixel_means
        self.high_light_center = high_light_center
        self.center_map = None
        self.spot_radius = 5

    def forward(self, is_train, req, in_data, out_data, aux):
        ctx = in_data[0].context
        if type(self.pixel_means) == np.ndarray:
            self.pixel_means = mx.nd.array(self.pixel_means, ctx=ctx)
        batch_size = in_data[0].shape[0]
        if self.center_map is None:
            self.center_map = np.zeros([batch_size, 3, self.height, self.width])
            spot_start_x = int(np.floor(float(self.width) / 2 - self.spot_radius))
            spot_end_x = int(np.ceil(float(self.width) / 2 + self.spot_radius))
            spot_start_y = int(np.floor(float(self.height) / 2 - self.spot_radius))
            spot_end_y = int(np.ceil(float(self.height) / 2 + self.spot_radius))
            self.center_map[:, 0, spot_start_y:spot_end_y, spot_start_x:spot_end_x] = 255.0  # red center
            self.center_map = mx.ndarray.array(self.center_map, ctx=ctx)
        image_real_array = in_data[1] + self.pixel_means
        image_rendered_array = in_data[2] + self.pixel_means

        grid_array = mx.ndarray.zeros([batch_size, 2, self.height, self.width], ctx=ctx)
        zoom_factor = in_data[0].asnumpy()
        for batch_idx in range(batch_size):
            wx, wy, tx, ty = zoom_factor[batch_idx]
            affine_matrix = mx.ndarray.array([[wx, 0, tx], [0, wy, ty]], ctx=ctx).reshape((1, 6))
            a = mx.ndarray.GridGenerator(
                data=affine_matrix, transform_type="affine", target_shape=(self.height, self.width)
            )
            grid_array[batch_idx] = a[0]

        zoom_image_rendered_array = mx.nd.BilinearSampler(image_rendered_array, grid_array)
        zoom_image_real_array = mx.nd.BilinearSampler(image_real_array, grid_array)
        if self.high_light_center:
            zoom_image_rendered_array = mx.ndarray.maximum(zoom_image_rendered_array, self.center_map)
        zoom_image_real_array -= self.pixel_means
        zoom_image_rendered_array -= self.pixel_means

        self.assign(out_data[0], req[0], zoom_image_real_array)
        self.assign(out_data[1], req[1], zoom_image_rendered_array)

    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        self.assign(in_grad[0], req[0], 0)
        self.assign(in_grad[1], req[1], 0)
        self.assign(in_grad[2], req[2], 0)

@mx.operator.register("ZoomImageWithFactor")
class ZoomImageWithFactorProp(mx.operator.CustomOpProp):
    def __init__(self, width=640, height=480, pixel_means="[0 0 0]", high_light_center="False"):
        super(ZoomImageWithFactorProp, self).__init__(True)
        self.height = int(height)
        self.width = int(width)
        self.pixel_means = np.fromstring(pixel_means[1:-1], dtype=np.float32, sep=" ").reshape(3)
        self.pixel_means = self.pixel_means[::-1]
        self.pixel_means = self.pixel_means[np.newaxis, :, np.newaxis, np.newaxis]
        self.hight_light_center = high_light_center.lower() == "true"

    def list_arguments(self):
        input_list = ["zoom_factor", "image_observed", "image_rendered"]
        return input_list

    def list_outputs(self):
        output_list = ["zoom_image_observed", "zoom_image_rendered"]
        return output_list

    def infer_shape(self, in_shape):
        out_shape = [in_shape[1], in_shape[2]]

        return in_shape, out_shape, []

    def infer_type(self, in_type):
        dtype = in_type[0]
        input_type = [dtype, dtype, dtype]
        output_type = [dtype, dtype]
        return input_type, output_type, []

    def create_operator(self, ctx, shapes, dtypes):
        return ZoomImageWithFactorOperator(self.height, self.width, self.pixel_means, self.hight_light_center)

if __name__ == "__main__":
    # configs
    thresh = 1e-3
    step = 1e-4
    ctx = mx.gpu(0)
    K = np.array([[1066.778, 0, 312.9869], [0, 1067.487, 241.3109], [0, 0, 1]])
    height = 480
    width = 640
    b_zoom_se3 = False
    batch_size = 8
    img_idx = ["005_{:06}".format(x) for x in [1, 11, 21, 31, 41, 51, 61, 71]]
    sub_idx1 = ["0" for x in range(batch_size)]
    sub_idx2 = ["5" for x in range(batch_size)]
    pixel_means = 120

    # initialize layer
    image_real = mx.sym.Variable("image_real")
    image_rendered = mx.sym.Variable("image_rendered")
    zoom_factor = mx.sym.Variable("zoom_factor")
    proj2d = mx.sym.Custom(
        zoom_factor=zoom_factor,
        image_real=image_real,
        image_rendered=image_rendered,
        pixel_means=np.array([pixel_means, pixel_means, pixel_means]).flatten(),
        name="updater",
        op_type="ZoomImageWithFactor",
        height=height,
        width=width,
        high_light_center=True,
    )
    v_zoom_factor = np.ones((batch_size, 4)) * 0.8
    v_zoom_factor[:, 1] = v_zoom_factor[:, 0]
    v_zoom_factor[:, 2:] = 0.0
    v_image_real = np.zeros((batch_size, 3, height, width), dtype=np.float32)
    v_image_rendered = np.zeros((batch_size, 3, height, width), dtype=np.float32)
    for idx in range(batch_size):
        v_image_rendered[idx] = cv2.imread(
            os.path.join(
                cur_dir, "../../data/render_v5/data/rendered/0003/{}_{}-color.png".format(img_idx[idx], sub_idx1[idx])
            ),
            cv2.IMREAD_COLOR,
        ).transpose([2, 0, 1])
        v_image_real[idx] = cv2.imread(
            os.path.join(
                cur_dir, "../../data/render_v5/data/rendered/0003/{}_{}-color.png".format(img_idx[idx], sub_idx2[idx])
            ),
            cv2.IMREAD_COLOR,
        ).transpose([2, 0, 1])

    exe1 = proj2d.simple_bind(
        ctx=ctx, zoom_factor=v_zoom_factor.shape, image_real=v_image_real.shape, image_rendered=v_image_rendered.shape
    )

    # forward
    exe1.arg_dict["zoom_factor"][:] = mx.nd.array(v_zoom_factor, ctx=ctx)
    exe1.arg_dict["image_real"][:] = mx.nd.array(v_image_real - pixel_means, ctx=ctx)
    exe1.arg_dict["image_rendered"][:] = mx.nd.array(v_image_rendered - pixel_means, ctx=ctx)
    import time
    import matplotlib.pyplot as plt

    t = time.time()
    exe1.forward(is_train=True)
    zoom_image_real = exe1.outputs[0].asnumpy().astype(np.uint8)
    zoom_image_rendered = exe1.outputs[1].asnumpy().astype(np.uint8)
    print("using time: {:.2f}s".format(time.time() - t))

    for batch_idx in range(batch_size):
        im_real = v_image_real[batch_idx].transpose([1, 2, 0]).astype(np.uint8)
        im_rendered = v_image_rendered[batch_idx].transpose([1, 2, 0]).astype(np.uint8)
        z_im_real = zoom_image_real[batch_idx].transpose([1, 2, 0]) + pixel_means
        z_im_rendered = zoom_image_rendered[batch_idx].transpose([1, 2, 0]) + pixel_means
        fig = plt.figure()
        tmp = fig.add_subplot(2, 2, 1)
        tmp.set_title("image_real")
        plt.axis("off")
        plt.imshow(im_real)

        tmp = fig.add_subplot(2, 2, 2)
        tmp.set_title("image_imagine")
        plt.axis("off")
        plt.imshow(im_rendered)

        tmp = fig.add_subplot(2, 2, 3)
        tmp.set_title("image_real after zoom")
        plt.axis("off")
        plt.imshow(z_im_real)

        tmp = fig.add_subplot(2, 2, 4)
        tmp.set_title("image_imagine after zoom")
        plt.axis("off")
        plt.imshow(z_im_rendered)
        plt.show()


from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
import numpy

extensions = [
            Extension( "rambutan.*",
                       [ "rambutan/*.pyx" ],
                       include_dirs=[numpy.get_include()] )
    ]

extensions = cythonize(extensions)

setup(
    name='rambutan',
    version='0.4.0',
    author='Jacob Schreiber',
    author_email='jmschr@cs.washington.edu',
    packages = ['rambutan'],
    license='LICENSE.txt',
    description='Rambutan is a package for the analysis of the 3D structure of the genome.',
    ext_modules=extensions,
    install_requires=[
        "cython >= 0.22.1",
        "numpy >= 1.8.0",
	    "pandas >= 0.20.2",
        "joblib >= 0.9.0b4",
        "mxnet >= 0.5a3"
    ],
)


import os
from enum import Enum
from flask import Flask, request, jsonify

from country import Country

app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False  # 日本語文字化け対策
app.config["JSON_SORT_KEYS"] = False  # ソートをそのまま

class Status(Enum):
    OK = 'OK'
    NG = 'NG'

@app.route('/hello', methods=['GET'])
def hello():
    return "hello_world"

@app.route('/hellojson', methods=['GET'])
def hello_json():
    data = [
        {"name": "山田"},
        {"age": 30}
    ]
    return jsonify({
        'status': Status.OK,
        'data': data
    })

@app.route('/testnews', methods=['GET'])
def test_news():
    data = [
        {'title': "最低賃金引き上げ額、過去最大の31円 平均961円に 中央審決定 - 毎日新聞 - 毎日新聞"},
        {'url': 'https://mainichi.jp/articles/20220801/k00/00m/020/097000c'},
        {'urlToImage': 'https://cdn.mainichi.jp/vol1/2022/06/28/20220628k0000m040241000p/0c10.jpg?1'},
        {'country': Country.JP}
    ]
    return jsonify({
        'status': Status.OK,
        'data': data
    })

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=int(os.getenv("PORT")))


from textblob import TextBlob

def senti(sentence):
    res = TextBlob(sentence).sentiment
    sentiment = ''
    if res.polarity <= 0:
        sentiment = 'negative'
    else:
        sentiment = 'positive'
    return sentiment

def sentimentCF(listTwts):
    for i, twt in enumerate(listTwts):
        senti(twt)
        listTwts[i] = listTwts[i] + ' ' + senti(twt)
    return listTwts

#Diff Method - Not using
"""import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
res = sid.polarity_scores("I am not happy")
print (res)

res = sid.polarity_scores("I am so so happy")
print (res)
"""


"""An extremely simple JSON-RPC 1.0 client.

This JSON-RPC client implements just enough functionality to interact with the
JSON API of the Deluge bittorrent client over HTTP, including using cookies for
session identification.  It is probably missing important aspects that would
make it fully compliant with all JSON-RPC 1.0 servers."""

import json
import cookielib

import requests

class ServerProxy(object):
    """JSON-RPC server proxy for proxying calls to the interface at *uri*."""
    def __init__(self, uri):
        self._uri = uri
        self._cookies = cookielib.CookieJar()
        self._req_id = 1

    def __getattr__(self, name):
        """Attributes of the :class:`ServerProxy` correspond to methods of the
        JSON-RPC server."""
        return _Method(self, name)

    def _next_id(self):
        """Get a session-unique request identifier."""
        tmp = self._req_id
        self._req_id += 1
        return tmp

    def _request(self, method, params):
        """Proxy a method call to the JSON-RPC server, returning the result."""
        # Build the request object
        req_id = self._next_id()
        request = {'id': req_id,
                   'method': method,
                   'params': params}
        # Get the response
        response = requests.post(self._uri,
                                 json.dumps(request),
                                 cookies=self._cookies)
        # Raise exception if there was an error at the HTTP layer
        response.raise_for_status()
        # Decode response
        content = json.loads(response.content)
        # Raise exception if the response ID is wrong
        if content['id'] != req_id:
            raise ProtocolError('Request and response ID do not match')
        # Raise exception if there was an error from the JSON-RPC server
        if content['error']:
            raise ProtocolError(content['error'])
        # Return response data
        return content['result']

class ProtocolError(Exception):
    """A JSON-RPC protocol error."""
    pass

class _Method(object):
    """Magic class for :class:`ServerProxy` methods, allowing calls like
    ``server.foo.bar()``."""
    def __init__(self, proxy, name):
        self._proxy = proxy
        self._name = name

    def __getattr__(self, name):
        """A call to ``s.foo.bar`` makes a call to the ``foo.bar`` method on
        the JSON-RPC server."""
        return _Method(self._proxy, '%s.%s' % (self._name, name))

    def __call__(self, *args, **kwargs):
        """When the method is called, make a request to the server."""
        if kwargs:
            raise Exception('Keyword arguments not supported')
        return self._proxy._request(self._name, args)


from django.core.management.base import BaseCommand
from input_data.models import County, State
from input_data.management.commands.utils import get_county_fips
import csv

"""
Takes a csv file of cities and populates the County model
Example path: /mnt/c/Users/nakaj/gdrive/REI/land/locations/uscounties.csv
"""

class Command(BaseCommand):
    help = "Your custom command description here"

    def add_arguments(self, parser):
        parser.add_argument("file_path", type=str, help="City file path")
        # parser.add_argument('--option1', action='store_true', help='Option 1 description')

    def handle(self, *args, **kwargs):
        path = kwargs["file_path"]
        with open(path) as f:
            reader = csv.DictReader(f)
            for i, row in enumerate(reader):
                data = {
                    "county": row["county"],
                    "population": row["population"],
                    "latitude": row["lat"],
                    "longitude": row["lng"],
                }
                data["county_fips"] = get_county_fips(row["county_fips"])
                try:
                    state = State.objects.get(state_id=row["state_id"])
                    data["state"] = state
                    County.objects.get_or_create(**data)
                    if i % 100 == 0:
                        print(i)
                except State.DoesNotExist:
                    pass

        self.stdout.write(self.style.SUCCESS("Script executed successfully"))


# https://leetcode.com/problems/group-anagrams/

from typing import List

class Solution:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        def reconstruct_str(_str):
            d = {}
            for c in _str:
                d.setdefault(c, 0)
                d[c] += 1
            res = ''
            for c in sorted(list(d.keys())):
                res += c + str(d[c])
            return res

        d_res = {}
        for s in strs:
            new_s = reconstruct_str(s)
            d_res.setdefault(new_s, [])
            d_res[new_s].append(s)

        return [v for _, v in d_res.items()]

if __name__ == '__main__':
    strs = ["a"]
    print(Solution().groupAnagrams(strs))


import sys, time
from modules import twitr, imgs

if len(sys.argv) < 5:
    exit("Usage: python3 facemaker.py <username> <picture> <x> <y>")

uname = sys.argv[1]

api = twitr.get_api_context()

im = imgs.open_image(sys.argv[2])
center = (int(sys.argv[3]),int(sys.argv[4]))

overlaid = False
old_unhinged=-1
old_likes=-1
reply_ids = set()

def calc_unhinged(tweet):
    return(len(
        [c for c in last_tweet.text if
         c.isupper() or c == '!']) /
           len(last_tweet.text) if len(last_tweet.text) > 0 else 0)

def process_img(img, replies, unhinged, likes):
    im = imgs.crop_img(img, unhinged, center)
    im.implode(amount= - (likes / 15))
    for rep in replies:
        if rep.id not in reply_ids:
            print(f"DAN NO DETECTED FROM {rep.user.screen_name}")
            im = imgs.overlay_pfp(api, rep.user.screen_name, im)
            reply_ids.add(rep.id)
    return(im)

def update_check(replies, unhinged, likes):
    return(unhinged != old_unhinged or
           likes != old_likes or
           {k.id for k in replies} != reply_ids)

while True:
    last_tweet = twitr.get_last_tweet(api, uname)
    unhinged_rating = calc_unhinged(last_tweet)
    kword_comments = twitr.search_tweet_comments(api, last_tweet, r'(dan.*((no)|(yes)))|hello')

    if update_check(kword_comments, unhinged_rating, last_tweet.favorite_count):
        if {k.id for k in kword_comments} != reply_ids:
            print("resetting overlays...")
            reply_ids = set()

        im = imgs.open_image(sys.argv[2])
        im = process_img(im,
                    kword_comments,
                    unhinged_rating,
                    last_tweet.favorite_count)

        print(f'Found tweet: {last_tweet.text}')
        if unhinged_rating != old_unhinged:
            print(f"New unhinged rating achieved: {unhinged_rating}")
        if last_tweet.favorite_count != old_likes:
            print(f"New like count achieved: {last_tweet.favorite_count}")

        old_likes = last_tweet.favorite_count
        old_unhinged = unhinged_rating

        twitr.update_pfp(api, im)

    time.sleep(5)


import configparser
import json
import logging
import os

config = configparser.ConfigParser()

CUR_DIR = os.path.abspath(os.path.dirname(__file__))
RUNNING_CONF = os.path.join(CUR_DIR, 'config/config.json')
DEFAULT_CONF = os.path.join(CUR_DIR, 'config/config_default.json')

def load():
    try:
        with open(RUNNING_CONF, 'r') as f:
            return json.load(f)
    except (FileNotFoundError, json.JSONDecodeError):
        logging.warning('empty running config, using default')
        with open(DEFAULT_CONF, 'r') as default_conf:
            with open(RUNNING_CONF, 'w') as running_conf:
                running_conf.write(default_conf.read())

def store(conf):
    with open(RUNNING_CONF, 'w') as f:
        f.write(json.dumps(conf, indent=4, sort_keys=True))


import argparse
import os
import time
import numpy as np
from tqdm import tqdm
from glob import glob
from collections import defaultdict

import torch
import torch.utils.data as data

from network.upsampler import Net
from model import Model
from network import operations
from utils import pc_utils, pytorch_utils
from misc import logger
from data import H5Dataset

parser = argparse.ArgumentParser()
parser.add_argument('--phase', default='test',
                    help='train or test [default: train]')
parser.add_argument('--gpu', type=int, default=0,
                    help='GPU to use [default: GPU 0]')
parser.add_argument('--id', default='demo',
                    help="experiment name, prepended to log_dir")
parser.add_argument('--log_dir', default='./model',
                    help='Log dir [default: log]')
parser.add_argument('--model', default='model_microscope', help='model name')
parser.add_argument('--root_dir', default='../',
                    help='project root, data and h5_data diretories')
parser.add_argument('--result_dir', help='result directory')
parser.add_argument('--ckpt', help='model to restore from')
parser.add_argument('--num_point', type=int,
                    help='Point Number [1024/2048] [default: 1024]')
parser.add_argument('--num_shape_point', type=int,
                    help="Number of points per shape")
parser.add_argument('--up_ratio', type=int, default=16,
                    help='Upsampling Ratio [default: 2]')
parser.add_argument('--max_epoch', type=int, default=160,
                    help='Epoch to run [default: 500]')
parser.add_argument('--batch_size', type=int, default=16,
                    help='Batch Size during training')
parser.add_argument('--h5_data', help='h5 file for training')
parser.add_argument('--record_data', help='record file for training')
parser.add_argument('--test_data', help='test data path')
parser.add_argument('--lr_init', type=float, default=0.0005)
parser.add_argument('--restore_epoch', type=int)
parser.add_argument('--stage_steps', type=int, default=15000,
                    help="number of updates per curriculums stage")
parser.add_argument('--step_ratio', type=int, default=2,
                    help="upscale ratio per step")
parser.add_argument('--patch_num_ratio', type=float, default=3)
parser.add_argument('--jitter', action="store_true",
                    help="jitter augmentation")
parser.add_argument('--jitter_sigma', type=float,
                    default=0.0025, help="jitter augmentation")
parser.add_argument('--jitter_max', type=float,
                    default=0.005, help="jitter augmentation")
parser.add_argument('--drop_out', type=float, default=1.0,
                    help="drop_out ratio. default 1.0 (no drop out) ")
parser.add_argument('--knn', type=int, default=32,
                    help="neighbood size for edge conv")
parser.add_argument('--dense_n', type=int, default=3,
                    help="number of dense layers")
parser.add_argument('--block_n', type=int, default=3,
                    help="number of dense blocks")
parser.add_argument('--fm_knn', type=int, default=5,
                    help="number of neighboring points for feature matching")

parser.add_argument('--growth_rate', type=int, default=12,
                    help='dense block growth rate')
parser.add_argument('--cd_threshold', default=2.0,
                    type=float, help="threshold for cd")
parser.add_argument('--fidelity_weight', default=50.0,
                    type=float, help="chamfer loss weight")

FLAGS = parser.parse_args()
PHASE = FLAGS.phase
DEVICE = torch.device('cuda', FLAGS.gpu)
ROOT_DIR = FLAGS.root_dir
MODEL_DIR = os.path.join(FLAGS.log_dir, FLAGS.id)
CKPT = FLAGS.ckpt

NUM_SHAPE_POINT = FLAGS.num_shape_point
NUM_POINT = FLAGS.num_point
assert(NUM_SHAPE_POINT is not None or NUM_POINT is not None)
NUM_POINT = NUM_POINT or int(NUM_SHAPE_POINT * FLAGS.drop_out)

BATCH_SIZE = FLAGS.batch_size
MAX_EPOCH = FLAGS.max_epoch
LR_INIT = FLAGS.lr_init
JITTER = FLAGS.jitter
JITTER_MAX = FLAGS.jitter_max
JITTER_SIGMA = FLAGS.jitter_sigma
STAGE_STEPS = FLAGS.stage_steps

STEP_RATIO = FLAGS.step_ratio
RESTORE_EPOCH = FLAGS.restore_epoch
FM_KNN = FLAGS.fm_knn
KNN = FLAGS.knn
GROWTH_RATE = FLAGS.growth_rate
DENSE_N = FLAGS.dense_n
CD_THRESHOLD = FLAGS.cd_threshold

UP_RATIO = FLAGS.up_ratio
TRAIN_H5 = FLAGS.h5_data
TRAIN_RECORD = FLAGS.record_data

TEST_DATA = FLAGS.test_data
PATCH_NUM_RATIO = FLAGS.patch_num_ratio

# build model
net = Net(max_up_ratio=UP_RATIO, step_ratio=STEP_RATIO,
          knn=KNN, growth_rate=GROWTH_RATE, dense_n=DENSE_N, fm_knn=FM_KNN)

def get_stage_progress(step):
    """
    return the stage (an integer from 0) and progress (float 0~1)
    """
    stage = (step + STAGE_STEPS) // (2 * STAGE_STEPS)
    progress = (step + STAGE_STEPS) / (2 * STAGE_STEPS) - stage
    return stage, progress

def train():
    net.to(DEVICE)
    net.train()
    model = Model(net, "train", FLAGS)
    # data loader
    if TRAIN_H5 is not None:
        from data import H5Dataset
        dataset = H5Dataset(
            h5_path=TRAIN_H5,
            num_shape_point=NUM_SHAPE_POINT, num_patch_point=NUM_POINT,
            batch_size=BATCH_SIZE, up_ratio=UP_RATIO, step_ratio=STEP_RATIO)
        dataloader = data.DataLoader(
            dataset, batch_size=1, pin_memory=True, num_workers=4)

    start_epoch = model.step // len(dataloader)
    # whenever progress is changed, we need to update:
    # 1. chamferloss threshold
    # 2. dataset.combined
    # 3. dataset.curr_threshold
    stage, progress = get_stage_progress(model.step)
    start_ratio = STEP_RATIO ** (stage + 1)
    dataset.set_max_ratio(start_ratio)
    if progress > 0.5:
        dataset.set_combined()
        if progress > 0.6:
            model.chamfer_criteria.set_threshold(CD_THRESHOLD)
    else:
        model.chamfer_criteria.unset_threshold()
        dataset.unset_combined()

    dataloader = data.DataLoader(dataset, batch_size=1, pin_memory=True)

    # visualization
    vis_logger = visdom.Visdom(env=FLAGS.id)
    for epoch in range(start_epoch + 1, MAX_EPOCH):
        for i, examples in enumerate(dataloader):
            input_pc, label_pc, ratio = examples
            ratio = ratio.item()
            # 1xBx3xN
            input_pc = input_pc[0].to(DEVICE)
            label_pc = label_pc[0].to(DEVICE)
            model.set_input(input_pc, ratio, label_pc=label_pc)
            # run gradient decent and increment model.step
            model.optimize()
            new_stage, new_progress = get_stage_progress(model.step)
            # advance to the next training stage with an added ratio
            if stage + 1 == new_stage:
                dataset.add_next_ratio()
                dataset.unset_combined()
                model.chamfer_criteria.unset_threshold()
            # advance to the combined stage
            if progress <= 0.5 and new_progress > 0.5:
                dataset.set_combined()
            # chamfer loss set ignore threshold
            if new_progress > 0.6:
                model.chamfer_criteria.set_threshold(CD_THRESHOLD)
            if model.step % 50 == 0:
                output = model.predicted.transpose(2, 1)[0].cpu()
                gt = model.gt.transpose(2, 1)[0].cpu()
                input_pc = input_pc.transpose(2, 1)[0].cpu()
                vis_logger.scatter(input_pc, win="x{}_input".format(ratio),
                                   opts=dict(title="x{}_input".format(ratio),
                                             markersize=2))
                vis_logger.scatter(output, win="x{}_output".format(ratio),
                                   opts=dict(title="x{}_output".format(ratio),
                                             markersize=2))
                vis_logger.scatter(gt, win="x{}_gt".format(ratio),
                                   opts=dict(title="x{}_label".format(ratio),
                                             markersize=2))
                vis_logger.line(
                    np.array([model.error_log["cd_loss_x{}".format(ratio)]]),
                    np.array([model.step]),
                    update="append",
                    win="x{}_loss".format(ratio),
                    opts=dict(title="x{}_loss".format(ratio)))

            stage, progress = new_stage, new_progress

        # end of epoch
        logger.info("epoch %d: " % epoch +
                    ", ".join(["{}={}".format(k, v) for k, v in model.error_log.items()]))
        if epoch % 20 == 0:
            pytorch_utils.save_network(net, MODEL_DIR,
                                       "model", epoch_label=str(epoch),
                                       step=str(model.step))

def pc_prediction(net, input_pc, patch_num_ratio=3):
    """
    upsample patches of a point cloud
    :param
        input_pc        1x3xN
        patch_num_ratio int, impacts number of patches and overlapping
    :return
        input_list      list of [3xM]
        up_point_list   list of [3xMr]
    """
    # divide to patches
    num_patches = int(input_pc.shape[2] / NUM_POINT * patch_num_ratio)
    # FPS sampling
    start = time.time()
    _, seeds = operations.furthest_point_sample(
        input_pc, num_patches, NCHW=True)
    print("number of patches: %d" % seeds.shape[-1])
    input_list = []
    up_point_list = []

    patches, _, _ = operations.group_knn(
        NUM_POINT, seeds, input_pc, NCHW=True)

    for k in tqdm(range(num_patches)):
        patch = patches[:, :, k, :]
        patch, centroid, radius = operations.normalize_point_batch(
            patch, NCHW=True)
        up_point = net.forward(patch.detach(), ratio=UP_RATIO)
        up_point = up_point * radius + centroid
        input_list.append(patch)
        up_point_list.append(up_point)

    return input_list, up_point_list

def pc_visualization(net, input_pc, patch_num_ratio=3):
    """
    upsample patches of a point cloud
    :param
        input_pc        1x3xN
        patch_num_ratio int, impacts number of patches and overlapping
    :return
        input_list      list of [3xM]
        up_point_list   list of [3xMr]
    """
    # divide to patches
    num_patches = int(input_pc.shape[2] / NUM_POINT * patch_num_ratio)
    # FPS sampling
    start = time.time()
    _, seeds = operations.furthest_point_sample(
        input_pc, num_patches, NCHW=True)
    print("number of patches: %d" % seeds.shape[-1])
    vis_xyz = defaultdict(list)
    vis_feat = defaultdict(list)
    vis_nnIdx = defaultdict(list)

    patches, _, _ = operations.group_knn(
        NUM_POINT, seeds, input_pc, NCHW=True)

    for k in tqdm(range(num_patches)):
        patch = patches[:, :, k, :]
        net.forward(patch.detach(), ratio=UP_RATIO, phase="vis")
        for k in net.vis:
            if "Idx" in k:
                xyz, nnIdx = net.vis[k]
                vis_nnIdx[k].append(nnIdx)
                vis_xyz[k].append(xyz)
            else:
                xyz, feat = net.vis[k]
                vis_xyz[k].append(xyz)
                vis_feat[k].append(feat)
    return vis_xyz, vis_feat, vis_nnIdx

def vis(result_dir):
    """
    upsample a point cloud
    """
    from sklearn.manifold import TSNE
    from utils.interactive_visualizer import Painter
    # loaded_states = np.load(CKPT).item()
    # net.load_state_dict(loaded_states)
    # pytorch_utils.save_network(net, os.path.dirname(CKPT), "final", "poisson")
    pytorch_utils.load_network(net, CKPT)
    net.to(DEVICE)
    net.eval()
    test_files = glob(TEST_DATA, recursive=True)
    for point_path in test_files:
        folder = os.path.basename(os.path.dirname(point_path))
        out_path = os.path.join(result_dir, folder,
                                point_path.split('/')[-1][:-4] + '.ply')
        data = pc_utils.load(point_path, NUM_SHAPE_POINT)
        data = data[np.newaxis, ...]
        num_shape_point = data.shape[1] * FLAGS.drop_out

        # transpose to NCHW format
        data = torch.from_numpy(data).transpose(2, 1).to(device=DEVICE)

        logger.info(os.path.basename(point_path))
        start = time.time()
        with torch.no_grad():
            # 1x3xN
            xyz_dictlist, feat_dictlist, nnIdx_dictlist = pc_visualization(
                net, data, patch_num_ratio=PATCH_NUM_RATIO)

        for k, v in nnIdx_dictlist.items():
            xyz = xyz_dictlist[k]
            for p in range(1, len(v)):
                # v shape is 1xNxK
                v[p] += v[p - 1].shape[1]
            xyz = torch.cat(xyz, dim=-1)
            xyz = xyz.transpose(2, 1).cpu().numpy()[0, ...]
            nnIdx = torch.cat(v, dim=1)
            nnIdx = nnIdx.cpu().numpy()[0, ...]
            painter = Painter("NN Feature")
            painter.nnIdx = nnIdx
            painter.interactive_3D_plot(xyz, k)

def test(result_dir):
    """
    upsample a point cloud
    """
    pytorch_utils.load_network(net, CKPT)
    net.to(DEVICE)
    net.eval()
    test_files = glob(TEST_DATA, recursive=True)
    for point_path in test_files:
        folder = os.path.basename(os.path.dirname(point_path))
        out_path = os.path.join(result_dir, folder,
                                point_path.split('/')[-1][:-4] + '.ply')
        data = pc_utils.load(point_path, NUM_SHAPE_POINT)
        data = data[np.newaxis, ...]
        num_shape_point = data.shape[1] * FLAGS.drop_out
        if FLAGS.drop_out < 1:
            _, data = operations.furthest_point_sample(
                data, int(num_shape_point))
        # normalize "unnecessarily" to apply noise
        data, centroid, furthest_distance = pc_utils.normalize_point_cloud(
            data)
        is_2D = np.all(data[:, :, 2] == 0)
        if JITTER:
            data = pc_utils.jitter_perturbation_point_cloud(
                data, sigma=FLAGS.jitter_sigma, clip=FLAGS.jitter_max, is_2D=is_2D)

        # transpose to NCHW format
        data = torch.from_numpy(data).transpose(2, 1).to(device=DEVICE)

        logger.info(os.path.basename(point_path))
        start = time.time()
        with torch.no_grad():
            # 1x3xN
            input_pc_list, pred_pc_list = pc_prediction(
                net, data, patch_num_ratio=PATCH_NUM_RATIO)

        # for i, patch_pair in enumerate(zip(input_pc_list, pred_pc_list)):
        #     in_patch, out_patch = patch_pair
        #     pc_utils.save_ply(in_patch.transpose(2, 1).cpu().numpy()[
        #                       0], path[:-4]+'_input_%d.ply' % i)
        #     pc_utils.save_ply(out_patch.transpose(2, 1).cpu().numpy()[
        #                       0], path[:-4]+'_output_%d.ply' % i)
        pred_pc = torch.cat(pred_pc_list, dim=-1)
        input_point = torch.cat(input_pc_list, dim=-1)
        end = time.time()
        print("total time: ", end - start)
        _, pred_pc = operations.furthest_point_sample(
            pred_pc, int(num_shape_point) * UP_RATIO, NCHW=True)
        pred_pc = pred_pc.transpose(2, 1).cpu().numpy()
        pred_pc = (pred_pc * furthest_distance) + centroid
        data = data.transpose(2, 1).cpu().numpy()
        data = (data * furthest_distance) + centroid
        data = data[0, ...]
        pred_pc = pred_pc[0, ...]

        pc_utils.save_ply(data, out_path[:-4] + '_input.ply')
        pc_utils.save_ply(pred_pc, out_path[:-4] + '.ply')

if __name__ == "__main__":
    append_name = []  # type: ignore
    if NUM_POINT is None:
        append_name += ["pWhole"]
    else:
        append_name += ["p%d" % NUM_POINT]
    if NUM_SHAPE_POINT is None:
        append_name += ["sWhole"]
    else:
        append_name += ["s%d" % NUM_SHAPE_POINT]

    if JITTER:
        append_name += ["s{}".format("{:.4f}".format(
            FLAGS.jitter_sigma).replace(".", ""))]
    else:
        append_name += ["clean"]

    if FLAGS.drop_out < 1:
        append_name += ["d{}".format(
            "{:.2f}".format(FLAGS.drop_out).replace(".", ""))]

    result_path = FLAGS.result_dir or os.path.join(
        MODEL_DIR, 'result', 'x%d' % (UP_RATIO), "_".join(append_name))

    if PHASE == "test":
        assert(CKPT is not None)
        test(result_path)
    elif PHASE == "vis":
        assert(CKPT is not None)
        vis(result_path)
    elif PHASE == "train":
        import visdom
        train()


import numpy as np
import cv2
import preproc as pp

res_x = 1366
res_y = 768

#PODZIELIC NA DWIE FUNKCJE
def map_to_img(img, gaze_df, res_x = 1366, res_y = 768):

	gaze_x = gaze_df["gaze_x"]*res_x
	gaze_y = gaze_df["gaze_y"]*res_y
	timestamp = gaze_df["timestamp"]

	pixels_coord = np.array([gaze_x, gaze_y, timestamp])
	pixels_coord = pixels_coord.T
	pixels_coord = pixels_coord.astype(np.int)

	coord_x, coord_y = int(gaze_x[0]), int(gaze_y[0])
	coord_x_prev, coord_y_prev = coord_x, coord_y

	cv2.circle(img, (coord_x, coord_y), 3, (0, 0, 0), -1)

	for coord in range(1, len(gaze_df.index)):
		coord_x, coord_y = int(gaze_x[coord]), int(gaze_y[coord])
		cv2.line(img, (coord_x_prev, coord_y_prev), (coord_x, coord_y), (0, 0, 0), 2)
		cv2.circle(img, (coord_x, coord_y), 3, (0, 0, 0), -1)
		coord_x_prev, coord_y_prev = coord_x, coord_y

	cv2.namedWindow(winname = "Gaze path")
	img2 = cv2.resize(img, (683, 384))
	cv2.imshow("Gaze path", img2)

	k = cv2.waitKey(0)

	cv2.imwrite("eye_path.jpg", img)

	cv2.destroyWindow("Gaze path")

	return pixels_coord

if __name__ == "__main__":

	print((map_to_img(res_x = 1366, res_y = 768))[0])

	k = cv2.waitKey(0)


import os
import numpy as np
from jinja2 import Template
import glob

# First and last mixer
print('First mixer to relaunch job :')
first_mixer = int(input())
print('Last mixer to relaunch job :')
last_mixer = int(input())

this_path = os.getcwd() + '/..'
for mixer in np.linspace(first_mixer, last_mixer, (last_mixer-first_mixer)+1, dtype=int):
    geo_path = this_path + '/mixer_' + str(mixer)
    os.chdir(geo_path)

    files_out = glob.glob('mixer_' + str(mixer) + '-*.out')
    id_files_out = [int(file.split('-')[1].split('.')[0]) for file in files_out]
    id_files_out.sort()
    last_file_out = id_files_out[-1]

    with open('mixer_' + str(mixer) + '-' + str(last_file_out) + '.out') as f:
        output = f.readlines()

    if not os.path.isfile('torque.00.dat') and output[-2] == 'Aborting!\n' and output[-3] != 'std::exception\n':
        os.system('cp ../utils/mixer.geo .')

        tag = open("mixer.txt","r")
        ratios = tag.read()
        ratio = ratios.split('\t')
        tag.close()

        last = open("last_theta.txt","w")
        last.write(str(float(ratio[13])))
        last.close()

        tag = open("mixer.txt","w")
        ratio[13] = '0.000000'
        ratios = "\t".join(ratio)
        tag.write(ratios)
        tag.close()

        geo = open("mixer.geo","r")
        cte = geo.read()
        template = Template(cte)
        geometries = template.render(ratioTD = float(ratio[1]),
                                     ratioHT = float(ratio[3]),
                                     ratioTC = float(ratio[5]),
                                     ratioDW = float(ratio[7]),
                                     ratioDW_Hub = float(ratio[9]),
                                     p_thick = float(ratio[11]),
                                     theta = 0.0,
                                     min_mesh_length = "{{min_mesh_length}}",
                                     max_mesh_length = "{{max_mesh_length}}")

        geo.close()
        geo = open("mixer.geo","w")
        geo.write(geometries)
        geo.close()

        os.system('cp ../utils/relaunch_lethe.sh .')
        os.system('cp ../utils/relaunch_lethe.py .')
        os.system('sbatch -J ' + 'mixer_' + str(mixer) + ' relaunch_lethe.sh')

os.chdir('../utils')


from readdata import read_file,build_batch,read_dict
import paddle.fluid as fluid
from net import SentimentClassifier

train_path = '/home/aistudio/data/data54118/en_text/train'
test_path = '/home/aistudio/data/data54118/en_text/test'
model_path = 'model_path'
batch_size = 128
epoch_num = 5
embedding_size = 256
step = 0
learning_rate = 0.01
max_seq_len = 128
vocab_size = 252173

train_data,train_label = read_file(path=train_path,train=True)
word2id_dict = read_dict('word2id_dict')
test_data,test_label = read_file(path = test_path,train=False)

use_gpu = True
place = fluid.CUDAPlace(0) if use_gpu else fluid.CPUPlace()
with fluid.dygraph.guard(place):
    # 创建一个用于情感分类的网络实例，sentiment_classifier
    sentiment_classifier = SentimentClassifier(
        embedding_size, vocab_size, num_steps=max_seq_len)
    # 创建优化器AdamOptimizer，用于更新这个网络的参数
    adam = fluid.optimizer.AdamOptimizer(learning_rate=learning_rate, parameter_list=sentiment_classifier.parameters())

    for sentences, labels in build_batch(
            word2id_dict, train_data,train_label, batch_size, epoch_num, max_seq_len):
        #print(sentences[0])
        #print(labels[0])
        sentences_var = fluid.dygraph.to_variable(sentences)
        labels_var = fluid.dygraph.to_variable(labels)
        pred, loss = sentiment_classifier(batch_size,embedding_size,sentences_var, labels_var)

        loss.backward()
        adam.minimize(loss)
        sentiment_classifier.clear_gradients()

        step += 1
        if step % 10 == 0:
            print("step %d, loss %.3f" % (step, loss.numpy()[0]))
        if step % 500 == 0:
            fluid.save_dygraph(sentiment_classifier.state_dict(), model_path + '/' +str(step))
    #最后一步保存
    fluid.save_dygraph(sentiment_classifier.state_dict(), model_path + '/' + str(step))
    sentiment_classifier.eval()
    # 这里我们需要记录模型预测结果的准确率
    # 对于二分类任务来说，准确率的计算公式为：
    # (true_positive + true_negative) /
    # (true_positive + true_negative + false_positive + false_negative)
    tp = 0.
    tn = 0.
    fp = 0.
    fn = 0.
    for sentences, labels in build_batch(
            word2id_dict, test_data,test_label, batch_size, 1, max_seq_len):

        sentences_var = fluid.dygraph.to_variable(sentences)
        labels_var = fluid.dygraph.to_variable(labels)

        # 获取模型对当前batch的输出结果
        pred, loss = sentiment_classifier(batch_size,embedding_size,sentences_var, labels_var)
        #print(labels)
        # 把输出结果转换为numpy array的数据结构
        # 遍历这个数据结构，比较预测结果和对应label之间的关系，并更新tp，tn，fp和fn
        pred = pred.numpy()
        for i in range(len(pred)):
            if labels[i][0] == 1:
                if pred[i][1] > pred[i][0]:
                    tp += 1
                else:
                    fn += 1
            else:
                if pred[i][1] > pred[i][0]:
                    fp += 1
                else:
                    tn += 1

    # 输出最终评估的模型效果
    print("the acc in the test set is %.3f" % ((tp + tn) / (tp + tn + fp + fn)))