text:
from spaceNetUtilities import evalTools as eT
from spaceNetUtilities import geoTools as gT
import numpy as np
import csv
import multiprocessing
import time
import argparse
import os
import glob
from osgeo import ogr, osr, gdal

def writeAOISummaryToCSV(resultsDict,csvwriter):

    csvwriter.writerow(['TruthFile', resultsDict['TruthFile']])
    csvwriter.writerow(['ProposalFile', resultsDict['ProposalFile']])
    csvwriter.writerow(['AOI_Name', resultsDict['AOI_Name']])
    csvwriter.writerow(['Summary Results'])
    csvwriter.writerow(['F1Score Total', resultsDict['F1ScoreTotal']])
    csvwriter.writerow(['Precision', resultsDict['PrecisionTotal']])
    csvwriter.writerow(['Recall', resultsDict['RecalTotal']])
    csvwriter.writerow(['True Positive Total', resultsDict['TruePositiveTotal']])
    csvwriter.writerow(['False Positive Total', resultsDict['FalsePositiveTotal']])
    csvwriter.writerow(['False Negative Total', resultsDict['FalseNegativeTotal']])
    csvwriter.writerow([''])

    # resultsDict = {'AOI_Name': aoiName
    #                    'TruthFile': truth_fp,
    #                'ProposalFile': test_fp,
    #                'F1ScoreTotal': F1ScoreTotal,
    #                'PrecisionTotal': precision,
    #                'RecalTotal': recall,
    #                'TruePositiveTotal': true_pos_total,
    #                'FalsePositiveTotal': false_pos_total,
    #                'FalseNegativeTotal': false_neg_total,
    #                'PerImageStatsResultList': result_list,
    #                'OutputSummaryFile': resultsOutputFile}

def writePerChipToCSV(resultsDictList, csvwriter):
    resultsDict = resultsDictList[0]
    csvwriter.writerow(['ImageId', 'F1Score', 'True Positive Count', 'False Positive Count', 'False Negative Count'])
    for result in resultsDict['PerImageStatsResultList']:
        tmpList = [result[1]]
        tmpList.extend(result[0])
        csvwriter.writerow(tmpList)

def writeResultsToScreen(resultsDict):

    print('AOI of Interest', resultsDict['AOI_Name'])
    print('True_Pos_Total', resultsDict['TruePositiveTotal'])
    print('False_Pos_Total', resultsDict['FalsePositiveTotal'])
    print('False_Neg_Total', resultsDict['FalseNegativeTotal'])
    print('F1ScoreTotal', resultsDict['F1ScoreTotal'])

# resultsDict = {'AOI_Name': aoiName
#                    'TruthFile': truth_fp,
#                'ProposalFile': test_fp,
#                'F1ScoreTotal': F1ScoreTotal,
#                'PrecisionTotal': precision,
#                'RecalTotal': recall,
#                'TruePositiveTotal': true_pos_total,
#                'FalsePositiveTotal': false_pos_total,
#                'FalseNegativeTotal': false_neg_total,
#                'PerImageStatsResultList': result_list,
#                'OutputSummaryFile': resultsOutputFile}

def evaluateSpaceNetSolution(summaryTruthFile, summaryProposalFile, resultsOutputFile='', processgeoJson=False,
                             useParallelProcessing=False, minPolygonSize=0,
                             iouThreshold=0.5,
                             AOIList=['Total',
                                      'AOI_1_Rio',
                                      'AOI_2_Vegas',
                                      'AOI_3_Paris',
                                      'AOI_4_Shanghai',
                                      'AOI_5_Khartoum']
                             ):

    truth_fp = summaryTruthFile
    test_fp = summaryProposalFile
    # check for cores available
    if useParallelProcessing:

        max_cpu = multiprocessing.cpu_count()
        parallel = True
    else:
        max_cpu = 1
        parallel = False

    # initialize scene counts
    true_pos_counts = []
    false_pos_counts = []
    false_neg_counts = []

    t0 = time.time()
    # Start Ingest Of Truth and Test Case
    if processgeoJson:
        sol_polys = gT.import_summary_geojson(truth_fp, removeNoBuildings=False)
        prop_polys = gT.import_summary_geojson(test_fp)
        polyFlag = 'poly'
    else:
        sol_polys = gT.readwktcsv(truth_fp, removeNoBuildings=False)
        prop_polys = gT.readwktcsv(test_fp, groundTruthFile=False)
        polyFlag = 'polyPix'

    t1 = time.time()
    total = t1 - t0
    print('time of ingest: ', total)

    # inspect polygons to ensure they are not too small
    sol_polys  = [item for item in sol_polys if item["ImageId"] > 0 and
                                item[polyFlag].GetArea()> minPolygonSize]
    prop_polys = [item for item in prop_polys if item["ImageId"] > 0 ]

    # Speed up search by preprocessing ImageId and polygonIds

    test_image_ids = [item['ImageId'] for item in prop_polys if item['ImageId'] > 0]
    test_image_ids2 = [item['ImageId'] for item in sol_polys if item['ImageId'] > 0]
    test_image_ids.extend(test_image_ids2)
    test_image_ids = set(test_image_ids)

    prop_polysIdList = np.asarray([item['ImageId'] for item in prop_polys if item["ImageId"] >= 0 and \
                                   item['BuildingId'] != -1])
    prop_polysPoly = np.asarray([item[polyFlag] for item in prop_polys if item["ImageId"] >= 0 and \
                                 item['BuildingId'] != -1])

    sol_polysIdsList = np.asarray([item['ImageId'] for item in sol_polys if item["ImageId"] >= 0 and \
                                   item['BuildingId'] != -1])
    sol_polysPoly = np.asarray([item[polyFlag] for item in sol_polys if item["ImageId"] >= 0 and \
                                item['BuildingId'] != -1])
    bad_count = 0
    F1ScoreList = []
    cpu_count = min(multiprocessing.cpu_count(), max_cpu)
    print('{}'.format(max_cpu))
    p = multiprocessing.Pool(processes=cpu_count)
    ResultList = []

    eval_function_input_list = eT.create_eval_function_input((test_image_ids,
                                                              (prop_polysIdList, prop_polysPoly),
                                                              (sol_polysIdsList, sol_polysPoly)))

    # Calculate Values
    t3 = time.time()
    print('time For DataCreation {}s'.format(t3 - t1))

    # result_list = p.map(eT.evalfunction, eval_function_input_list)
    if parallel == False:
        result_list = []
        for eval_input in eval_function_input_list:
            if resultsOutputFile != '':
                result_list.append(eT.evalfunction(eval_input,
                                                   resultGeoJsonName=os.path.splitext(resultsOutputFile)[0]+"_"+eval_input[0]+".geojson",
                                                   threshold=iouThreshold))
            else:
                result_list.append(eT.evalfunction(eval_input,
                                                   threshold=iouThreshold))
    else:
        result_list = p.map(eT.evalfunction, eval_function_input_list)

    result_listNP = np.asarray([item[0] for item in result_list])
    result_listName = [item[1] for item in result_list]
    AOIIndexList = []
    resultsDictList = []
    for AOI in AOIList:
        if AOI !='Total':
            AOIIndex = [i for i, s in enumerate(result_listName) if AOI in s]
            AOIIndexList.append(AOIIndex)
            result_sum = np.sum(result_listNP[AOIIndex], axis=0)
        else:
            AOIIndex = [i for i, s in enumerate(result_listName) if '' in s]
            AOIIndexList.append(AOIIndex)
            result_sum = np.sum(result_listNP, axis=0)

        #result_sum = np.sum(result_listNP, axis=0)
        true_pos_total = result_sum[1]
        false_pos_total = result_sum[2]
        false_neg_total = result_sum[3]
        if (float(true_pos_total) + float(false_pos_total))  > 0:
            precision = float(true_pos_total) / (float(true_pos_total) + float(false_pos_total))
        else:
            precision = 0

        if (float(true_pos_total) + float(false_neg_total)) > 0:
            recall = float(true_pos_total) / (float(true_pos_total) + float(false_neg_total))
        else:
            recall = 0

        if (precision + recall) > 0:
            F1ScoreTotal = 2.0 * precision * recall / (precision + recall)
        else:
            F1ScoreTotal = 0

        resultsDict = {'AOI_Name': AOI,
                       'TruthFile': truth_fp,
                       'ProposalFile': test_fp,
                       'F1ScoreTotal': F1ScoreTotal,
                       'PrecisionTotal': precision,
                       'RecalTotal': recall,
                       'TruePositiveTotal': true_pos_total,
                       'FalsePositiveTotal': false_pos_total,
                       'FalseNegativeTotal': false_neg_total,
                       'PerImageStatsResultList': result_list,
                       'OutputSummaryFile': resultsOutputFile}

        resultsDictList.append(resultsDict)
        writeResultsToScreen(resultsDict)

    if resultsOutputFile != '':
        with open(resultsOutputFile, 'w') as csvFile:
            csvwriter = csv.writer(csvFile, delimiter=',')
            for resultsDict in resultsDictList:
                writeAOISummaryToCSV(resultsDict, csvwriter)

            writePerChipToCSV(resultsDictList, csvwriter)

    return resultsDictList

def combineGeoJsonAndConvertToWGS84(baseName, rasterLocationList,
                                    AOIList=['Total',
                                             'AOI_1_Rio',
                                             'AOI_2_Vegas',
                                             'AOI_3_Paris',
                                             'AOI_4_Shanghai',
                                             'AOI_5_Khartoum'],
                                    removeGeoJsonAfter=True):
    srcBaseName = os.path.splitext(baseName)[0]
    geoJsonList = glob.glob(srcBaseName+"_*.geojson")
    print(geoJsonList)

    rasterList = []
    for rasterLocation in rasterLocationList:
        rasterList.extend(glob.glob(os.path.join(rasterLocation, '*.tif')))

    for AOI in AOIList:
        AOIgeoJsonList = [s for i, s in enumerate(geoJsonList) if AOI in s]
        AOIimageList   = [s for i, s in enumerate(rasterList) if AOI in s]

        #print AOIimageList
        outShapeFile = srcBaseName+"_"+AOI+"Summary.shp"
        outDriver = ogr.GetDriverByName("ESRI Shapefile")
        # Remove output shapefile if it already exists
        if os.path.exists(outShapeFile):
            outDriver.DeleteDataSource(outShapeFile)
        outDataSource = outDriver.CreateDataSource(outShapeFile)
        srs = osr.SpatialReference()
        srs.ImportFromEPSG(4326)
        outLayer = outDataSource.CreateLayer("AOI_Results", srs, geom_type=ogr.wkbPolygon)

        inDataSource = ogr.Open(geoJsonList[0], 0)
        inLayer = inDataSource.GetLayer()
        # Add input Layer Fields to the output Layer
        inLayerDefn = inLayer.GetLayerDefn()
        for i in range(0, inLayerDefn.GetFieldCount()):
            fieldDefn = inLayerDefn.GetFieldDefn(i)
            outLayer.CreateField(fieldDefn)
        # Get the output Layer's Feature Definition
        outLayerDefn = outLayer.GetLayerDefn()

        inLayerDefn = 0
        inLayer = 0
        inDataSource = 0
        for AOIgeoJson in AOIgeoJsonList:
            imageId = AOIgeoJson.replace(srcBaseName+'_', "").replace('.geojson', '')
            rasterName = [s for i, s in enumerate(AOIimageList) if imageId in s]

            if len(rasterName)>0:
                rasterName = rasterName[0]
                #print rasterName
                inDataSource = ogr.Open(AOIgeoJson, 0)
                inLayer = inDataSource.GetLayer()

                targetRaster = gdal.Open(rasterName)
                geomTransform = targetRaster.GetGeoTransform()

                for i in range(0, inLayer.GetFeatureCount()):
                    # Get the input Feature
                    inFeature = inLayer.GetFeature(i)
                    # Create output Feature
                    outFeature = ogr.Feature(outLayerDefn)
                    # Add field values from input Layer
                    for i in range(0, outLayerDefn.GetFieldCount()):
                        outFeature.SetField(outLayerDefn.GetFieldDefn(i).GetNameRef(), inFeature.GetField(i))
                    # Set geometry as centroid
                    geom = inFeature.GetGeometryRef()
                    #print geom.ExportToWkt()
                    # [GeoWKT, PixWKT])
                    geomList = gT.pixelGeomToGeoGeom(geom, rasterName, targetSR='', geomTransform='', breakMultiPolygonPix=False)
                    #print geomList[0][0].ExportToWkt()
                    if geomList:
                        outFeature.SetGeometry(geomList[0][0])

                    # Add new feature to output Layer
                        outLayer.CreateFeature(outFeature)
                        outFeature = None
                        inFeature = None

    if removeGeoJsonAfter:
        for f in geoJsonList:
            os.remove(f)

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Evaluate Score for SpaceNet')
    parser.add_argument("summaryTruthFile",
                        help="The Location of Summary Ground Truth csv File"
                             "Summary File should have a header = ImageId, BuildingId, polygonPixWKT, polygonGeoPix "
                             "Format is '{},{},{},{}.format(ImageId, BuildingId, polygonPixWKT, polygonGeoPix)',"
                             "unless --geoJson flag is set"
                             "See spaceNet competition details for more information about file format"
                        )
    parser.add_argument("summaryProposalFile",
                        help="The Location of summary Propsal csv File"
                             "Summary File should have a header = ImageId, BuildingId, polygonPixWKT, Confidence "
                             "followed by values"
                             "Format is '{},{},{},{}.format(ImageId, BuildingId, polygonPixWKT, Confidence)'"
                             "unless --geoJson flag is set"
                        )
    parser.add_argument("--polygonMinimumPixels",
                        help="The minimum number of pixels a polygon must have to be considered valid"
                             "The minimum for spacenet round 2 is 20 pixels",
                        type=int,
                        default=20)
    parser.add_argument("--iouThreshold",
                        help="The IOU threshold for a True Positive"
                             "Spacenet uses 0.5",
                        type=float,
                        default=0.5)

    parser.add_argument("--resultsOutputFile",
                        help="If you would like summary data outwritten to a file, specify the file",
                        default='')
    parser.add_argument("--geoJson",
                        help='results Submitted are in geoJson Format',
                        action='store_true')

    parser.add_argument("--useParallelProcessing",
                        help='Convert Image from Native format to 8bit',
                        action='store_true')

    parser.add_argument("--rasterLocation",
                        help='Image Directory List',
                        action='append',
                        default=[]
                        )

    args = parser.parse_args()
    # load Truth and Test File Locations
    AOIList = ['Total',
               'AOI_1_Rio',
               'AOI_2_Vegas',
               'AOI_3_Paris',
               'AOI_4_Shanghai',
               'AOI_5_Khartoum']
    resultsOutputFile = args.resultsOutputFile
    summaryDict = evaluateSpaceNetSolution(args.summaryTruthFile,
                                           args.summaryProposalFile,
                                           resultsOutputFile=resultsOutputFile,
                                           processgeoJson=args.geoJson,
                                           useParallelProcessing=args.useParallelProcessing,
                                           minPolygonSize=args.polygonMinimumPixels,
                                           iouThreshold=args.iouThreshold,
                                           AOIList=AOIList)

    if resultsOutputFile != '':
        combineGeoJsonAndConvertToWGS84(resultsOutputFile,
                                        rasterLocationList=args.rasterLocation)

max_stars_repo_name: giswqs/SpaceNet

max_issues_repo_name: giswqs/SpaceNet

max_forks_repo_name: giswqs/SpaceNet


text:
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K

from scipy.optimize import Bounds, minimize

def get_distortion(x, node, target, mean, covariance, model, mode="diag"):
    x_tensor = tf.constant(x, dtype=tf.float32)
    m_tensor = tf.constant(mean, dtype=tf.float32)
    c_tensor = tf.constant(covariance, dtype=tf.float32)
    s_flat = tf.placeholder(tf.float32, (np.prod(x_tensor.shape),))
    s_tensor = tf.reshape(s_flat, x.shape)
    mean_in = s_tensor * x_tensor + (1 - s_tensor) * m_tensor
    if mode == "diag":
        covariance_in = tf.square(1 - s_tensor) * c_tensor
    elif mode == "half":
        covariance_in = c_tensor * (1 - s_tensor)
    elif mode == "full":
        covrank = len(c_tensor.get_shape().as_list())
        perm = (
            [0]
            + list(range((covrank - 1) // 2 + 1, covrank))
            + list(range(1, (covrank - 1) // 2 + 1))
        )
        covariance_in = c_tensor * (1 - s_tensor)
        covariance_in = K.permute_dimensions(covariance_in, perm,)
        covariance_in = covariance_in * (1 - s_tensor)
        covariance_in = K.permute_dimensions(covariance_in, perm,)
    out_mean, out_covariance = model([mean_in, covariance_in])
    if mode == "diag":
        loss = (
            1
            / 2
            * (
                K.mean(K.square(out_mean[..., node] - target))
                + K.mean(out_covariance[..., node])
            )
        )
    elif mode == "half":
        out_covariance = K.sum(K.square(out_covariance), axis=1)
        loss = (
            1
            / 2
            * (
                K.mean(K.square(out_mean[..., node] - target))
                + K.mean(out_covariance[..., node])
            )
        )
    elif mode == "full":
        loss = (
            1
            / 2
            * (
                K.mean(K.square(out_mean[..., node] - target))
                + K.mean(out_covariance[..., node, node])
            )
        )
    gradient = K.gradients(loss, [s_flat])[0]
    f_out = K.function([s_flat], [loss])
    f_gradient = K.function([s_flat], [gradient])
    return lambda s: f_out([s])[0], lambda s: f_gradient([s])[0]

def get_sparsity_regularizer():
    def g(s):
        return np.mean(np.abs(s))

    def dg(s):
        return 1 / np.prod(s.shape) * np.sign(s)

    return g, dg

def get_bounds():
    return Bounds(0.0, 1.0)

def adf_rde_lagrangian(
    x, node, target, s0, mean, covariance, model, mu, mode="diag"
):
    f, df = get_distortion(x, node, target, mean, covariance, model, mode)
    g, dg = get_sparsity_regularizer()

    def obj(s):
        return f(s) + np.square(target) * mu * g(s)

    def dobj(s):
        return df(s) + np.square(target) * mu * dg(s)

    result = minimize(
        obj,
        s0.flatten(),
        jac=dobj,
        method="l-bfgs-b",  # alternatively 'tnc'
        bounds=get_bounds(),
        options={"disp": True, "verbose": 2},
    )
    if not result.success:
        print("Warning: Optimization failed. Result might be useless.")
        print(result.message)

    print("distortion pre:\t {:1.2e}".format(f(s0)))
    print("distortion post: {:1.2e}".format(f(result.x)))
    print("sparsity pre:\t {:1.2e}".format(g(s0)))
    print("sparsity post:\t {:1.2e}".format(g(result.x)))
    return np.reshape(result.x, x.shape)

ext: py

max_stars_repo_name: jmaces/rde

max_issues_repo_name: jmaces/rde

max_forks_repo_name: jmaces/rde

reason: import numpy,from scipy


text:
import numpy as np
import math
from bokeh.plotting import figure

####### Print solutions for u_y, u_z, sigma_xx and tau_xy on console to validate calculations   - on=1, off=0 -
## If you want to print the solutions, set glCantileverPrint to 1. Run the App and set both Forces sliders to the far right (Py=100 and Pz=100).
## You will get the properties of the beam and the calculated displacements / stresses printed on the console. (note: location (x,y,z) of calculated values vary)
global glCantileverPrint
glCantileverPrint = 0
#######

## Define element class with node coordinates as data members
class element():

    def __init__( self, lowerLeftPos,lowerRightPos, upperLeftPos,upperRightPos):
        self.lowerLeftPosition = lowerLeftPos
        self.upperRightPosition = upperRightPos
        self.lowerRightPosition = lowerRightPos
        self.upperLeftPosition = upperLeftPos

## Define coords class
class coords():

    def __init__( self, data):
        self.data = data

def construct_deformed_beam_centerLine( Pz, Py, E,
                                        noElementsX,
                                        thickness, height, length, elementSizeX,
                                        amplificationFactor, glCantileverCrossSection):

    ## Determine moment of inertia depending on active cross section:
    ## Iy = Sum(Iz_i) + Sum(ez_i^2*A_i)
    if(glCantileverCrossSection==0):
        Iy = thickness*height**3.0/12.0
    if(glCantileverCrossSection==1):
        Iy = 2*(height*(height/10.0)**3.0/12.0) + (height-2*height/10.0)**3*height/10.0/12.0 + 2.0*((height/2.0-height/20)**2.0*height*height/10.0)
    if(glCantileverCrossSection==2):
        Iy = math.pi*(height)**4.0/64.0
    if(glCantileverCrossSection==3):
        Iy = thickness**4.0/36.0

    ## Iz = Sum(Iy_i) + Sum(ey_i^2*A_i) = Sum(Iy_i) + 0
    if(glCantileverCrossSection==0):
        Iz = height*thickness**3.0/12.0
    if(glCantileverCrossSection==1):
        Iz = 2*(height/10.0*height**3.0/12.0) + (height/10.0)**3.0*(height-2*height/10.0)/12.0
    if(glCantileverCrossSection==2):
        Iz = math.pi*(height)**4.0/64.0
    if(glCantileverCrossSection==3):
        Iz = thickness**4.0/48.0

    ###### DEFORMATION IN XZ DIRECTION ######

    ## Define list for deformed XZ-centerline
    deformedBeamXZ = list()

    ## Calculate deformation and append element coordinates to list.
    # Take into account decreasing x-distance between elements due to deformation.
    xComponent = 0.0
    zComponent = 0.0
    for i in range(noElementsX + 1):
        zComponent = - amplificationFactor*Pz*xComponent*xComponent*( 3.0*length - xComponent )/(6.0*E*Iy)
        deformedBeamXZ.append( [xComponent, zComponent] )
        angle = np.arctan( ( Pz/E*Iy )*( length*xComponent - xComponent*xComponent/2.0) )
        xIncrement = elementSizeX*np.cos( angle )
        xComponent += xIncrement

    ###### DEFORMATION IN XY DIRECTION ######

    ## Define list for deformed XY-centerline
    deformedBeamXY = list()

    ## Calculate deformation and append element coordinates to list.
    # Take into account decreasing x-distance between elements due to deformation.
    xComponent = 0.0
    yComponent = 0.0
    for i in range(noElementsX + 1):
        yComponent = - amplificationFactor*Py*xComponent*xComponent*( 3.0*length - xComponent )/(6.0*E*Iz)
        deformedBeamXY.append( [xComponent, yComponent] )
        angle = np.arctan( ( Py/E*Iz )*( length*xComponent - xComponent*xComponent/2.0 ) )
        xIncrement = elementSizeX*np.cos( angle )
        xComponent += xIncrement

    ### PRINTSTART
    if (glCantileverPrint == 1):
        if xComponent > 5.0 and Py==100.0 and Pz==100.0:
            if glCantileverCrossSection == 0:
                print ("Rectangular CS")
                print ("(length, width, height) = ", "(", length, ", ", thickness, ", ", height, ")")
                print ("Iy = ", Iy)
                print ("Iz = ", Iz)
            elif glCantileverCrossSection == 1:
                print ("Double-T CS")
                print ("(length, width_total, width_elements, height) = ", "(", length, ", ", thickness, ", ", thickness/10, ", ", height, ")" )
                print ("Iy = ", Iy)
                print ("Iz = ", Iz)
            elif glCantileverCrossSection == 2:
                print ("Circular CS")
                print ("(length, d) = ", "(", length, ", ", thickness, ")")
                print ("Iy = ", Iy)
                print ("Iz = ", Iz)
            elif glCantileverCrossSection == 3:
                print ("Triangular CS")
                print ("(length, width, height) = ", "(", length, ", ", thickness, ", ", height, ")" )
                print ("Iy = ", Iy)
                print ("Iz = ", Iz)
            print ("Pz = ", Pz)
            print ("Py = ", Py)
            print ("u_y(x=5, y=0, z=0) = ", zComponent/amplificationFactor)
            print ("u_z(x=5, y=0, z=0) = ", yComponent/amplificationFactor)
    ### PRINTEND

    return deformedBeamXZ, deformedBeamXY

## Return color depending on range and current value:
def color_determiner( minimumValue, maximumValue, currentValue, glCantileverCrossSection ):
    if minimumValue == maximumValue:
        return '#'+'66'+'ff'+'66'
    else:
        # Define the four regions seperating between the essential colors: Blue, Sky, Green, Yellow, Red
        minimumPoint = [minimumValue , 816]
        maximumPoint = [maximumValue , 0 ]
        if glCantileverCrossSection != 0: #linear color distribution
            colorIndex = -(minimumPoint[1]-maximumPoint[1])/(maximumPoint[0] - minimumPoint[0]) * currentValue + (minimumPoint[1]-maximumPoint[1])/2
        else: #quadric color distribution
            if currentValue >= 0:
                colorIndex = (minimumPoint[1]-maximumPoint[1])/2 - math.sqrt( currentValue/(maximumPoint[0]) ) * (minimumPoint[1]-maximumPoint[1])/2
            else:
                colorIndex = (minimumPoint[1]-maximumPoint[1])/2 + math.sqrt( -currentValue/(maximumPoint[0]) ) * (minimumPoint[1]-maximumPoint[1])/2
        ratio = colorIndex / minimumPoint[1]
        # Prevent colorindex from getting too extreme, as this will cause irregularities in painting of beam:
        if colorIndex<20:
            colorIndex = 20
        if colorIndex>780:
            colorIndex = 780

        if glCantileverCrossSection != 0:
            # Cases
            if ratio <= 0.25:
                R = '33'
                B = 'ff'
                # determine G
                value = (255.0 / 204.0) * ( colorIndex )
                hexG = hex( int(value) )
                FstringHexG,SstringHexG = hexG.split( 'x' )
                G = SstringHexG

            elif ratio <= 0.50:
                R = '33'
                G = 'ff'
                # determine B
                value = ((51.0 - 255.0) / 204.0) * ( colorIndex - 408.0 ) + 51.0
                hexB = hex( int(value) )
                FstringHexB,SstringHexB = hexB.split( 'x' )
                B = SstringHexB

            elif ratio <= 0.75:
                G = 'ff'
                B = '33'
                # determine R
                value = ((255.0 - 51.0) / 204.0) * ( colorIndex - 612.0 ) + 255.0
                hexR = hex( int(value) )
                FstringHexR,SstringHexR = hexR.split( 'x' )
                R = SstringHexR

            else:
                R = 'ff'
                B = '33'
                # determine G
                value = ((51.0 - 255.0) / 204.0) * ( colorIndex - 816.0 ) + 51.0
                hexG = hex( int(value) )
                FstringHexG,SstringHexG = hexG.split( 'x' )
                G = SstringHexG
        else:
            # Cases
            if ratio <= 0.25:
                R = '33'
                B = 'ff'
                # determine G
                value = (255.0 / 204.0) * ( colorIndex )
                hexG = hex( int(value) )
                FstringHexG,SstringHexG = hexG.split( 'x' )
                G = SstringHexG

            elif ratio <= 0.50:
                R = '33'
                G = 'ff'
                # determine B
                value = ((51.0 - 255.0) / 204.0) * ( colorIndex - 408.0 ) + 51.0
                hexB = hex( int(value) )
                FstringHexB,SstringHexB = hexB.split( 'x' )
                B = SstringHexB

            elif ratio <= 0.75:
                G = 'ff'
                B = '33'
                # determine R
                value = ((255.0 - 51.0) / 204.0) * ( colorIndex - 612.0 ) + 255.0
                hexR = hex( int(value) )
                FstringHexR,SstringHexR = hexR.split( 'x' )
                R = SstringHexR

            else:
                R = 'ff'
                B = '33'
                # determine G
                value = ((51.0 - 255.0) / 204.0) * ( colorIndex - 816.0 ) + 51.0
                hexG = hex( int(value) )
                FstringHexG,SstringHexG = hexG.split( 'x' )
                G = SstringHexG

        return '#'+R+G+B

def construct_normal_vectors( deformedBeam ):
    # Constructing normals (parallel to deformed centerline, pointing back and forward)
    normalVector1 = list()
    normalVector2 = list()
    for i in range( len( deformedBeam ) ):
        if i == 0:
            # These appended normal vectors are valid for beam clamped at its left end
            normalVector1.append( [0, 1] )
            normalVector2.append( [0,-1] )
        else:
            dx = deformedBeam[i][0] - deformedBeam[i-1][0]
            dy = deformedBeam[i][1] - deformedBeam[i-1][1]
            length = np.sqrt(dx*dx + dy*dy)
            normal1 = [-dy/length , dx/length]
            normal2 = [ dy/length ,-dx/length]
            normalVector1.append( normal1 )
            normalVector2.append( normal2 )

    # Averaging normals
    averageNormalVector1 = list()
    averageNormalVector2 = list()
    for i in range( len( deformedBeam ) ):
        if i == len( deformedBeam ) - 1:
            normal1 = normalVector1[i]
            normal2 = normalVector2[i]
            averageNormalVector1.append( normal1 )
            averageNormalVector2.append( normal2 )
        elif i == 0:
            averageNormalVector1.append( [0,1] )
            averageNormalVector2.append( [0,-1] )
        else:
            xComp1 = (normalVector1[i][0] + normalVector1[i+1][0]) / 2.0
            xComp2 = (normalVector2[i][0] + normalVector2[i+1][0]) / 2.0
            yComp1 = (normalVector1[i][1] + normalVector1[i+1][1]) / 2.0
            yComp2 = (normalVector2[i][1] + normalVector2[i+1][1]) / 2.0
            normal1 = [xComp1,yComp1]
            normal2 = [xComp2,yComp2]
            averageNormalVector1.append(normal1)
            averageNormalVector2.append(normal2)

    return averageNormalVector1, averageNormalVector2

def values_determiner( Pz, Py, length, height, thickness, E, elementSizeX, glCantileverCrossSection):

    biggestValue = 1.5*calculate_normal_stress(0.0, 0.0, height/2.0, length, height, thickness, 2, 0, 100)
    smallestValue = 1.5*calculate_normal_stress(0.0, 0.0, -height/2.0, length, height, thickness, 2, 0, 100)

    listValuesUpperXY = list()
    listValuesLowerXZ = list()
    rangeSize = length/elementSizeX

    for i in range( int(rangeSize) ):

        valueUpperXY = calculate_normal_stress(i*elementSizeX, height/2.0, 0.0, length, height, thickness, glCantileverCrossSection, Pz, Py)
        listValuesUpperXY.append( valueUpperXY )

        valueLowerXZ = calculate_normal_stress(i*elementSizeX, 0.0, thickness/2.0, length, height, thickness, glCantileverCrossSection, Pz, Py)
        listValuesLowerXZ.append( valueLowerXZ )

    return biggestValue, smallestValue, listValuesUpperXY, listValuesLowerXZ

## Determine color for each element of input list
def elements_color_determiner( deformed,
                               orientation,
                               listElements,
                               noElementsX ,
                               noElementsZ ,
                               E , height  ,
                               thickness   ,
                               length, Pz, Py,
                               biggestValue,
                               smallestValue,
                               listAdditionalValues,
                               glCantileverCrossSection
                             ):

    # Define list for color of elements
    colorList = list()

    if deformed == True:
        elementSize = length/noElementsX
        xIncrement = 0.0
        verticalMultiplier = 1.0
        counter = 0

        if orientation == 'XZ':
            # For loop  to
            for i in range( int( len(listElements)/2 ) ):
                if i %(noElementsZ/2) == 0 and i != 0:
                # update verticalMultiplier to the original value
                    verticalMultiplier = 1.0
                    xIncrement += elementSize
                    counter += 1
                else:
                    pass

                x_pos = xIncrement
                if glCantileverCrossSection==0:
                    y_pos = -thickness/2.0
                    z_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                elif glCantileverCrossSection==1:
                    z_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    if abs(z_pos) < height*4.0/10.0:
                        y_pos = -thickness/10.0/2.0
                    else:
                        y_pos = -thickness/2.0
                elif glCantileverCrossSection==2:
                    z_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    y_pos = -np.cos(np.arcsin(z_pos/(thickness/2.0)))
                elif glCantileverCrossSection==3:
                    z_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)*1.0/2.0-(1.0/6.0)*height
                    y_pos = -(2.0/3.0 + ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0-(1.0/6.0)*height)*0.5
                strainXXup = calculate_normal_stress(x_pos, z_pos, y_pos, length, height, thickness, glCantileverCrossSection, Pz, Py)

                ### PRINTSTART
                if (glCantileverPrint == 1):
                    if (x_pos > 1.45 and x_pos < 1.55 and z_pos > 0.45 and Pz==100 and Py==100):
                        if (glCantileverCrossSection==0):
                            print ("Sigma(x=1.5, y=0.5, z=0.5) = ", strainXXup)
                        elif (glCantileverCrossSection==1):
                            print ("Sigma(x=1.5, y=0.5, z=0.5) = ", strainXXup)
                        elif (glCantileverCrossSection==2):
                            print ("Sigma(x=1.5, y=0.5, z=0.0) = ", strainXXup)
                    if (glCantileverCrossSection==3):
                            if (x_pos > 1.45 and x_pos < 1.55 and z_pos > 0.33 and Pz==100 and Py==100):
                                print ("Sigma(x=1.5, y=0.33, z=0.5) = ", strainXXup)
                ### PRINTEND

                x_pos = xIncrement
                if glCantileverCrossSection==0:
                    z_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    y_pos = -thickness/2.0
                elif glCantileverCrossSection==1:
                    z_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    if abs(z_pos) < height*4.0/10.0:
                        y_pos = -thickness/10.0/2.0
                    else:
                        y_pos = -thickness/2.0
                elif glCantileverCrossSection==2:
                    z_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    y_pos = -np.cos(np.arcsin(z_pos/(thickness/2.0)))
                elif glCantileverCrossSection==3:
                    z_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)*1.0/2.0-(1.0/6.0)*height
                    y_pos = -(2.0/3.0 - ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0-(1.0/6.0)*height)*0.5

                strainXXbottom = calculate_normal_stress(x_pos, z_pos, y_pos, length, height, thickness, glCantileverCrossSection, Pz, Py)

                elementColor = color_determiner( smallestValue , biggestValue , strainXXup, glCantileverCrossSection )
                colorList.append(elementColor)
                elementColor = color_determiner( smallestValue , biggestValue , strainXXbottom, glCantileverCrossSection )
                colorList.append(elementColor)

                verticalMultiplier += 1

        if orientation == 'XY':

            for i in range( int( len(listElements)/2 ) ):
                if i %(noElementsZ/2) == 0 and i != 0:
                # update verticalMultiplier to the original value
                    verticalMultiplier = 1.0
                    xIncrement += elementSize
                    counter += 1
                else:
                    pass

                x_pos = xIncrement
                if glCantileverCrossSection==0:
                    y_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    z_pos = height/2.0
                elif glCantileverCrossSection==1:
                    y_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    z_pos = height/2.0
                elif glCantileverCrossSection==2:
                    y_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    z_pos = np.sin(np.arccos(abs(y_pos)/(height/2.0)))
                elif glCantileverCrossSection==3:
                    y_pos = -((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)*1.0/2.0
                    z_pos = height*1.0/3.0

                strainXXbottom = calculate_normal_stress(x_pos, z_pos, y_pos, length, height, thickness, glCantileverCrossSection, Pz, Py)

                x_pos = xIncrement
                if glCantileverCrossSection==0:
                    y_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    z_pos = height/2.0
                elif glCantileverCrossSection==1:
                    y_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    z_pos = height/2.0
                elif glCantileverCrossSection==2:
                    y_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)/2.0
                    z_pos = np.sin(np.arccos(abs(y_pos)/(height/2.0)))
                elif glCantileverCrossSection==3:
                    y_pos = ((verticalMultiplier-1.0)*height)/(noElementsZ/2.0-1.0)*1.0/2.0
                    z_pos = height*1.0/3.0

                strainXXup = calculate_normal_stress(x_pos, z_pos, y_pos, length, height, thickness, glCantileverCrossSection, Pz, Py)

                elementColor = color_determiner( smallestValue , biggestValue , strainXXup , glCantileverCrossSection )
                colorList.append(elementColor)
                elementColor = color_determiner( smallestValue , biggestValue , strainXXbottom, glCantileverCrossSection )
                colorList.append(elementColor)

                verticalMultiplier += 1

    ## If beam undeformed, paint all green:
    else:
        for i in range( int( len(listElements) ) ):
            colorList.append( '#B3B6B7' )

    return colorList

def construct_deformed_elements( deformedBeam         ,
                                 axialDirElementSize  ,
                                 lateralDirElementSize,
                                 noElementsX          ,
                                 noElementsZ          ,
                                 normalVectors1       ,
                                 normalVectors2
                               ):

    listDeformedElements = list()

    for i in range(noElementsX):
        for j in range(int(noElementsZ/2)):

            lowerLeftPos1  = [ deformedBeam[i][0]   + float(j)  *axialDirElementSize  *normalVectors1[i][0]  ,
                               deformedBeam[i][1]   + float(j)  *lateralDirElementSize  *normalVectors1[i][1]   ]
            lowerRightPos1 = [ deformedBeam[i+1][0] + float(j)  *axialDirElementSize  *normalVectors1[i+1][0],
                               deformedBeam[i+1][1] + float(j)  *lateralDirElementSize  *normalVectors1[i+1][1] ]
            upperLeftPos1  = [ deformedBeam[i][0]   + float(j+1)*axialDirElementSize  *normalVectors1[i][0]  ,
                               deformedBeam[i][1]   + float(j+1)*lateralDirElementSize*normalVectors1[i][1]   ]
            upperRightPos1 = [ deformedBeam[i+1][0] + float(j+1)*axialDirElementSize  *normalVectors1[i+1][0],
                               deformedBeam[i+1][1] + float(j+1)*lateralDirElementSize*normalVectors1[i+1][1] ]

            lowerLeftPos2  = [ deformedBeam[i][0]   + float(j)  *axialDirElementSize  *normalVectors2[i][0]  ,
                               deformedBeam[i][1]   + float(j)  *lateralDirElementSize*normalVectors2[i][1]   ]
            lowerRightPos2 = [ deformedBeam[i+1][0] + float(j)  *axialDirElementSize  *normalVectors2[i+1][0],
                               deformedBeam[i+1][1] + float(j)  *lateralDirElementSize*normalVectors2[i+1][1] ]
            upperLeftPos2  = [ deformedBeam[i][0]   + float(j+1)*axialDirElementSize  *normalVectors2[i][0]  ,
                               deformedBeam[i][1]   + float(j+1)*lateralDirElementSize*normalVectors2[i][1]   ]
            upperRightPos2 = [ deformedBeam[i+1][0] + float(j+1)*axialDirElementSize  *normalVectors2[i+1][0],
                               deformedBeam[i+1][1] + float(j+1)*lateralDirElementSize*normalVectors2[i+1][1] ]

            listDeformedElements.append( element(lowerLeftPos1 ,lowerRightPos1, upperLeftPos1, upperRightPos1) )
            listDeformedElements.append( element(lowerLeftPos2 ,lowerRightPos2, upperLeftPos2, upperRightPos2) )

    return listDeformedElements

def create_coordinates_list( listElements):

    listXCoord = list()
    listYCoord = list()
    for element in listElements:
        listXCoord.append([ element.lowerLeftPosition[0],
                            element.upperLeftPosition[0],
                            element.upperRightPosition[0],
                            element.lowerRightPosition[0]])
        listYCoord.append([ element.lowerLeftPosition[1],
                            element.upperLeftPosition[1],
                            element.upperRightPosition[1],
                            element.lowerRightPosition[1]])

    return listXCoord , listYCoord

def calculate_stresses_xz_element(x_pos, z_pos, length, height, thickness, glCantileverCrossSection, Pz, Py):
    sigma_x_l = list()
    sigma_x_r = list()
    tau_xz = list()

    ##Element Properties:
    y_pos = 0
    length_of_element = 2.0

    if(glCantileverCrossSection==3):
        height_of_element = height*2.0/3.0
    else:
        height_of_element = height/2.0

    ## Determine moment of inertia depending on active cross section:
    ## Iy = Sum(Iz_i) + Sum(ez_i^2*A_i)
    if(glCantileverCrossSection==0):
        Iy = thickness*height**3.0/12.0
    if(glCantileverCrossSection==1):
        Iy = 2*(height*(height/10.0)**3.0/12.0) + (height-2*height/10.0)**3*height/10.0/12.0 + 2.0*((height/2.0-height/20)**2.0*height*height/10.0)
    if(glCantileverCrossSection==2):
        Iy = math.pi*(height)**4.0/64.0
    if(glCantileverCrossSection==3):
        Iy = thickness**4.0/36.0

    ## Iz = Sum(Iy_i) + Sum(ey_i^2*A_i) = Sum(Iy_i) + 0
    if(glCantileverCrossSection==0):
        Iz = height*thickness**3.0/12.0
    if(glCantileverCrossSection==1):
        Iz = 2*(height/10.0*height**3.0/12.0) + (height/10.0)**3.0*(height-2*height/10.0)/12.0
    if(glCantileverCrossSection==2):
        Iz = math.pi*(height)**4.0/64.0
    if(glCantileverCrossSection==3):
        Iz = thickness**4.0/48.0

    ## Iyz = Sum(Iyz_i) + Sum(ey_i*ez_i*A_i) = 0 + Sum(ey_i*ez_i*A_i) - Deviation momentum is zero because of symmetry of cross sections
    if(glCantileverCrossSection==0):
        Iyz = 0.0
    if(glCantileverCrossSection==1):
        Iyz = 0.0
    if(glCantileverCrossSection==2):
        Iyz = 0.0
    if(glCantileverCrossSection==3):
        Iyz = 0.0

    ## Calculation of Momentum M_y & M_z
    M_y_l = -(length-(x_pos-length_of_element/2.0)) * Pz
    M_y_r = -(length-(x_pos+length_of_element/2.0)) * Pz
    M_z_l = -(length-(x_pos-length_of_element/2.0)) * Py
    M_z_r = -(length-(x_pos+length_of_element/2.0)) * Py

    ## Calculation of sigma(x,y,z)
    n=11
    for i in range(n):
        # sigma(x,y,z) = (N(x)/A) + (My*Iz-Mz*Iyz)/(Iy*Iz-Iyz**2)*z + (Mz*Iy-My*Iyz)/(Iy*Iz-Iyz**2)*y
        sigma_x_l.append(M_z_l/Iz*y_pos + M_y_l/Iy*((i-n+1)/20.0))
        sigma_x_r.append(M_z_r/Iz*y_pos + M_y_r/Iy*((i-n+1)/20.0))

    ## Calculation of tau_xz
    m=10
    s_pos=0 #starting at the bottom
    if glCantileverCrossSection == 0:
        for i in range(m):
            s_pos=float(i)/float(m)*height_of_element
            tau_xz.append( - Pz/Iz * 1/2 * (height*s_pos-s_pos**2) )
    elif glCantileverCrossSection == 1:
        for i in range(m):
            s_pos=float(i)/float(m)*height_of_element
            tau_xz.append( - Pz/Iz * 1/2 * (height*s_pos-s_pos**2) )
    elif glCantileverCrossSection == 2:
        for i in range(m):
            s_pos=height_of_element-float(i)/float(m)*height_of_element
            tau_xz.append( - Pz * 4/(3*math.pi) * ((height/2)**2-s_pos**2)/((height/2)**4 ) )
    elif glCantileverCrossSection == 3:
        for i in range(m):
            s_pos=float(i)/float(m)*height_of_element
            tau_xz.append( - Pz/Iz * 1/3 * (height*s_pos-s_pos**2) )

### PRINTSTART
    if (glCantileverPrint == 1):
        if (Pz==100 and Py==100):
            if(glCantileverCrossSection==3):
                print ("Sigma_xx(x=1.5, y=-0,66, z=0) = ", -1*(sigma_x_l[0]))
            else:
                print ("Sigma_xx(x=1.5, y=-0,5, z=0) = ", -1*(sigma_x_l[0]))
            print ("Tau_xz(x=1.5, y=0, z=0) = ", tau_xz[-1])
            print ("")
### PRINTEND

    return sigma_x_l,sigma_x_r,tau_xz

def calculate_normal_stress(x_pos, z_pos, y_pos, length, height, thickness, glCantileverCrossSection, Pz, Py):

    ## Declare and initialize sigma:
    sigma = 0

    ## Calculate Iz, Iy and Iyz:
    ## Determine moment of inertia depending on active cross section:
    ## Iy = Sum(Iz_i) + Sum(ez_i^2*A_i)
    if(glCantileverCrossSection==0):
        Iy = thickness*height**3.0/12.0
    if(glCantileverCrossSection==1):
        Iy = 2*(height*(height/10.0)**3.0/12.0) + (height-2*height/10.0)**3*height/10.0/12.0 + 2.0*((height/2.0-height/20)**2.0*height*height/10.0)
    if(glCantileverCrossSection==2):
        Iy = math.pi*(height)**4.0/64.0
    if(glCantileverCrossSection==3):
        Iy = thickness**4.0/36.0

    ## Iz = Sum(Iy_i) + Sum(ey_i^2*A_i) = Sum(Iy_i) + 0
    if(glCantileverCrossSection==0):
        Iz = height*thickness**3.0/12.0
    if(glCantileverCrossSection==1):
        Iz = 2*(height/10.0*height**3.0/12.0) + (height/10.0)**3.0*(height-2*height/10.0)/12.0
    if(glCantileverCrossSection==2):
        Iz = math.pi*(height)**4.0/64.0
    if(glCantileverCrossSection==3):
        Iz = thickness**4.0/48.0

    #  Calculation of Iyz:   Iyz = Sum(Iyz_i) + Sum(ey_i*ez_i*A_i) = 0 + Sum(ey_i*ez_i*A_i)
    #  Deviation momentum is zero because of symmetry of cross sections
    if(glCantileverCrossSection==0):
        Iyz = 0.0
    if(glCantileverCrossSection==1):
        Iyz = 0.0
    if(glCantileverCrossSection==2):
        Iyz = 0.0
    if(glCantileverCrossSection==3):
        Iyz = 0.0

    ## Calculation of momentum M_y and M_z:
    M_z = (length-x_pos) * Py
    M_y = (length-x_pos) * Pz

    ## Calculation of sigma(x,y,z):
    #  Formula: sigma(x,y,z) = (N(x)/A) + (My*Iz-Mz*Iyz)/(Iy*Iz-Iyz**2)*z + (Mz*Iy-My*Iyz)/(Iy*Iz-Iyz**2)*y
    sigma = M_z/Iz*y_pos + M_y/Iy*z_pos

    return sigma

reason: import numpy


text:
import csv
import os
import pickle
import random
from collections import defaultdict

import numpy as np
import torch
import torchvision.transforms as transforms
from PIL import Image
from torch.utils import data
from torchvision.datasets.folder import default_loader
from tqdm import tqdm

from masker import compute_item_mask
from models import VLBertTokenizer
from utils import collate_fn

normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                 std=[0.229, 0.224, 0.225])

train_transform = lambda s: transforms.Compose(
    [transforms.Resize(s), transforms.RandomCrop(s), transforms.ToTensor(), normalize])
test_transform = lambda s: transforms.Compose(
    [transforms.Resize(s), transforms.CenterCrop(s), transforms.ToTensor(), normalize])

class EpicKitchens(data.Dataset):
    def __init__(self, split, img_root, annotation_root, bbox_size, max_img_seq_len, max_txt_seq_len,
                 bbox_transform=None, include_whole_img=False, tokenizer=None, **kwargs):
        self.tokenizer = tokenizer or VLBertTokenizer.from_pretrained('bert-base-uncased')
        self.max_img_seq_len = max_img_seq_len
        self.max_txt_seq_len = max_txt_seq_len
        self.include_whole_img = include_whole_img
        self.split = split
        self.img_root = img_root
        self.annotation_root = annotation_root
        self.bbox_transform = bbox_transform(bbox_size) or transforms.Lambda(lambda x: x)
        self.actions = self.process_actions(
            torch.load(os.path.join(annotation_root, 'processed_EPIC_train_action_labels.pth'))['actions'])
        with open(os.path.join(annotation_root, 'EPIC_train_object_labels.csv')) as f:
            self.objects = self.process_objects(csv.DictReader(f))
        with open(os.path.join(annotation_root, 'EPIC_video_info.csv')) as f:
            self.video_info = self.process_video_info(csv.DictReader(f))
        split_data = torch.load(os.path.join(annotation_root, 'splits.pth'))
        self.split_data = split_data
        vocab = split_data['vocab']
        self.vocab = vocab
        if split == 'train':
            split_data = split_data['train_action_uids']
        elif split == 'val':
            split_data = split_data['test_action_uids']
        self.action_uids = set(e for r in split_data for c in r for e in c)
        self.sel_action_uids = set(
            e for i, r in enumerate(split_data) for j, c in enumerate(r) for e in c if
            i in vocab['sel_verb_idxs'] and j in vocab['sel_noun_idxs'])
        self.frames = self.get_frames()

    def get_frames(self):
        frames = []
        for vid_id, d in self.actions.items():
            for act_id, row in enumerate(d):
                if int(row['uid']) in self.action_uids:
                    i = int(row['start_frame'])
                    j = int(row['stop_frame'])
                    frames.extend([(vid_id, act_id, n) for n in range(i, j) if self.objects[vid_id][n]])
        return frames

    def get_raw_image(self, index, bbox=False):
        vid_id, act_id, frame_id = self.frames[index]
        participant_id = vid_id.split('_')[0]
        img_path = os.path.join(self.img_root, participant_id, vid_id, f'frame_{frame_id:010d}.jpg')
        # To use high resolution images, the videos need to be downloaded first
        img = default_loader(img_path)  # this loads a smaller version of the image
        if bbox:
            img_bboxes = []
            objects = self.objects[vid_id][frame_id]
            orig_w, orig_h = self.video_info[vid_id]['res']
            img_w, img_h = img.size
            for obj in objects:
                for t, l, h, w in obj['bbox']:
                    h_scale = img_h / orig_h
                    w_scale = img_w / orig_w
                    t *= h_scale
                    h *= h_scale
                    l *= w_scale
                    w *= w_scale
                    if h < 10 or w < 10: continue  # too thin or narrow? do not add bbox
                    bbox = [int(l), int(t), int(l + w), int(t + h)]
                    img_bboxes.append(img.crop(bbox))
            return img, img_bboxes
        return img

    @staticmethod
    def process_video_info(d):
        ret = {}
        for row in d:
            ret[row['video']] = {'res': list(map(int, row['resolution'].split('x'))), 'fps': row['fps'],
                                 'dur': float(row['duration'])}
        return ret

    @staticmethod
    def process_actions(d):
        ret = defaultdict(list)
        for row in d:
            ret[row['video_id']].append(row)
        return ret

    @staticmethod
    def process_objects(d):
        ret = defaultdict(lambda: defaultdict(list))
        for row in d:
            obj = row
            obj['bbox'] = obj['bounding_boxes']
            if obj['bbox']: obj['bbox'] = eval(obj['bbox'])
            if len(obj['bbox']):
                ret[row['video_id']][int(row['frame'])].append(obj)
        return ret

    def __len__(self):
        return len(self.frames)

    def __getitem__(self, i, pad=True, add_special_toks=True):
        vid_id, act_id, frame_id = self.frames[i]
        action = self.actions[vid_id][act_id]
        objects = self.objects[vid_id][frame_id]
        participant_id = vid_id.split('_')[0]

        # Text
        text_tokens = self.tokenizer.tokenize(action['narration'])
        if add_special_toks:
            text_tokens = self.tokenizer.add_special_tokens_vl(text_tokens)
        if pad:
            text_tokens, text_len = self.pad_text(text_tokens)
        else:
            text_tokens = text_tokens[:self.max_txt_seq_len]
            text_len = len(text_tokens)
        text_token_ids = torch.LongTensor(self.tokenizer.convert_tokens_to_ids(text_tokens))

        # Images
        img_path = os.path.join(self.img_root, participant_id, vid_id, f'frame_{frame_id:010d}.jpg')
        img = default_loader(img_path)
        imgs = []
        img_bboxes = []
        img_nouns = []
        if self.include_whole_img and self.max_img_seq_len:
            imgs.append(img)
            img_nouns.append('whole image')
            img_bboxes.append((0, 0, 100, 100))
        orig_w, orig_h = self.video_info[vid_id]['res']
        img_w, img_h = img.size
        for obj in objects:
            if len(imgs) >= self.max_img_seq_len:
                break
            for t, l, h, w in obj['bbox']:
                h_scale = img_h / orig_h
                w_scale = img_w / orig_w
                t *= h_scale
                h *= h_scale
                l *= w_scale
                w *= w_scale
                if h < 10 or w < 10: continue  # too thin or narrow? do not add bbox
                bbox = (l, t, l + w, t + h)
                rel_bbox = (l / img_w, t / img_h, (l + w) / img_w, (t + h) / img_h)
                rel_bbox = tuple(min(100, round(100 * _)) for _ in rel_bbox)
                img_bboxes.append(rel_bbox)
                img_nouns.append(obj['noun'])
                imgs.append(img.crop(bbox))

        img_len = len(imgs)
        if img_len == 0:  # no candidate bboxes apply, create padding
            imgs.append(Image.fromarray(np.zeros((10, 10, 3)).astype(np.uint8)))
            img_bboxes = [(0, 0, 0, 0)]

        img_sequence = torch.stack([self.bbox_transform(img) for img in imgs]) if len(imgs) > 0 else torch.tensor([])
        img_bboxes = torch.LongTensor(img_bboxes)

        if img_len == 0 and not pad:
            img_sequence = img_sequence[:0]
            img_bboxes = img_bboxes[:0]

        if pad:
            img_sequence = self.pad_imgs(img_sequence)
            img_bboxes = self.pad_imgs(img_bboxes)

        return {'imgs': img_sequence, 'text': text_token_ids, 'text_tokens': '/'.join(text_tokens),
                'img_nouns': img_nouns, 'img_bboxes': img_bboxes, 'imgs_len': img_len, 'text_len': text_len,
                'vid_ids': vid_id, 'frame_ids': frame_id, 'act_ids': act_id, 'indices': i}

    def pad_text(self, txt, l=None):
        """
        :param txt: list of text tokens
        """
        l = l or self.max_txt_seq_len
        txt = txt[:l]
        return (txt + [self.tokenizer.pad_token for i in range(l - len(txt))]), len(txt)

    def pad_imgs(self, imgs):
        ret = torch.zeros((self.max_img_seq_len, *tuple(imgs.shape[1:]))).to(dtype=imgs.dtype, device=imgs.device)
        k = min(len(imgs), self.max_img_seq_len)
        ret[:k] = imgs[:k]
        return ret

class EpicKitchensMultiple(EpicKitchens):
    """
    Return multiple related examples in the same sample.

    EpicKitchensMultiple outputs (for each element):

    - `imgs`: tensor(N x 3 x H x W), where N is the total number of images for all the sequences
    - `text`: tensor(T). Contains the IDs of the text for all the sequences, concatenated, and without any special token
    - `text_tokens`: List of length S (number of sequences). Each element contains the text of a sequence, represented
    by the (text) tokens separated by `/`
    - `img_nouns`: List (size S) of lists. Each sublist contains words representing the nouns in the different bboxes of
    the sequence,
    - `img_bboxes`: tensor(N x 4). Bounding boxes for each of the N images
    - `imgs_len`: tensor(S). Number of images for each sequence. Its sum adds up to N
    - `text_len`: tensor(S). Number of tokens for each sequence. Its sum adds up to T
    - `vid_ids`: List (size S).
    - `frame_ids`: List (size S).
    - `act_ids`: List (size S).
    - `indices`: List (size S).
    - `seq_type`: tensor(S). For each sequence, indicates if it is the target (0), a positive (1) of a negative (-1)
    - `target_token_ids`: tensor(M). List of the tokens (represented by token ID, not position) that have to be masked.
    They are determined according to the mode and masking policy. Note that in `text` they are still NOT masked
    - `num_seqs`: integer representing S,
    - `num_tgt_toks`: integer representing M,
    - `tok_groups`: List of length L where L is the number of words to be predicted. For each word, `tok_group` contains
    the number of tokens that represent that word. The sum of all the values in `tok_groups` is M. These tokens are in
    order in `target_token_ids`, so if `tok_groups` is [2, 3], it means that `target_token_ids` has 5 elements, the
    first 2 represent one word, and the last 3 another word. `tok_groups` is only used in testing, when
    self.test_masking_policy is specified and not `random`. During training it is [].
    - `tok_group_labels`: List of length L, where each element contains a string mentioning the part of speech of the
    corresponding target word. During training it is [].
    """

    def __init__(self, split, img_root, annotation_root, bbox_size, max_img_seq_len, max_txt_seq_len,
                 bbox_transform=None, include_whole_img=False, only_verb_noun=False, tokenizer=None, max_negatives=0,
                 max_positives=0, min_negatives=0, min_positives=0, test_masking_policy=None, negs_per_pos=1, **kwargs):
        super().__init__(split, img_root, annotation_root, bbox_size, max_img_seq_len, max_txt_seq_len,
                         bbox_transform=bbox_transform, include_whole_img=include_whole_img,
                         only_verb_noun=only_verb_noun, tokenizer=tokenizer, **kwargs)

        self.min_negatives = min_negatives
        self.min_positives = min_positives
        self.max_negatives = max_negatives
        self.max_positives = max_positives
        self.negs_per_pos = negs_per_pos
        self.test_masking_policy = test_masking_policy

        self.frame_words = defaultdict(list)

        frame_words_path = os.path.join(annotation_root, f'{self.split}_frame_words.pth')
        try:
            self.frame_words = torch.load(frame_words_path)
        except:
            for i, (vid_id, act_id, frame_id) in tqdm(enumerate(self.frames), total=len(self.frames),
                                                      desc=f'Creating frame words for dataset {self.split} split'):
                narration = self.actions[vid_id][act_id]['narration']
                token_ids = self.tokenizer.encode(narration)
                for token_id in token_ids:
                    self.frame_words[token_id].append((i, vid_id, act_id))
            torch.save(self.frame_words, frame_words_path)

    def __getitem__(self, i):
        data0 = super().__getitem__(i, pad=False, add_special_toks=False)
        data0['seq_type'] = 0

        data = [data0]
        vid_id = data0['vid_ids']

        eval_mask = self.test_masking_policy is not None and self.test_masking_policy != 'random'

        if eval_mask:
            vid_id, act_id, frame_id = self.frames[data0['indices']]
            item_p_mask, pos_list = compute_item_mask(self.actions[vid_id][act_id], data0['text'],
                                                      self.split_data, self.test_masking_policy,
                                                      self.tokenizer)

            tok_groups = []
            tok_group_labels = []
            tok_starts_ends = []
            try:
                for pos, l, grp in pos_list:
                    tok_groups.append(l)
                    tok_group_labels.append(grp)
                    tok_starts_ends.append((pos, pos + l))
                tgt_token_ids = list(set(data0['text'][item_p_mask.bool()].tolist()))
                n_neg = len(tgt_token_ids) * self.negs_per_pos  # farm as many negatives as positives
            except:
                n_neg = 0
                tgt_token_ids = []

        else:
            n_pos = random.randint(self.min_positives, self.max_positives)
            n_neg = random.randint(self.min_negatives, self.max_negatives)

            tgt_token_ids = set(
                filter(lambda t: t not in self.tokenizer.convert_tokens_to_ids(self.tokenizer.all_special_tokens),
                       data0['text'].tolist()))
            tgt_token_ids = random.sample(tgt_token_ids, n_pos) if len(tgt_token_ids) > n_pos else tgt_token_ids

        pos_txt_ids = set()
        cnt_positives = 0
        for idx, token_id in enumerate(tgt_token_ids):
            if token_id in pos_txt_ids:
                continue
            if cnt_positives >= self.max_positives:
                # tgt_token_ids[idx] = 0
                break
            # select positive
            candidate_positives = self.frame_words[token_id]
            if not len(candidate_positives):
                tgt_token_ids[idx] = 0
                continue
            # remove all frames belonging to the same video
            candidate_positives_same = [i for i, _, _ in filter(lambda x: x[1] == vid_id, candidate_positives)]
            candidate_positives_different = [i for i, _, _ in filter(lambda x: x[1] != vid_id, candidate_positives)]
            if len(candidate_positives_different) > 0:
                j = random.choice(candidate_positives_different)
            else:
                # only if that word does not exist in any other example, use element from the same action as positive
                j = random.choice(candidate_positives_same)
            data_pos = super().__getitem__(j, pad=False, add_special_toks=False)
            if data_pos['imgs'] is None:
                tgt_token_ids[idx] = 0
                continue
            data_pos['seq_type'] = 1
            data.append(data_pos)
            pos_txt_ids.update(data_pos['text'].tolist())
            cnt_positives += 1

        n_neg = min(n_neg, self.max_negatives)
        n_neg = max(n_neg, self.min_negatives)
        while n_neg:
            j = random.randint(0, self.__len__() - 1)
            data_neg = super().__getitem__(j, pad=False, add_special_toks=False)
            if data_neg['imgs'] is None or set(data_neg['text'].tolist()) & set(tgt_token_ids):
                continue
            data_neg['seq_type'] = -1
            data.append(data_neg)
            n_neg -= 1

        random.shuffle(data)

        collated_data = collate_fn(data, cat_tensors=True)
        collated_data['target_token_ids'] = torch.LongTensor(list(tgt_token_ids))
        collated_data['num_seqs'] = len(data)
        collated_data['num_tgt_toks'] = len(tgt_token_ids)
        if eval_mask:
            collated_data['tok_groups'] = tok_groups
            collated_data['tok_group_labels'] = tok_group_labels
        else:
            collated_data['tok_groups'] = []
            collated_data['tok_group_labels'] = []
        return collated_data

max_stars_repo_name: cvlab-columbia/expert

max_issues_repo_name: xujinglin/expert

max_forks_repo_name: xujinglin/expert


text:
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import dgl
from dgl.nn import GraphConv
from dgl.nn import GATConv

import torch.autograd.function as Function

class Encoder(nn.Module):
  def __init__(self, seq_len, n_features, embedding_dim):
    super(Encoder, self).__init__()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    self.device = device
    self.seq_len, self.n_features = seq_len, n_features
    self.rnn1_dim, self.rnn2_dim = embedding_dim, 2 * embedding_dim
    self.dense1_dim, self.embedding_dim = 2 * embedding_dim, embedding_dim
    self.rnn1 = nn.LSTM(
      input_size=self.n_features,
      hidden_size=self.rnn1_dim,
      num_layers=1,
      #dropout=0.4,
      batch_first=True  # (batch, seq, feature) as output and (num_layers, batch, hidden_size) for h, c
    )  # (batch, seq, feature)  for input if batch_first is true
    self.rnn2 = nn.LSTM(
      input_size=self.rnn1_dim,
      hidden_size=self.dense1_dim,
      num_layers=1,
      #dropout=0.4,
      batch_first=True
    )
    self.dense1 = nn.Linear(
        in_features=self.dense1_dim,
        out_features=self.embedding_dim
    )
    # self.batchnorm = nn.BatchNorm1d(self.embedding_dim)

  def forward(self, x):
    # x.to(self.device)
    x = x.float()
    x, (_, _) = self.rnn1(x)
    x = F.leaky_relu(x)
    _, (hidden_n, _) = self.rnn2(x)
    h_size = hidden_n.shape
    x = hidden_n.reshape(h_size[1], h_size[2])
    x = F.leaky_relu(x)
    x = self.dense1(x)
    # x = self.batchnorm(x) #some models have this some not

    return torch.tanh(x)

class Decoder(nn.Module):
  def __init__(self, seq_len, n_features, input_dim):
      super(Decoder, self).__init__()
      device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
      self.device = device
      self.seq_len, self.input_dim, self.n_features = seq_len, input_dim, n_features
      self.dense1_dim, self.rnn1_dim, self.rnn2_dim = self.seq_len * self.input_dim * 2, 2 * input_dim, input_dim

      self.dense1 = nn.Linear(
          in_features=self.input_dim,
          out_features=self.dense1_dim
      )
      self.rnn1 = nn.LSTM(
          input_size=self.rnn1_dim,
          hidden_size=self.rnn2_dim,
          num_layers=1,
          batch_first=True
      )
      self.rnn2 = nn.LSTM(
          input_size=self.rnn2_dim,
          hidden_size=self.n_features,
          num_layers=1,
          batch_first=True
      )

  def forward(self, x):
      # x.to(self.device)
      x = x.float()
      x = self.dense1(x)
      x = F.leaky_relu(x)
      size = x.shape
      x = x.reshape((size[0], self.seq_len, self.rnn1_dim))
      x, (_, _) = self.rnn1(x)
      x = F.leaky_relu(x)
      x, (_, _) = self.rnn2(x)
      return F.leaky_relu(x)

class RecurrentAutoencoder(nn.Module):
  def __init__(self, seq_len, n_features, embedding_dim):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    self.device = device
    super(RecurrentAutoencoder, self).__init__()
    self.encoder = Encoder(seq_len, n_features, embedding_dim).to(device)
    self.encoder.float()
    self.decoder = Decoder(seq_len, n_features, embedding_dim).to(device)
    self.decoder.float()

  def forward(self, x):
    # x = x.to(self.device)
    x = x.float()
    x = self.encoder(x)
    x = self.decoder(x)
    return x

class GEC(nn.Module):
    def __init__(self, in_feats, h1_feats, last_space_feature):
        super(GEC, self).__init__()
        # self.conv1 = GATConv(in_feats, h1_feats, 1)
        self.conv1 = GraphConv(in_feats, h1_feats)
        self.conv2 = GraphConv(h1_feats, last_space_feature)
        # self.conv2 = GATConv(h1_feats, last_space_feature, 1)

    def forward(self, g, in_feat):
        h = self.conv1(g, in_feat)
        h = F.leaky_relu(h)
        h = self.conv2(g, h)
        h = F.leaky_relu(h)
        # h = self.conv3(g, h)
        g.ndata['h'] = h
        return dgl.mean_nodes(g, 'h')

class EncGraph(nn.Module):
  def __init__(self, encoder, graph, pmu_number, src, dst):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    self.device = device
    super(EncGraph, self).__init__()

    self.encoder = encoder
    self.graph = graph.to(self.device)
    self.pmu_number = pmu_number
    self.src = src
    self.dst = dst

  def forward(self, x):
    # x = x.to(self.device)
    x = x.float()
    y = np.split(x, self.pmu_number, axis=-1)
    z = [self.encoder(i) for i in y] #all latent z for each pmu in a batch
    size = z[0].shape
    #TODO
    #define the graph to pass it to the self.graph
    # z = torch.cat([x.float() for x in z], dim=0).reshape(self.pmu_number, size[0], size[1]).to(self.device)
    z = torch.cat([x.float() for x in z], dim=0).reshape(self.pmu_number, size[0], size[1])
    # print(z.shape)
    x = []
    for idx in range(size[0]):
        g = dgl.graph((self.src, self.dst), num_nodes=self.pmu_number)
        g = dgl.add_self_loop(g)
        g = g.to(self.device)
        g.ndata['features'] = z[:, idx].clone()
        x.append(g)
    # print(len(x))
    x = dgl.batch(x)
    # print(x)

    x = self.graph(x, x.ndata['features'])
    xs = x.shape
    x = x.reshape(xs[0], xs[-1]).clone()

    return x

class LatentMu(torch.autograd.Function):
    @staticmethod
    def forward(ctx, z, mu):
        # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        # ctx.device = device
        z = z.float()
        mu = mu.float()
        ctx.alpha = 1
        ctx.mu = mu
        ctx.save_for_backward(z, mu)

        def q_measure(ctx, z, mu):
            p = 1.0
            dist = torch.cdist(z, mu, 2).float()
            q = 1.0 / (1 + dist ** 2 / ctx.alpha)
            q = (q.T / q.sum(axis=1)).T
            # print('q: ',q[0])
            return q

        def aux_dist(ctx, z, mu):
            q = q_measure(ctx, z, mu)
            q = (q.T / q.sum(axis=1)).T
            p = (q ** 2)
            p = (p.T / p.sum(axis=1)).T

            return p

        def loss_fun(ctx, z):
            # q = q_measure(ctx, z, mu).to(ctx.device)
            q = q_measure(ctx, z, mu)
            # p = aux_dist(ctx, z, mu).to(ctx.device)
            p = aux_dist(ctx, z, mu)
            return F.kl_div(p, q, reduction='batchmean')

        # print(loss_fun(ctx, z))
        return loss_fun(ctx, z)

    @staticmethod
    def backward(ctx, grad_output):
        z, mu = ctx.saved_tensors
        def q_measure(ctx, z, mu):
            p = 1.0
            dist = torch.cdist(z, mu, 2).float()
            q = 1.0 / (1 + dist ** 2 / ctx.alpha)#TODO think about other measures as I removed 1+
            q = (q.T / q.sum(axis=1)).T
            # print('q: ', q[0])
            return q

        def aux_dist(ctx, z, mu):
            z = z
            q = q_measure(ctx, z, mu)
            q = (q.T / q.sum(axis=1)).T
            p = (q ** 2)
            p = (p.T / p.sum(axis=1)).T
            # print('p: ', p[0])
            return p

        def grad(ctx, z, mu):
            # z = z.float().to(ctx.device)
            z = z.float()
            # mu = mu.float().to(ctx.device)
            mu = mu.float()
            q = q_measure(ctx, z, mu)
            p = aux_dist(ctx, z, mu)
            d = 1 + torch.cdist(z, mu, 2) ** 2
            constant_val = (p - q) / d

            constant_val_T = constant_val.T

            extented_z = z.repeat((mu.shape[0], 1))
            extented_mu = torch.repeat_interleave(mu, z.shape[0], dim=0)
            simple_dist_z_mu = (extented_z - extented_mu).reshape(mu.shape[0], z.shape[0], z.shape[-1])

            constant_val_T_rep = constant_val_T.repeat_interleave(z.shape[-1], axis=1)
            constant_val_T_rep = constant_val_T_rep.reshape(mu.shape[0], z.shape[0], z.shape[-1])

            all_grads = 2 * constant_val_T_rep * simple_dist_z_mu

            # z = z
            # q = q_measure(ctx, z, mu)
            # p = aux_dist(ctx, z, mu)
            # extented_z = z.repeat((mu.shape[0],1))
            # extented_mu = torch.repeat_interleave(mu, z.shape[0], dim = 0)
            # grad = 2 * (p - q) * torch.cdist(z, mu, 1) / (1 + torch.cdist(z, mu, 2)**2)
            # grad_z = torch.sum(grad, axis=1)
            # grad_mu = -torch.sum(grad, axis=0)

            return all_grads
        all_grads = grad(ctx, z, mu)
        # grad_z = torch.sum(all_grads, axis=0).to(ctx.device)
        grad_z = torch.sum(all_grads, axis=0)
        # grad_mu = -torch.sum(all_grads, axis=1).to(ctx.device)
        grad_mu = -torch.sum(all_grads, axis=1)
        # print(grad_z)
        # grad_z = grad_z.view(-1, 1).expand_as(z).to(ctx.device)
        # grad_mu = grad_mu.view(-1, 1).expand_as(mu).to(ctx.device)
        # print(grad_z)
        # grad_z = z.mul(grad_z) * grad_output
        # grad_mu = mu.mul(grad_mu) * grad_output
        # print(grad_output.shape)
        # print(grad_z.shape)
        # print(grad_mu.shape)
        return grad_z*grad_output, grad_mu*grad_output

# #%%
# #test latent
# z=torch.rand((3,2), requires_grad=True, device=device)
# m=torch.rand((2,2), requires_grad=True, device=device)
# ls=LatentMu.apply(z,m)
# ls.backward()

class EncoderFlex(nn.Module):
  def __init__(self, seq_len, n_features, rnn1_dim, rnn2_dim, embedding_dim):
    super(EncoderFlex, self).__init__()
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    self.device = device
    self.seq_len, self.n_features = seq_len, n_features
    self.rnn1_dim = rnn1_dim
    self.rnn2_dim = rnn2_dim
    self.dense1_dim = rnn2_dim
    self.embedding_dim = embedding_dim
    self.rnn1 = nn.LSTM(
      input_size=self.n_features,
      hidden_size=self.rnn1_dim,
      num_layers=1,
      #dropout=0.4,
      batch_first=True  # (batch, seq, feature) as output and (num_layers, batch, hidden_size) for h, c
    )  # (batch, seq, feature)  for input if batch_first is true
    self.rnn2 = nn.LSTM(
      input_size=self.rnn1_dim,
      hidden_size=self.dense1_dim,
      num_layers=1,
      #dropout=0.4,
      batch_first=True
    )
    self.dense1 = nn.Linear(
        in_features=self.dense1_dim,
        out_features=self.embedding_dim
    )
    # self.batchnorm = nn.BatchNorm1d(self.embedding_dim)

  def forward(self, x):
    # x.to(self.device)
    x = x.float()
    x, (_, _) = self.rnn1(x)
    x = F.leaky_relu(x)
    _, (hidden_n, _) = self.rnn2(x)
    h_size = hidden_n.shape
    x = hidden_n.reshape(h_size[1], h_size[2])
    x = F.leaky_relu(x)
    x = self.dense1(x)
    # x = self.batchnorm(x) #some models have this some not

    return F.leaky_relu(x)

class DecoderFlex(nn.Module):
  def __init__(self, seq_len, n_features, rnn1_dim, rnn2_dim,  embedding_dim):
      super(DecoderFlex, self).__init__()
      device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
      self.device = device
      self.seq_len, self.n_features = seq_len, n_features
      self.rnn1_dim = rnn1_dim
      self.rnn2_dim = rnn2_dim
      self.dense1_dim = rnn1_dim * self.seq_len
      self.embedding_dim = embedding_dim

      self.dense1 = nn.Linear(
          in_features=self.embedding_dim,
          out_features=self.dense1_dim
      )
      self.rnn1 = nn.LSTM(
          input_size=self.rnn1_dim,
          hidden_size=self.rnn2_dim,
          num_layers=1,
          batch_first=True
      )
      self.rnn2 = nn.LSTM(
          input_size=self.rnn2_dim,
          hidden_size=self.n_features,
          num_layers=1,
          batch_first=True
      )

  def forward(self, x):
      # x.to(self.device)
      x = x.float()
      x = self.dense1(x)
      x = F.leaky_relu(x)
      size = x.shape
      # print(size)
      x = x.reshape((size[0], self.seq_len, self.rnn1_dim))
      # print(x.shape)
      x, (_, _) = self.rnn1(x)
      x = F.leaky_relu(x)
      x, (_, _) = self.rnn2(x)
      return F.leaky_relu(x)

class RecurrentAEDFlex(nn.Module):
  def __init__(self, seq_len, n_features, rnn1_dim, rnn2_dim, embedding_dim):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    self.device = device
    super(RecurrentAEDFlex, self).__init__()
    self.encoder = EncoderFlex(seq_len, n_features, rnn1_dim, rnn2_dim, embedding_dim).to(device)
    self.encoder.float()
    self.decoder = DecoderFlex(seq_len, n_features,  rnn2_dim, rnn1_dim, embedding_dim).to(device)
    self.decoder.float()

  def forward(self, x):
    # x = x.to(self.device)
    x = x.float()
    x = self.encoder(x)
    x = self.decoder(x)
    return x

max_stars_repo_name: arminalgln/graphpmu

max_issues_repo_name: arminalgln/graphpmu

max_forks_repo_name: arminalgln/graphpmu


text:
# -*- coding: utf-8 -*-

import pandas as pd
from datetime import datetime
import numpy as np

from DataDictObject import DataDictObject
from TablonReader import TablonReader
from TablonEncoder import TablonEncoder
from TablonLoader import TablonLoaderDB

class DataManagementObject:
    """This is the class which contains the whole parser pipeline in order to
    transform between the different possible states of the data.
    These states are:
        * RawData: this format is identified by a string which is the path
        and the filename of the raw data.
        * Tablon: this format is the data parsed and each column formatted
        to one of the possible accepted formats of a common DB.
        It is characterized by a pandas dataframe.
        * DataFrame: this format contains special types of objects which can
        describe better the essence of the data and allow to be analize in
        special ways. It is characterized by a pandas dataframe.
        * DB: this is the server database. It is characterized by the
        direction of the database.
    """

    def __init__(self, client='', client_code='', pathdata='',
                 date_compilationdata='', delimiter='', typefile=''):
        """This object is recommended to be initialized which as information as
        we can give to it.
        """
        # common for the client. It is supposed not to change
        self.client = client
        self.client_code = client_code

        # for the same client we can use different files. This could change.
        self.pathdata = pathdata
        self.date_compilationdata = date_compilationdata
        self.date_treatmentdata = datetime.now()
        self.delimiter = delimiter

        # Initialization of the central object of the data process.
        #It will save all the information.
        self.datatransformation = DataProcessCenter()

    # The script only requires this one.
    def add_datadictobject(self, dataprocessobject):
        self.datatransformation = dataprocessobject

    # Collection of functions to set the class DataProcessCenter at this level.
    def add_parser(self, parser):
        self.datatransformation.parser = parser

    def add_encoder(self, encoder):
        self.datatransformation.encoder = encoder

    def add_format(self, format):
        self.datatransformation.format = format

    def add_loader(self, loader):
        self.datatransformation.loader = loader

    def add_preexploratory(self, preexploratory):
        self.datatransformation.preexploratory = preexploratory

    # Collection of functions for specify information of the
    def add_parser_parserdict(self, parserdict):
        self.datatransformation.parser.add_transformationdict(parserdict)

    def add_parser_expansiondict(self, expansiondict):
        self.datatransformation.parser.add_expansiondict(expansiondict)

    def add_parser_filteringdict(self, filteringdict):
        self.datatransformation.parser.add_filteringdict(filteringdict)

    def add_encode_encoderdict(self, encoderdict):
        self.datatransformation.encoder.add_transformationdict(encoderdict)

    def add_encode_expansiondict(self, expansiondict):
        self.datatransformation.encoder.add_expansiondict(expansiondict)

    def add_encode_filteringdict(self, filteringdict):
        self.datatransformation.encoder.add_filteringdict(filteringdict)

    def add_format_formatdict(self, formatdict):
        self.datatransformation.format.add_transformationdict(formatdict)

    def add_format_expansiondict(self, expansiondict):
        self.datatransformation.format.add_expansiondict(expansiondict)

    def add_format_filtering(self, filteringdict):
        self.datatransformation.format.add_filteringdict(filteringdict)

    def add_loader_loaderdict(self, loaderdict):
        self.datatransformation.loader.add_transformationdict(loaderdict)

    def add_loader_expansiondict(self, expansiondict):
        self.datatransformation.loader.add_expansiondict(expansiondict)

    def add_loader_filtering(self, filteringdict):
        self.datatransformation.loader.add_filteringdict(filteringdict)

    # calling to self.datatransformation
    # TODO: it has to be implemented the inputs of the data.
    def parse(self, filepath):
        self.datatransformation.parse(filepath)

#   def encode(self,tablon):
#   def format(self,tablon):
#   def load(self,tablon):

    def apply_pipeline(self, pipelineselection, startdataframe=''):
        """Run all the process of the data for the determine processes in the
        pipelineselection list or numpy array ({0,1} or boolean).
        """
        self.datatransformation.apply_pipeline(pipelineselection,
                                               startdataframe)

    # TODO: probably has to be done outside
    ################
    def save_Manager(self, filename_manager, path_manager=''):
        """With this function we are able to save the manager and all the
        information stored in order to redo the process of the data.
        """
        pass

    def load_Manager(self, filename_manager, path_manager=''):
        """With this function we are able to load the manager and all the
        information stored in order to redo the process with other data.
        It returns an object which we can change the main parameters but we
        keep the parser, format, codification and loading configuration.
        """
        pass

    #pipeline?
    def show(self):
        """TODO:"""
        return

class DataProcessCenter:
    """This class is oriented to agglomerate all the processes related with
    the dictionaries and settings of the data treatment.
    It works as an interface between the general pipeline class
    (DataManagementObject) and the class related to the basic treatment
    information of the data (DataDictObject).
    """
    # TODO: generalize self.input, self.process, self.output
    # TODO: exploratory and preexploratory in TablonDescriptor

    # TODO: probablente mejor un dictionary como input
    def __init__(self, client):
        """The initialization of the DataDictCente. It initializes all the the
        subprocesses.
        """
        # Read parameter file.
        ## That file has to retrieve:
        # key
        # filename
        # path_data
        # baseurl
        execfile("Scripts/" + client + "_parameters.py")
        compulsoryvars = {'filename': '', 'path_data': 'Data/', 'key': '',
                          'baseurl': ''}
        for e in compulsoryvars:
            if not e in locals():
                exec("global " + e)
                exec(e + ' = ' + '"' + compulsoryvars[e] + '"')
        self.filename = filename
        self.path_data = path_data

        # Initialize operators
        self.parser = TablonReader(DataDictObject('parser'))
        self.encoder = TablonEncoder(DataDictObject('encoder'))
        self.loader = TablonLoaderDB(DataDictObject('loader'), key, baseurl)
        self.preexploratory = DataDictObject('preexploratory')
        self.exploratory = DataDictObject('exploratory')

        # Initialize pipeline
        self.pipeline = [self.parser, self.preexploratory, self.encoder,
                         self.exploratory, self.loader]
        self.pipelineselection = ['parser', 'preexploratory', 'encoder',
                                  'loader', 'exploratory']

        # The processes that could generate an input to this system
        self.inputters = ['parser', 'downloader']

        # The processes that generate a tablon
        self.tabloners = ['encoder'] + self.inputters

        # The other parematers
        self.typeoutput = ''  # tablon, formattablon,analyticaltablon, others
        self.defaultpipe = ['parser', 'preexploratory', 'encoder', 'loader',
                            'exploratory']

    def apply_pipeline(self, pipelineselection, startdataframe=''):
        """This function apply a selection of tasks indicated in the numpy
        {0,1} or logical vector, or list of tasks indicated in order.
        The list of tasks could produce error because of the selection of an
        incorrect order.
            The possible tasks we could select are:
                * parser: only requires the file with structured data in order
                to
                * format: only requires the tablon data.
                This is obtained if there is a parser before that.
                * encoder: only requires the tablon data.
                This is obtained if there is a parser before that.
                * loader: only requires the tablon data.
                This is obtained if there is a parser before that.
                * preexploratory: only requires the tablon data.
                This is obtained if there is a parser before that.
                * exploratory: requires a parser and a encoder before this
                function.

        """

        ## 1. Creation of the pipelineselection
        ''' Reglas:
                * First an inputter.
                * Only of each type.
        '''
        # most general form of representation of the pipeline is the list,
        # because allow to the user to define an order.
        # We will transform the other ways of expressing the pipeline to this
        #one.

        # Initial setting
        default_pipeline = ['encoder', 'loader', 'exploratory']
        boolstartdf = True
        if not startdataframe:
            boolstartdf = False
            default_pipeline = ['parser', 'encoder', 'loader', 'exploratory']

        # Check and transform to list
        if type(pipelineselection) == list:
            # Only the first as inputter or any depend on the input.
            if boolstartdf:
                ind_inputters_incorrect =\
                    [i for i, x in enumerate(pipelineselection)
                     if x in self.inputters]
            else:
                ind_inputters_incorrect =\
                    [i for i, x in enumerate(pipelineselection)
                     if x in self.inputters and i != 0]

            pipelineselection = list(np.delete(pipelineselection,
                                               ind_inputters_incorrect))

            # Delete the repeated elements:
            pipelineorder = []
            for e in pipelineselection:
                if e not in pipelineorder:
                    pipelineorder.append(e)
                #occurrencies = [operation for operation in pipelineselection
                #                if operation == e]
                #if len(occurrencies) > 1:

            #TODO: Not only one of each type.
            #TODO: Delete consecutive encoders
            #TODO: Delete consecutive descriptions:
            pipelineselection = pipelineorder

        else:
            message = "WARNING: The pipeline given it is incorrect. "
            message += "It will be done all the tasks in the default order"
            print(message)
            pipelineselection = default_pipeline
            # Now we have a list with processes in order.

        self.pipelineselection = pipelineselection

        ## 2. Apply in this order.
        if boolstartdf:
            output = startdataframe
        else:
            output = []

        for operatoname in pipelineselection:
            if operatoname in self.tabloners:
                output = self.apply_operator(operatoname, output)
            else:
                self.apply_operator(operatoname, output)
        return output

    ################################ Processes ################################
    ###########################################################################
    def apply_operator(self, operatorname, dataframe):
        # TOFINISH
        if operatorname == 'parser':
            # check if parser the object
            if type(dataframe) == tuple:
                dataframe = dataframe[0]
            return self.parser.parse(self.filename, self.path_data)
        elif operatorname == 'format':
            # check if format the object
            if type(dataframe) == tuple:
                dataframe = dataframe[0]
            return self.format.format()
        elif operatorname == 'encoder':
            # check if encoder the object
            if type(dataframe) == tuple:
                dataframe = dataframe[0]
            return self.encoder.encode(dataframe)
        elif operatorname == 'loader':
            # check if loader the object
            if type(dataframe) == tuple:
                dataframe = pd.concat(dataframe, axis=1)
            self.loader.load(dataframe)
            return
        #TODO:
        elif operatorname == 'preexploratory':
            # check if preexploratory the object
            return
        elif operatorname == 'exploratory':
            # check if exploratory the object
            return
        elif operatorname == 'general':
            # check if exploratory the object
            # raise Exception
            pass
        else:
            # raise Exception
            pass

    def parse(self, filename, pathfile):
        return self.parser.parse(filename, pathfile)

    def format(self):
        return self.format.format()

    def encode(self):
        return self.encoder.encode()

    def load(self, input_):
        return self.loader.load(input_)

    def preexplore(self):
        return self.preexploratory.explore()

    def explore(self):
        return self.exploratory.explore()

    ############################# Setting objects #############################
    ###########################################################################
    def set_TablonReader(self, TablonReader):
        self.parser = TablonReader

    def set_TablonFormater(self, TablonFormater):
        self.format = TablonFormater

    def set_TablonEncoder(self, TablonEncoder):
        self.encoder = TablonEncoder

    def set_TablonLoader(self, TablonLoader):
        self.loader = TablonLoader

    def set_TablonMakers(self, tablondicts):
        for e in tablondicts:
            if e == 'TablonReader' or e == 'parser':
                self.parser = tablondicts[e]
            elif e == 'TablonFormater' or e == 'formater' or e == 'format':
                self.format = tablondicts[e]
            elif e == 'TablonEncoder' or e == 'encoder':
                self.encoder = tablondicts[e]
            elif e == 'TablonLoader' or e == 'loader':
                self.loader = tablondicts[e]
            else:
                message = "The only options of names to set are: TablonReader,"
                message += " parser, TablonFormater, formater, format, "
                message += "TablonEncoder, encoder, TablonLoader or loader"
                print(message)

    ################################# TOFINISH ################################
    ###########################################################################
    def set_datadictobject(self, datadictobject):
        # TOFINISH
        filename, pathfile = self.filename, self.pathfile
        if datadictobject.processtype == 'parser':
            # check if parser the object
            self.parser = TablonReader(filename, pathfile, object)

        elif datadictobject.processtype == 'format':
            # check if format the object
            self.format = TablonFormater(filename, pathfile, object)

        elif datadictobject.processtype == 'encoder':
            # check if encoder the object
            self.encoder = TablonEncoder(filename, pathfile, object)

        elif datadictobject.processtype == 'loader':
            # check if loader the object
            self.loader = TablonLoaderDB(filename, pathfile, object)

        #TODO:
        elif name == 'preexploratory':
            # check if preexploratory the object
            pass

        elif name == 'exploratory':
            # check if exploratory the object
            pass

        elif name == 'general':
            # check if exploratory the object
            # raise Exception
            pass

        else:
            # raise Exception
            pass


text:
"""Line sorting."""
import numpy as np

from .reduction import brightness

def multisort(image, params):
    """Sort an image by performing multiple transforms sequentially."""
    for x in params:
        image = sort(image, **x)

    return image

def sort(image, heuristic, reduction=brightness, num_rotations=0, copy=True):
    """Sort an image by some heuristic."""
    num_rotations = num_rotations % 4
    if copy:
        image = np.copy(image)
    if num_rotations:
        image = np.rot90(image, k=num_rotations)

    reduced = reduction(image)
    padding = np.zeros((reduced.shape[0], 1)) + heuristic(0)
    m = heuristic(np.concatenate([padding, reduced, padding], axis=1))

    d = np.diff(m.astype(np.int))
    s = np.where(d > 0)
    e = np.where(d < 0)

    for line, start, length in zip(s[0], s[1], e[1] - s[1]):
        end = start + length
        segment = image[line, start:end]
        sort = np.argsort(reduced[line, start:end])
        segment[:] = segment[sort]

    if num_rotations:
        image = np.rot90(image, k=4 - num_rotations)

    return image

max_stars_repo_name: stephantul/orst

max_issues_repo_name: stephantul/orst

max_forks_repo_name: stephantul/orst


text:
# https://github.com/NVIDIA/TensorRT/blob/master/quickstart/SemanticSegmentation/tutorial-runtime.ipynb

# Import required modules

import numpy as np
import os
import pycuda.driver as cuda
import pycuda.autoinit
import tensorrt as trt

import matplotlib.pyplot as plt
from PIL import Image

TRT_LOGGER = trt.Logger()

# Filenames of TensorRT plan file and input/output images.
engine_file = "fcn-resnet101.engine"
input_file  = "input.ppm"
output_file = "output.ppm"

#Utilities for input / output processing

# For torchvision models, input images are loaded in to a range of [0, 1] and
# normalized using mean = [0.485, 0.456, 0.406] and stddev = [0.229, 0.224, 0.225].
def preprocess(image):
    # Mean normalization
    mean = np.array([0.485, 0.456, 0.406]).astype('float32')
    stddev = np.array([0.229, 0.224, 0.225]).astype('float32')
    data = (np.asarray(image).astype('float32') / float(255.0) - mean) / stddev
    # Switch from HWC to to CHW order
    return np.moveaxis(data, 2, 0)

def postprocess(data):
    num_classes = 21
    # create a color palette, selecting a color for each class
    palette = np.array([2 ** 25 - 1, 2 ** 15 - 1, 2 ** 21 - 1])
    colors = np.array([palette*i%255 for i in range(num_classes)]).astype("uint8")
    # plot the segmentation predictions for 21 classes in different colors
    img = Image.fromarray(data.astype('uint8'), mode='P')
    img.putpalette(colors)
    return img

# Load TensorRT engine
def load_engine(engine_file_path):
    assert os.path.exists(engine_file_path)
    print("Reading engine from file {}".format(engine_file_path))
    with open(engine_file_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
        return runtime.deserialize_cuda_engine(f.read())

# Inference pipeline

def infer(engine, input_file, output_file):
    print("Reading input image from file {}".format(input_file))
    with Image.open(input_file) as img:
        input_image = preprocess(img)
        image_width = img.width
        image_height = img.height

    with engine.create_execution_context() as context:
        # Set input shape based on image dimensions for inference
        context.set_binding_shape(engine.get_binding_index("input"), (1, 3, image_height, image_width))
        # Allocate host and device buffers
        bindings = []
        for binding in engine:
            binding_idx = engine.get_binding_index(binding)
            size = trt.volume(context.get_binding_shape(binding_idx))
            dtype = trt.nptype(engine.get_binding_dtype(binding))
            if engine.binding_is_input(binding):
                input_buffer = np.ascontiguousarray(input_image)
                input_memory = cuda.mem_alloc(input_image.nbytes)
                bindings.append(int(input_memory))
            else:
                output_buffer = cuda.pagelocked_empty(size, dtype)
                output_memory = cuda.mem_alloc(output_buffer.nbytes)
                bindings.append(int(output_memory))

        stream = cuda.Stream()
        # Transfer input data to the GPU.
        cuda.memcpy_htod_async(input_memory, input_buffer, stream)
        # Run inference
        context.execute_async_v2(bindings=bindings, stream_handle=stream.handle)
        # Transfer prediction output from the GPU.
        cuda.memcpy_dtoh_async(output_buffer, output_memory, stream)
        # Synchronize the stream
        stream.synchronize()

    with postprocess(np.reshape(output_buffer, (image_height, image_width))) as img:
        print("Writing output image to file {}".format(output_file))
        img.convert('RGB').save(output_file, "PPM")

# plot output image
# plt.imshow(Image.open(input_file))
# print(Image.open(input_file))
# plt.show()
# print(Image.open(output_file))

# run infer
print("Running TensorRT inference for FCN-ResNet101")
with load_engine(engine_file) as engine:
    infer(engine, input_file, output_file)

# # segment output
# plt.imshow(Image.open(output_file))

# 2nd infer with diff img size (cat 388x386)
import requests
from io import BytesIO

output_image="cat_input.ppm"

# Read sample image input and save it in ppm format
print("Exporting ppm image {}".format(output_image))
# response = requests.get("https://github.com/huukim911/onnx-exp/blob/main/cat.jpg")
with Image.open("cat.jpg") as img:
    ppm = Image.new("RGB", img.size, (255, 255, 255))
    ppm.paste(img, mask=img.split()[2])
    ppm.save(output_image)

# run infer 2nd
input_file  = "cat_input.ppm"
output_file = "cat_output.ppm"
print("Running TensorRT inference for FCN-ResNet101 2nd time")
with load_engine(engine_file) as engine:
    infer(engine, input_file, output_file)

reason: import numpy,import pycuda