def loadBeamModel(beamModelFileName, displayInfo=False):
    import pandas as pd
    if displayInfo:
        print('# Loading beam model from {:s}'.format(beamModelFileName), end=' ')
    try:
        beamModel=pd.read_csv(beamModelFileName, delim_whitespace=True, index_col='Energy')
        if {'Energy_MeV','dEnergy_pr','scalingFactor','FWHMX_cm','alphaX','betaX_cm','epsilonX_cm','FWHMY_cm','alphaY','betaY_cm','epsilonY_cm'}.issubset(beamModel.columns):
            if displayInfo:
                print('...OK')
            return beamModel
        else:
            print('\nError while loading beam model from \'{:s}\'. Missing columns or wrong column names.'.format(beamModelFileName))
            exit(1)
    except:
        print('\nError while loading beam model from \'{:s}\'. No such file.'.format(beamModelFileName))
        exit(1)

def loadDicomRT(dicomRTFileName, displayInfo=False):
    import pydicom as dicom
    if displayInfo:
        print('# Loading RT plan from {:s}'.format(dicomRTFileName), end=' ')
    try:
        dicomRT=dicom.read_file(dicomRTFileName)
        if dicomRT.SOPClassUID=='1.2.840.10008.5.1.4.1.1.481.8':
            if displayInfo:
                print('...OK')
            return dicomRT
        else:
            print('\nError while loading RT plan from {:s}. The file does not contain RT plan.'.format(dicomRTFileName))
            exit(1)
    except:
        print('\nError while loading RT plan from {:s}. No such file.'.format(dicomRTFileName))
        exit(1)

def interpolateBeamModel(beamModel, Energy, method='slinear'): # do not use methods 'linear'
    import pandas as pd
    beamModelEnergy=beamModel.reindex(beamModel.index.values.tolist()+[Energy]).sort_index().interpolate(method=method).loc[Energy]
    if isinstance(beamModelEnergy,pd.DataFrame):
        beamModelEnergy=beamModelEnergy.iloc[0]
    return beamModelEnergy

def buildPLAN(RTplanFileName, beamModelFileName, RSModelFileName, materialDefinitionFileName, beamModelInterpolationMethod='slinear', displayInfo=False):
    import sys, os
    import pandas as pd
    import pydicom as dicom
    import numpy as np
    from shutil import copyfile

    dicomRT=loadDicomRT(RTplanFileName, displayInfo=displayInfo)
    planInfo={'beamModelFileName': beamModelFileName,
              'RSModelFileName': RSModelFileName,
              'materialDefinitionFileName': materialDefinitionFileName,
              'RTplanFileName': RTplanFileName}

    # get field number and types
    NumberOfFields = dicomRT.FractionGroupSequence[0].NumberOfBeams
    NumberOfFieldsTreatment=0
    NumberOfFieldsSetup=0
    NumberOfFieldsOther=0
    for ifield in range(NumberOfFields):
        if dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType=='TREATMENT':
            NumberOfFieldsTreatment+=1
            planInfo['TreatmentMachineName']=dicomRT.IonBeamSequence[ifield].TreatmentMachineName
        elif dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType=='SETUP':
            NumberOfFieldsSetup+=1
        else:
            NumberOfFieldsOther+=1
    # get number of fractions
    NumberOfFractionsPlanned = dicomRT.FractionGroupSequence[0].NumberOfFractionsPlanned

    # load beam model
    beamModel=loadBeamModel(beamModelFileName, displayInfo=displayInfo)

    if displayInfo:
        print('# Building plan:')
        print('# Nominal gantry: {:s}'.format(planInfo['TreatmentMachineName']))
        print('# Number of fields: {:d} treatment, {:d} setup,  {:d} other'.format(NumberOfFieldsTreatment, NumberOfFieldsSetup, NumberOfFieldsOther))
        print('# Number of fraction planned: {:d}'.format(NumberOfFractionsPlanned))

    planInfo['pencilBeamNumberAll']=0
    planInfo['protonNumberAll']=0
    planInfo['planTotalEnergy']=0
    planInfo['fieldIDList']=[]
    planInfo['fieldNameList']=[]

    fieldsInfo=[]
    for ifield in range(NumberOfFields):
        # convert if is a TREATMENT field (not SETUP or OTHER)
        if dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType!='TREATMENT':
            continue;

        if displayInfo:
            print('Field {:d} / {:d}:'.format(ifield+1,NumberOfFieldsTreatment))

        IonBeamSequence = dicomRT.IonBeamSequence[ifield]

        # check if RadiationType is 'PROTON'
        if IonBeamSequence.RadiationType != 'PROTON':
            print('\t\tWarning: No PROTON plan. Field skipped.')
            npbVec.append(0)
            continue

        # check if the field is MODULATED which means that some properties are the same for all pb
        if IonBeamSequence.ScanMode != 'MODULATED' or IonBeamSequence.BeamType != 'STATIC':
            print('Error this RTPLAN is not for Raster Scan')
            continue;

        # check cumulative MU
        for ReferencedBeamSequence in dicomRT.FractionGroupSequence[0].ReferencedBeamSequence:
            if ReferencedBeamSequence.ReferencedBeamNumber==IonBeamSequence.BeamNumber:
                BeamMeterset=ReferencedBeamSequence.BeamMeterset
                break;
            else:
                BeamMeterset=np.nan
        if np.isnan(BeamMeterset):
            print('Error: cannot find BeamMeterset for beam number ID {:d}'.format(IonBeamSequence.BeamNumber))
            exit(1)

        # check presence of Range Shifter and Range Shifter ID
        if IonBeamSequence.NumberOfRangeShifters==1:
            if 'RangeShifterSequence' in IonBeamSequence:
                RangeShifterID=IonBeamSequence.RangeShifterSequence[0].RangeShifterID
        else:
            RangeShifterID=''

        # check distance of scanning magnets to isocentre
        for LateralSpreadingDeviceSettingsSequence in IonBeamSequence.IonControlPointSequence[0].LateralSpreadingDeviceSettingsSequence:
            for LateralSpreadingDeviceSequence in IonBeamSequence.LateralSpreadingDeviceSequence:
                if LateralSpreadingDeviceSettingsSequence.ReferencedLateralSpreadingDeviceNumber==LateralSpreadingDeviceSequence.LateralSpreadingDeviceNumber:
                    if LateralSpreadingDeviceSequence.LateralSpreadingDeviceID=='MagnetX':
                        MagnetX_mm=LateralSpreadingDeviceSettingsSequence.IsocenterToLateralSpreadingDeviceDistance
                    elif LateralSpreadingDeviceSequence.LateralSpreadingDeviceID=='MagnetY':
                        MagnetY_mm=LateralSpreadingDeviceSettingsSequence.IsocenterToLateralSpreadingDeviceDistance
        if MagnetX_mm<MagnetY_mm:
            print('Error: script assumes that the first scanning magnet is deflecting along X (MagnetX_mm={:.3f}, MagnetY_mm={:.3f},)'.format(MagnetX_mm,MagnetY_mm))
            exit(1)
        # define distance from field origin to isocentre
        fieldOriginToIsoDist_mm=MagnetY_mm

        # collect field information
        fieldInfo={'BeamNumber': int(IonBeamSequence.BeamNumber),  # check beam number ID (field ID)
                   'BeamName': IonBeamSequence.BeamName,
                   'RadiationType': IonBeamSequence.RadiationType, # type of radiation (PROTON)
                   'BeamType': IonBeamSequence.BeamType,
                   'ScanMode': IonBeamSequence.ScanMode,
                   'RangeShifterID': RangeShifterID,
                   'NumberOfControlPoints': int(IonBeamSequence.NumberOfControlPoints), # number of control points (2 x energy slice number) in field
                   'energySliceNumber': int(IonBeamSequence.NumberOfControlPoints/2), # number of energy slices in field
                   'MUsum': float(BeamMeterset), # cumulative MU for field
                   'WeightSum': float(IonBeamSequence.FinalCumulativeMetersetWeight), # cumulative weights for field (the sum of spot weights is used instead)
                   'GantryAngle_deg': float(IonBeamSequence.IonControlPointSequence[0].GantryAngle),
                   'CouchAngle_deg': float(IonBeamSequence.IonControlPointSequence[0].PatientSupportAngle),
                   'fieldOriginToIsoDist_mm': fieldOriginToIsoDist_mm, # distance of field origin point from Iso
                   'SnoutPosition': 368.5, # value taken from TPS eclipse (SnoutPosition 430 means 368.5 mm distance from Iso to RS surface) #IonBeamSequence.IonControlPointSequence[0].SnoutPosition, # snaut position
                   'IsocenterPosition_mm': np.array(IonBeamSequence.IonControlPointSequence[0].IsocenterPosition), # isocentre position
                   'VaccumExitToISODist_mm': 1680
                  }

        if displayInfo:
            print('\tBeam Type: {:s}'.format(fieldInfo['RadiationType']))
            print('\tBeam number ID = {:d}'.format(fieldInfo['BeamNumber']))
            print('\tBeam name = \'{:s}\''.format(fieldInfo['BeamName']))
            print('\tNumber of energy  = {:d}'.format(fieldInfo['energySliceNumber']))
            print('\tCumulative MU = {:.4f}'.format(fieldInfo['MUsum']))
            print('\tRange Shifter = {:s}'.format('None' if not fieldInfo['RangeShifterID'] else fieldInfo['RangeShifterID']))
            print('\tGantry Angle = {:.2f}°'.format(fieldInfo['GantryAngle_deg']))
            print('\tCouch Angle = {:.2f}°'.format(fieldInfo['CouchAngle_deg']))
            print('\tField Origin To Iso Dist = {:.2f} mm'.format(fieldInfo['fieldOriginToIsoDist_mm']))
            print('\tSnout position = {:.2f} mm'.format(fieldInfo['SnoutPosition']))
            print('\tIso position Vec. =              [ {:.2f} {:.2f} {:.2f}] mm'.format(fieldInfo['IsocenterPosition_mm'][0],fieldInfo['IsocenterPosition_mm'][1],fieldInfo['IsocenterPosition_mm'][2]))

        fieldInfo['slicesInfo']=[]
        for IonControlPointSequence in IonBeamSequence.IonControlPointSequence:
            if np.array(IonControlPointSequence.ScanSpotMetersetWeights).sum()==0:
                continue
            NumberOfScanSpotPositions=int(IonControlPointSequence.NumberOfScanSpotPositions)
            beamModelEnergy=interpolateBeamModel(beamModel, float(IonControlPointSequence.NominalBeamEnergy), method=beamModelInterpolationMethod)

            sliceInfo={ 'NominalBeamEnergy': float(IonControlPointSequence.NominalBeamEnergy),
                        'NumberOfPaintings': int(IonControlPointSequence.NumberOfPaintings),
                        'FRED_beamEnergy': beamModelEnergy.Energy_MeV,
                        'FRED_pSpread': beamModelEnergy.dEnergy_pr,
                        'FRED_epsilonX_cm': beamModelEnergy.epsilonX_cm,
                        'FRED_epsilonY_cm': beamModelEnergy.epsilonY_cm,
                        'FRED_alphaX': beamModelEnergy.alphaX,
                        'FRED_alphaY': beamModelEnergy.alphaY,
                        'FRED_betaX_cm': beamModelEnergy.betaX_cm,
                        'FRED_betaY_cm': beamModelEnergy.betaY_cm,
                        'FRED_scalingFactor': beamModelEnergy.scalingFactor,
                        'ScanSpotTuneID': float(IonControlPointSequence.ScanSpotTuneID)}

            # calculate spots parameters
            sliceSpots=pd.DataFrame({'ScanSpotWeight': IonControlPointSequence.ScanSpotMetersetWeights,
                                     'ScanSpotPositionX_mm': IonControlPointSequence.ScanSpotPositionMap[0::2],
                                     'ScanSpotPositionY_mm': IonControlPointSequence.ScanSpotPositionMap[1::2]})
            sliceSpots['ScanSpotMU'] = sliceSpots.ScanSpotWeight/fieldInfo['WeightSum']*fieldInfo['MUsum']
            sliceSpots['FRED_protonNumber'] = sliceSpots.ScanSpotMU*sliceInfo['FRED_scalingFactor']*sliceInfo['NumberOfPaintings']
            sliceSpots['FRED_energyDelivered'] = sliceSpots['FRED_protonNumber']*sliceInfo['FRED_beamEnergy']
            sliceInfo['PencilBeams']=len(sliceSpots.index)

            P_mm=np.zeros((3,sliceInfo['PencilBeams']))
            V_mm=np.array([-sliceSpots.ScanSpotPositionX_mm, sliceSpots.ScanSpotPositionY_mm, np.repeat(fieldInfo['fieldOriginToIsoDist_mm'],sliceInfo['PencilBeams'])])-P_mm
            V=V_mm/np.linalg.norm(V_mm, axis=0)
            sliceSpots['FRED_Px']=P_mm[0]/10
            sliceSpots['FRED_Py']=P_mm[1]/10
            sliceSpots['FRED_Pz']=P_mm[2]/10
            sliceSpots['FRED_Vx']=V[0]
            sliceSpots['FRED_Vy']=V[1]
            sliceSpots['FRED_Vz']=V[2]

            sliceInfo['sliceSpots']=sliceSpots

            fieldInfo['slicesInfo'].append(sliceInfo)

        fieldsInfo.append(fieldInfo)

        fieldInfo['PencilBeams']=0
        fieldInfo['PencilBeamsProtonNumber']=0
        fieldInfo['energyDelivered']=0
        for sliceInfo in fieldInfo['slicesInfo']:
            fieldInfo['PencilBeams']+=sliceInfo['PencilBeams']
            fieldInfo['PencilBeamsProtonNumber']+=sliceInfo['sliceSpots'].FRED_protonNumber.sum()
            fieldInfo['energyDelivered']=sliceInfo['sliceSpots'].FRED_energyDelivered.sum()

        planInfo['pencilBeamNumberAll']+=fieldInfo['PencilBeams']
        planInfo['protonNumberAll']+=fieldInfo['PencilBeamsProtonNumber']
        planInfo['planTotalEnergy']+=fieldInfo['energyDelivered']
        planInfo['fieldIDList'].append(fieldInfo['BeamNumber'])
        planInfo['fieldNameList'].append(fieldInfo['BeamName'])

        if displayInfo:
            print('\tPencil Beams loaded = {:d}'.format(fieldInfo['PencilBeams']))
    if displayInfo:
        print('# Total pencil beams loaded in the plan = {:d}'.format(planInfo['pencilBeamNumberAll']))
        print('# Total energy delivered by the plan = {:.20E}'.format(planInfo['planTotalEnergy']))

    return planInfo, fieldsInfo

def writeRtplan(rtplanfileName, planInfo, fieldsInfo, getPlanVersion='', fieldToSave=None, displayInfo=True):
    import sys, os, glob
    import pandas as pd
    import pydicom as dicom
    import numpy as np
    from shutil import copyfile
    Xsec='emittance'

    rtplanfileName=os.path.abspath(rtplanfileName)

    # remove old beam model file, RS model file and material definition file from simulation folder
    if glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*beamModel.txt')):
        os.remove(glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*beamModel.txt'))[0])
    if glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*RSModel.txt')):
        os.remove(glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*RSModel.txt'))[0])
    if glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*materialDefinition.txt')):
        os.remove(glob.glob(os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'*materialDefinition.txt'))[0])

    # copy current beam model file, RS model file and material definition file to simulation folder
    copyfile(planInfo['beamModelFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],os.path.basename(planInfo['beamModelFileName'])))
    copyfile(planInfo['RSModelFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],os.path.basename(planInfo['RSModelFileName'])))
    copyfile(planInfo['RSModelFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'regions.inp'))
    copyfile(planInfo['materialDefinitionFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],os.path.basename(planInfo['materialDefinitionFileName'])))
    copyfile(planInfo['materialDefinitionFileName'], os.path.join(os.path.split(os.path.abspath(rtplanfileName))[0],'materials.inp'))
    rtplan_H=open(rtplanfileName,'w')

    # choose field to save or save all fields
    if not fieldToSave==None:
        if   isinstance(fieldToSave, int):
            if fieldToSave in planInfo['fieldIDList']:
                fieldToSaveIdx=planInfo['fieldIDList'].index(fieldToSave)
                print('# Requested to save the rtplan only for the field ID {:d}'.format(fieldToSave))
            else:
                print('# Error: cannot find field ID {:d} in the treatment plan. Available field IDs are '.format(fieldToSave)+str(planInfo['fieldIDList']))
                exit(-1)
        elif isinstance(fieldToSave, str):
            if fieldToSave in planInfo['fieldNameList']:
                fieldToSaveIdx=planInfo['fieldNameList'].index(fieldToSave)

                print('# Requested to save the rtplan only for the field name \'{:s}\''.format(fieldToSave))
            else:
                print('# Error: cannot find field name \'{:s}\' in the treatment plan. Available field names are '.format(fieldToSave)+str(planInfo['fieldNameList']))
                exit(-1)
        fieldsInfo=[fieldsInfo[fieldToSaveIdx]]

        # write information header
    print('###############################################################################################################################################', file=rtplan_H)
    print('###### Beam plan for {:s} machine prepared in getPlan v.{:s}'.format(planInfo['TreatmentMachineName'], getPlanVersion), file=rtplan_H)
    print('###### DICOM plan {:s}'.format(planInfo['RTplanFileName']), file=rtplan_H)
    print('###### numPB {:d}, totNumParticles {:.20E}'.format(planInfo['pencilBeamNumberAll'],planInfo['protonNumberAll']), file=rtplan_H)
    print('###############################################################################################################################################', file=rtplan_H)

    # write field for G0
    for fieldInfo in fieldsInfo:
        print('field: {:d} ; '.format(fieldInfo['BeamNumber']), file=rtplan_H, end='')
        print('O=[ {:+.4f}, {:+.4f}, {:+.4f} ];\t'.format(0,-fieldInfo['fieldOriginToIsoDist_mm']/10,0), file=rtplan_H, end='')
        print('f=[ {:+.3f}, {:+.3f}, {:+.3f} ];\t'.format(0, +1, 0), file=rtplan_H, end='')
        print('u=[ {:+.3f}, {:+.3f}, {:+.3f} ];\t'.format(0, 0, +1), file=rtplan_H, end='')
        print('exitWindowPlane={:.4f};'.format((fieldInfo['fieldOriginToIsoDist_mm']-fieldInfo['VaccumExitToISODist_mm'])/10), file=rtplan_H)

    #write master for field
    for fieldInfo in fieldsInfo:
        print('pbmaster: {:d} ; '.format(fieldInfo['BeamNumber']), file=rtplan_H, end='')
        print('particle={:s};\t'.format(fieldInfo['RadiationType'].lower()), file=rtplan_H, end='')
        print('Xsec={:s};\t'.format(Xsec), file=rtplan_H, end='')
        print('emittanceRefPlaneDistance={:.3f};\t'.format(fieldInfo['fieldOriginToIsoDist_mm']/10), file=rtplan_H, end='')
        print('columns=[P.x,P.y,P.z,v.x,v.y,v.z,T,pSpread,N,twissAlphaX,twissBetaX,emittanceX,twissAlphaY,twissBetaY,emittanceY] ', file=rtplan_H)

    # group fields, prepare gantry region, save the default regions FoR
    print('group: fields {:s}'.format(' '.join('field_{:d}'.format(fieldInfo['BeamNumber'])  for fieldInfo in fieldsInfo)), file=rtplan_H)
    print('region: gantry ; O = [0,0,0] ; L = [1,1,1] ; lAdaptiveSize=t ; material = vacuum', file=rtplan_H)
    print('set_parent: gantry fields NozzleGroup', file=rtplan_H)
    print('save_regions: 0', file=rtplan_H)

    print('##################################################', file=rtplan_H)
    print('###### Start of setup and delivery sequence ######', file=rtplan_H)
    print('##################################################', file=rtplan_H)
    for fieldInfo in fieldsInfo:
        print('###### setup sequence for field {:d}'.format(fieldInfo['BeamNumber']), file=rtplan_H)

        print('# translate nozzle regions by snaut position', file=rtplan_H)
        print('transform: NozzleGroup shift_by {:.4f} {:.4f} {:.4f}'.format(0,-fieldInfo['SnoutPosition']/10,0), file=rtplan_H)

        print('# rotate field and NozzleGroup by gantry angle', file=rtplan_H)
        print('transform: gantry rotate {:s} {:.4f}'.format('z',fieldInfo['GantryAngle_deg']), file=rtplan_H)

        print('# translate and rotate phantom by iso position and couch rotation', file=rtplan_H)
        print('transform: phantom shift_by {:+.4f} {:+.4f} {:+.4f}'.format(-fieldInfo['IsocenterPosition_mm'][0]/10,-fieldInfo['IsocenterPosition_mm'][1]/10,-fieldInfo['IsocenterPosition_mm'][2]/10), file=rtplan_H)
        print('transform: phantom rotate {:s} {:+.4f}'.format('y',-fieldInfo['CouchAngle_deg']), file=rtplan_H)

        print('# delivery sequences for slices in field {:d}'.format(fieldInfo['BeamNumber']), file=rtplan_H)
        print('deactivate: fields, NozzleGroup', file=rtplan_H)
        print('activate: field_{:d} {:s}'.format(fieldInfo['BeamNumber'],fieldInfo['RangeShifterID']), file=rtplan_H)
        print('deliver: field_{:d}'.format(fieldInfo['BeamNumber']), file=rtplan_H)

        print('# reset field, nozzle regions and phantom to default FoR', file=rtplan_H)
        print('restore: 0', file=rtplan_H)

    print('##################################################', file=rtplan_H)
    print('###### End of setup and delivery sequence ########', file=rtplan_H)
    print('##################################################', file=rtplan_H)

    spotNumber=0
    for fieldInfo in fieldsInfo:
        for sliceInfo in fieldInfo['slicesInfo']:
            for _,sliceSpot in sliceInfo['sliceSpots'].iterrows():
                spotNumber+=1
                print('pb: {:d}\t{:d}\t'.format(spotNumber,fieldInfo['BeamNumber']), file=rtplan_H, end='')
                print('{:+.10f}\t{:+.10f}\t{:+.10f}\t'.format(sliceSpot['FRED_Px'],sliceSpot['FRED_Py'],sliceSpot['FRED_Pz']), file=rtplan_H, end='')
                print('{:+.10f}\t{:+.10f}\t{:+.10f}\t'.format(sliceSpot['FRED_Vx'],sliceSpot['FRED_Vy'],sliceSpot['FRED_Vz']), file=rtplan_H, end='')
                print('{:.3f}\t'.format(sliceInfo['FRED_beamEnergy']), file=rtplan_H, end='')
                print('{:.5E}\t'.format(sliceInfo['FRED_pSpread']), file=rtplan_H, end='')
                print('{:.10E}\t'.format(sliceSpot['FRED_protonNumber']), file=rtplan_H, end='')
                print('{:+.10E}\t{:+.10E}\t{:+.10E}\t'.format(sliceInfo['FRED_alphaX'],sliceInfo['FRED_betaX_cm'],sliceInfo['FRED_epsilonX_cm']), file=rtplan_H, end='')
                print('{:+.10E}\t{:+.10E}\t{:+.10E}\t'.format(sliceInfo['FRED_alphaY'],sliceInfo['FRED_betaY_cm'],sliceInfo['FRED_epsilonY_cm']), file=rtplan_H)
    if displayInfo:
        print('# RT plan saved to {:s}'.format(rtplanfileName))
    rtplan_H.close()

def getPlan(RTplanFileName, fieldToSave=None, FREDrtpalnFileName='current', defaultModelsFileName='default', beamModelInterpolationMethod='slinear', displayInfo=False):
    import os, re
    # version of getPlan
    version='3.4'
    if displayInfo:
        print('############################################################')
        print('# Importer of RTPLAN dicom for Fred at CCB v.{:s}'.format(version))
        print('# Implementation works for fred version 3.0.z2 or higher')
        print('############################################################')
    # get default beam model
    if defaultModelsFileName=='default':
        defaultModelsFileName='/home/shared/fred/beamModel/fred/defaultBeamModels/CCB_defaultModels.txt'
    else:
        if not os.path.isfile(defaultModelsFileName):
            print('Could not load file with default beam models (file {:s} does not exist)'.format(defaultModelsFileName))
            exit(1)

    # get gantry name
    dicomRT=loadDicomRT(RTplanFileName, displayInfo=False)
    for ifield in range(dicomRT.FractionGroupSequence[0].NumberOfBeams):
        if dicomRT.IonBeamSequence[ifield].TreatmentDeliveryType=='TREATMENT':
            TreatmentMachineName=dicomRT.IonBeamSequence[ifield].TreatmentMachineName
    if not TreatmentMachineName in ['GTR3','GTR4']:
        print('Room name \'{:s}\' not recognised'.format(TreatmentMachineName))
        exit(1)

    # get beam model, RS model and material definition file names
    with open(defaultModelsFileName) as f:
        for num, line in enumerate(f, 1):
            beamModelFileName_re = re.search('{:s}beamModelFileName\W+=\W+\'(\S+)\''.format(TreatmentMachineName), line)
            if beamModelFileName_re:
                beamModelFileName=beamModelFileName_re.group(1)
            RSModelFileName_re = re.search('{:s}RSModelFileName\W+=\W+\'(\S+)\''.format(TreatmentMachineName), line)
            if RSModelFileName_re:
                RSModelFileName=RSModelFileName_re.group(1)
            materialDefinitionFileName_re = re.search('{:s}materialDefinitionFileName\W+=\W+\'(\S+)\''.format(TreatmentMachineName), line)
            if materialDefinitionFileName_re:
                materialDefinitionFileName=materialDefinitionFileName_re.group(1)
    # build Plan
    planInfo, fieldsInfo=buildPLAN(RTplanFileName, beamModelFileName, RSModelFileName, materialDefinitionFileName, displayInfo=displayInfo)
    # find fred rtplan file name
    if FREDrtpalnFileName=='current':
        FREDrtpalnFileName=os.path.join(os.getcwd(),'rtplan.inp')
    # write plan
    writeRtplan(FREDrtpalnFileName, planInfo, fieldsInfo, fieldToSave=fieldToSave, getPlanVersion=version, displayInfo=displayInfo)


from django import template

from recipes.models import Favourite, Follow, Purchase, Tag

register = template.Library()

@register.simple_tag
def current_url(request):
    if request.resolver_match:
        current_url = request.resolver_match.view_name
        return current_url

@register.filter
def decline(word, amount):
    amount_by_ten = amount % 10
    amount_by_hundred = amount % 100
    if amount_by_ten == 1 and amount_by_hundred != 11:
        return word
    if amount_by_ten in [2, 3, 4]:
        if amount_by_hundred not in [11, 12, 13, 14]:
            return word + "а"
        else:
            return word + "ов"
    elif amount_by_ten in [5, 6, 7, 8, 9, 0]:
        return word + "ов"

@register.filter
def request_get(request, tag):
    tags = request.GET.getlist("tags")
    if tag in tags:
        tags = [item for item in tags if item != tag]
    else:
        tags.append(tag)
    new = request.GET.copy()
    new.setlist("tags", tags)
    return new.urlencode()

@register.filter
def get_tags(request):
    tags = request.GET.getlist("tags")
    if len(tags) == 3:
        tags.clear()
    return tags

@register.simple_tag
def is_favourite(recipe, user):
    return Favourite.objects.filter(user=user, recipe=recipe).exists()

@register.simple_tag
def purchases_amount(user):
    return Purchase.objects.filter(user=user).count()

@register.simple_tag
def is_purchased(recipe, user):
    return Purchase.objects.filter(user=user, recipe=recipe).exists()

@register.simple_tag
def is_followed(author, user):
    return Follow.objects.filter(user=user, author=author).exists()

@register.filter
def startswith(text, starts):
    if isinstance(text, str):
        return text.startswith(starts)
    return False

@register.filter
def tags_for_page(tags):

    tags = "".join([f"&tags={tag}" for tag in tags])
    return tags


import copy
import numpy as np
import pandas as pd
import torch
from torch import nn as nn

import torch.nn.functional as F
from torch_geometric.nn import MessagePassing
from torch_geometric.nn.inits import glorot
from torch_geometric.utils import softmax
from torch_sparse.tensor import SparseTensor
import pytorch_lightning as pl

class LATTE(nn.Module):
    def __init__(self, t_order: int, embedding_dim: int, in_channels_dict: dict, num_nodes_dict: dict, metapaths: list,
                 activation: str = "relu", attn_heads=1, attn_activation="sharpening", attn_dropout=0.5,
                 use_proximity=True, neg_sampling_ratio=2.0):
        super(LATTE, self).__init__()
        self.metapaths = metapaths
        self.node_types = list(num_nodes_dict.keys())
        self.embedding_dim = embedding_dim * t_order
        self.use_proximity = use_proximity
        self.t_order = t_order
        self.neg_sampling_ratio = neg_sampling_ratio

        layers = []
        t_order_metapaths = copy.deepcopy(metapaths)
        for t in range(t_order):
            layers.append(
                LATTEConv(embedding_dim=embedding_dim, in_channels_dict=in_channels_dict, num_nodes_dict=num_nodes_dict,
                          metapaths=t_order_metapaths, activation=activation, attn_heads=attn_heads,
                          attn_activation=attn_activation, attn_dropout=attn_dropout, use_proximity=use_proximity,
                          neg_sampling_ratio=neg_sampling_ratio,
                          first=True if t == 0 else False,
                          embeddings=layers[0].embeddings if t > 0 else None))
            t_order_metapaths = LATTE.join_metapaths(t_order_metapaths, metapaths)
        self.layers = nn.ModuleList(layers)

    @staticmethod
    def join_metapaths(metapath_A, metapath_B):
        metapaths = []
        for relation_a in metapath_A:
            for relation_b in metapath_B:
                if relation_a[-1] == relation_b[0]:
                    new_relation = relation_a + relation_b[1:]
                    metapaths.append(new_relation)
        return metapaths

    @staticmethod
    def get_edge_index_values(edge_index_tup: [tuple, torch.Tensor]):
        if isinstance(edge_index_tup, tuple):
            edge_index = edge_index_tup[0]
            edge_values = edge_index[1]

            if edge_values.dtype != torch.float:
                edge_values = edge_values.to(torch.float)
        elif isinstance(edge_index_tup, torch.Tensor) and edge_index_tup.size(1) > 0:
            edge_index = edge_index_tup
            edge_values = torch.ones_like(edge_index_tup[0], dtype=torch.float)
        else:
            return None, None

        return edge_index, edge_values

    @staticmethod
    def join_edge_indexes(edge_index_dict_A, edge_index_dict_B, global_node_idx):
        output_dict = {}
        for metapath_a, edge_index_a in edge_index_dict_A.items():
            if is_negative(metapath_a): continue
            edge_index_a, values_a = LATTE.get_edge_index_values(edge_index_a)
            if edge_index_a is None: continue

            for metapath_b, edge_index_b in edge_index_dict_B.items():
                if metapath_a[-1] != metapath_b[0] or is_negative(metapath_b): continue

                new_metapath = metapath_a + metapath_b[1:]
                edge_index_b, values_b = LATTE.get_edge_index_values(edge_index_b)
                if edge_index_b is None: continue
                try:
                    new_edge_index = adamic_adar(indexA=edge_index_a, valueA=values_a, indexB=edge_index_b,
                                                 valueB=values_b,
                                                 m=global_node_idx[metapath_a[0]].size(0),
                                                 k=global_node_idx[metapath_a[-1]].size(0),
                                                 n=global_node_idx[metapath_b[-1]].size(0),
                                                 coalesced=True, sampling=True)
                    if new_edge_index[0].size(1) == 0: continue
                    output_dict[new_metapath] = new_edge_index

                except Exception as e:
                    print(f"{str(e)} \n {metapath_a}: {edge_index_a.size(1)}, {metapath_b}: {edge_index_b.size(1)}")
                    continue

        return output_dict

    def forward(self, X: dict, edge_index_dict: dict, global_node_idx: dict, save_betas=False):
        """
        This
        :param X: Dict of <node_type>:<tensor size (batch_size, in_channels)>. If nodes are not attributed, then pass an empty dict.
        :param global_node_idx: Dict of <node_type>:<int tensor size (batch_size,)>
        :param edge_index_dict: Dict of <metapath>:<tensor size (2, num_edge_index)>
        :param save_betas: whether to save _beta values for batch
        :return embedding_output, proximity_loss, edge_pred_dict:
        """
        # device = global_node_idx[list(global_node_idx.keys())[0]].device
        proximity_loss = torch.tensor(0.0, device=self.layers[0].device) if self.use_proximity else None

        h_layers = {node_type: [] for node_type in global_node_idx}
        for t in range(self.t_order):
            if t == 0:
                h_dict, t_loss, edge_pred_dict = self.layers[t].forward(x_l=X, x_r=X,
                                                                        edge_index_dict=edge_index_dict,
                                                                        global_node_idx=global_node_idx,
                                                                        save_betas=save_betas)
                next_edge_index_dict = edge_index_dict
            else:
                next_edge_index_dict = LATTE.join_edge_indexes(next_edge_index_dict, edge_index_dict, global_node_idx)
                h_dict, t_loss, _ = self.layers[t].forward(x_l=h_dict, x_r=X,
                                                           edge_index_dict=next_edge_index_dict,
                                                           global_node_idx=global_node_idx,
                                                           save_betas=save_betas)

            for node_type in global_node_idx:
                h_layers[node_type].append(h_dict[node_type])

            if self.use_proximity:
                proximity_loss += t_loss

        concat_out = {node_type: torch.cat(h_list, dim=1) for node_type, h_list in h_layers.items() \
                      if len(h_list) > 0}

        return concat_out, proximity_loss, edge_pred_dict

    def get_attn_activation_weights(self, t):
        return dict(zip(self.layers[t].metapaths, self.layers[t].alpha_activation.detach().numpy().tolist()))

    def get_relation_weights(self, t):
        return self.layers[t].get_relation_weights()

class LATTEConv(MessagePassing, pl.LightningModule):
    def __init__(self, embedding_dim: int, in_channels_dict: {str: int}, num_nodes_dict: {str: int}, metapaths: list,
                 activation: str = "relu", attn_heads=4, attn_activation="sharpening", attn_dropout=0.2,
                 use_proximity=False, neg_sampling_ratio=1.0, first=True, embeddings=None) -> None:
        super(LATTEConv, self).__init__(aggr="add", flow="target_to_source", node_dim=0)
        self.first = first
        self.node_types = list(num_nodes_dict.keys())
        self.metapaths = list(metapaths)
        self.num_nodes_dict = num_nodes_dict
        self.embedding_dim = embedding_dim
        self.use_proximity = use_proximity
        self.neg_sampling_ratio = neg_sampling_ratio
        self.attn_heads = attn_heads
        self.attn_dropout = attn_dropout

        self.activation = activation.lower()
        if self.activation not in ["sigmoid", "tanh", "relu"]:
            print(f"Embedding activation arg `{self.activation}` did not match, so uses linear activation.")

        self.conv = torch.nn.ModuleDict(
            {node_type: torch.nn.Conv1d(
                in_channels=in_channels_dict[
                    node_type] if first and node_type in in_channels_dict else embedding_dim,
                out_channels=self.num_head_relations(node_type),
                kernel_size=1) \
                for node_type in self.node_types})  # W_phi.shape (D x F)

        if first:
            self.linear_l = nn.ModuleDict(
                {node_type: nn.Linear(in_channels, embedding_dim, bias=True) \
                 for node_type, in_channels in in_channels_dict.items()})  # W.shape (F x D_m)
            self.linear_r = nn.ModuleDict(
                {node_type: nn.Linear(in_channels, embedding_dim, bias=True) \
                 for node_type, in_channels in in_channels_dict.items()})  # W.shape (F x D_m)
        else:
            self.linear_l = nn.ModuleDict(
                {node_type: nn.Linear(embedding_dim, embedding_dim, bias=True) \
                 for node_type in self.node_types})  # W.shape (F x F)
            self.linear_r = nn.ModuleDict(
                {node_type: nn.Linear(in_channels, embedding_dim, bias=True) \
                 for node_type, in_channels in in_channels_dict.items()})  # W.shape (F x D_m}

        self.out_channels = self.embedding_dim // attn_heads
        self.attn_l = nn.ModuleList(
            [nn.Linear(embedding_dim, 1, bias=True) for metapath in self.metapaths])
        self.attn_r = nn.ModuleList(
            [nn.Linear(embedding_dim, 1, bias=True) for metapath in self.metapaths])
        # self.attn_q = nn.ModuleList(
        #     [nn.Sequential(nn.Tanh(), nn.Linear(2 * self.out_channels, 1, bias=False)) for metapath in self.metapaths])

        if attn_activation == "sharpening":
            self.alpha_activation = nn.Parameter(torch.Tensor(len(self.metapaths)).fill_(1.0))
        elif attn_activation == "PReLU":
            self.alpha_activation = nn.PReLU(init=0.2)
        elif attn_activation == "LeakyReLU":
            self.alpha_activation = nn.LeakyReLU(negative_slope=0.2)
        else:
            print(f"WARNING: alpha_activation `{attn_activation}` did not match, so used linear activation")
            self.alpha_activation = None

        # If some node type are not attributed, instantiate embeddings for them
        non_attr_node_types = (num_nodes_dict.keys() - in_channels_dict.keys())
        if first and len(non_attr_node_types) > 0:
            if embedding_dim > 256 or sum([v for k, v in self.num_nodes_dict.items()]) > 1000000:
                print("INFO: Embedding.device = 'cpu'")
                self.embeddings = {node_type: nn.Embedding(num_embeddings=self.num_nodes_dict[node_type],
                                                           embedding_dim=embedding_dim,
                                                           sparse=True).cpu() for node_type in non_attr_node_types}
            else:
                self.embeddings = nn.ModuleDict(
                    {node_type: nn.Embedding(num_embeddings=self.num_nodes_dict[node_type],
                                             embedding_dim=embedding_dim,
                                             sparse=False) for node_type in non_attr_node_types})
        elif embeddings is not None:
            self.embeddings = embeddings
        else:
            self.embeddings = None

        self.reset_parameters()

    def reset_parameters(self):
        for i, metapath in enumerate(self.metapaths):
            glorot(self.attn_l[i].weight)
            glorot(self.attn_r[i].weight)

        # glorot(self.attn_q[-1].weight)

        for node_type in self.linear_l:
            glorot(self.linear_l[node_type].weight)
        for node_type in self.linear_r:
            glorot(self.linear_r[node_type].weight)
        for node_type in self.conv:
            glorot(self.conv[node_type].weight)

        if self.embeddings is not None and len(self.embeddings.keys()) > 0:
            for node_type in self.embeddings:
                self.embeddings[node_type].reset_parameters()

    def forward(self, x_l, edge_index_dict, global_node_idx, x_r=None, save_betas=False):
        """

        :param x_l: a dict of node attributes indexed node_type
        :param global_node_idx: A dict of index values indexed by node_type in this mini-batch sampling
        :param edge_index_dict: Sparse adjacency matrices for each metapath relation. A dict of edge_index indexed by metapath
        :param x_r: Context embedding of the previous order, required for t >= 2. Default: None (if first order). A dict of (node_type: tensor)
        :return: output_emb, loss
        """
        # H_t = W_t * x
        l_dict = self.get_h_dict(x_l, global_node_idx, left_right="left")
        r_dict = self.get_h_dict(x_r, global_node_idx, left_right="right")

        # Predict relations attention coefficients
        beta = self.get_beta_weights(x_dict=x_l, h_dict=l_dict, h_prev=l_dict, global_node_idx=global_node_idx)
        # Save beta weights from testing samples
        if not self.training: self.save_relation_weights(beta, global_node_idx)

        # Compute node-level attention coefficients
        alpha_l, alpha_r = self.get_alphas(edge_index_dict, l_dict, r_dict)

        # For each metapath in a node_type, use GAT message passing to aggregate h_j neighbors
        out = {}
        for node_type in global_node_idx:
            out[node_type] = self.agg_relation_neighbors(node_type=node_type, alpha_l=alpha_l, alpha_r=alpha_r,
                                                         l_dict=l_dict, r_dict=r_dict, edge_index_dict=edge_index_dict,
                                                         global_node_idx=global_node_idx)
            out[node_type][:, -1] = l_dict[node_type]
            # Soft-select the relation-specific embeddings by a weighted average with beta[node_type]
            out[node_type] = torch.bmm(out[node_type].permute(0, 2, 1), beta[node_type]).squeeze(-1)

            # Apply \sigma activation to all embeddings
            out[node_type] = self.embedding_activation(out[node_type])

        proximity_loss, edge_pred_dict = None, None
        if self.use_proximity:
            proximity_loss, edge_pred_dict = self.proximity_loss(edge_index_dict,
                                                                 alpha_l=alpha_l, alpha_r=alpha_r,
                                                                 global_node_idx=global_node_idx)
        return out, proximity_loss, edge_pred_dict

    def agg_relation_neighbors(self, node_type, alpha_l, alpha_r, l_dict, r_dict, edge_index_dict, global_node_idx):
        # Initialize embeddings, size: (num_nodes, num_relations, embedding_dim)
        emb_relations = torch.zeros(
            size=(global_node_idx[node_type].size(0),
                  self.num_head_relations(node_type),
                  self.embedding_dim)).type_as(self.conv[node_type].weight)

        for i, metapath in enumerate(self.get_head_relations(node_type)):
            if metapath not in edge_index_dict or edge_index_dict[metapath] == None: continue
            head, tail = metapath[0], metapath[-1]
            num_node_head, num_node_tail = len(global_node_idx[head]), len(global_node_idx[tail])

            edge_index, values = LATTE.get_edge_index_values(edge_index_dict[metapath])
            if edge_index is None: continue
            # Propapate flows from target nodes to source nodes
            out = self.propagate(
                edge_index=edge_index,
                x=(r_dict[tail], l_dict[head]),
                alpha=(alpha_r[metapath], alpha_l[metapath]),
                size=(num_node_tail, num_node_head),
                metapath_idx=self.metapaths.index(metapath))
            emb_relations[:, i] = out

        return emb_relations

    def message(self, x_j, alpha_j, alpha_i, index, ptr, size_i, metapath_idx):
        alpha = alpha_j if alpha_i is None else alpha_j + alpha_i
        # alpha = self.attn_q[metapath_idx].forward(torch.cat([alpha_i, alpha_j], dim=1))
        alpha = self.attn_activation(alpha, metapath_idx)
        alpha = softmax(alpha, index=index, ptr=ptr, num_nodes=size_i)
        alpha = F.dropout(alpha, p=self.attn_dropout, training=self.training)
        return x_j * alpha

    def get_h_dict(self, input, global_node_idx, left_right="left"):
        h_dict = {}
        for node_type in global_node_idx:
            if node_type in input:
                if left_right == "left":
                    h_dict[node_type] = self.linear_l[node_type].forward(input[node_type])
                elif left_right == "right":
                    h_dict[node_type] = self.linear_r[node_type].forward(input[node_type])
            else:
                h_dict[node_type] = self.embeddings[node_type].weight[global_node_idx[node_type]] \
                    .to(self.conv[node_type].weight.device)
        return h_dict

    def get_alphas(self, edge_index_dict, l_dict, r_dict):
        alpha_l, alpha_r = {}, {}

        for i, metapath in enumerate(self.metapaths):
            if metapath not in edge_index_dict or edge_index_dict[metapath] is None:
                continue
            head_type, tail_type = metapath[0], metapath[-1]
            alpha_l[metapath] = self.attn_l[i].forward(l_dict[head_type])
            alpha_r[metapath] = self.attn_r[i].forward(r_dict[tail_type])
        return alpha_l, alpha_r

    def get_beta_weights(self, x_dict, h_dict, h_prev, global_node_idx):
        beta = {}
        for node_type in global_node_idx:
            # beta[node_type] = self.conv[node_type].forward(h_dict[node_type].unsqueeze(-1))
            if self.first:
                if node_type in x_dict:
                    beta[node_type] = self.conv[node_type].forward(x_dict[node_type].unsqueeze(-1))
                else:
                    # node_type is not attributed, use h_dict contains self.embeddings in first layer
                    beta[node_type] = self.conv[node_type].forward(h_dict[node_type].unsqueeze(-1))
            else:
                beta[node_type] = self.conv[node_type].forward(h_prev[node_type].unsqueeze(-1))

            beta[node_type] = torch.softmax(beta[node_type], dim=1)
        return beta

    def predict_scores(self, edge_index, alpha_l, alpha_r, metapath, logits=False):
        assert metapath in self.metapaths, f"If metapath `{metapath}` is tag_negative()'ed, then pass it with untag_negative()"

        # e_pred = self.attn_q[self.metapaths.index(metapath)].forward(
        #     torch.cat([alpha_l[metapath][edge_index[0]], alpha_r[metapath][edge_index[1]]], dim=1)).squeeze(-1)

        e_pred = self.attn_activation(alpha_l[metapath][edge_index[0]] + alpha_r[metapath][edge_index[1]],
                                      metapath_id=self.metapaths.index(metapath)).squeeze(-1)
        if logits:
            return e_pred
        else:
            return F.sigmoid(e_pred)

    def proximity_loss(self, edge_index_dict, alpha_l, alpha_r, global_node_idx):
        """
        For each relation/metapath type given in `edge_index_dict`, this function both predict link scores and computes
        the NCE loss for both positive and negative (sampled) links. For each relation type in `edge_index_dict`, if the
        negative metapath is not included, then the function automatically samples for random negative edges. And, if it
        is included, then computes the NCE loss over the given negative edges. This function returns the scores of the
        predicted positive and negative edges.

        :param edge_index_dict (dict): Dict of <relation/metapath>: <Tensor(2, num_edges)>
        :param alpha_l (dict): Dict of <node_type>:<alpha_l tensor>
        :param alpha_r (dict): Dict of <node_type>:<alpha_r tensor>
        :param global_node_idx (dict): Dict of <node_type>:<Tensor(node_idx,)>
        :return loss, edge_pred_dict: NCE loss. edge_pred_dict will contain both positive relations of shape (num_edges,) and negative relations of shape (num_edges*num_neg_edges, )
        """
        loss = torch.tensor(0.0, dtype=torch.float, device=self.conv[self.node_types[0]].weight.device)
        edge_pred_dict = {}
        for metapath, edge_index in edge_index_dict.items():
            # KL Divergence over observed positive edges or negative edges (if included)
            if isinstance(edge_index, tuple):  # Weighted edges
                edge_index, values = edge_index
            else:
                values = 1.0
            if edge_index is None: continue

            if not is_negative(metapath):
                e_pred_logits = self.predict_scores(edge_index, alpha_l, alpha_r, metapath, logits=True)
                loss += -torch.mean(values * F.logsigmoid(e_pred_logits), dim=-1)
            elif is_negative(metapath):
                e_pred_logits = self.predict_scores(edge_index, alpha_l, alpha_r, untag_negative(metapath), logits=True)
                loss += -torch.mean(F.logsigmoid(-e_pred_logits), dim=-1)

            edge_pred_dict[metapath] = F.sigmoid(e_pred_logits.detach())

            # Only need to sample for negative edges if negative metapath is not included
            if not is_negative(metapath) and tag_negative(metapath) not in edge_index_dict:
                neg_edge_index = negative_sample(edge_index,
                                                 M=global_node_idx[metapath[0]].size(0),
                                                 N=global_node_idx[metapath[-1]].size(0),
                                                 n_sample_per_edge=self.neg_sampling_ratio)
                if neg_edge_index is None or neg_edge_index.size(1) <= 1: continue

                e_neg_logits = self.predict_scores(neg_edge_index, alpha_l, alpha_r, metapath, logits=True)
                loss += -torch.mean(F.logsigmoid(-e_neg_logits), dim=-1)
                edge_pred_dict[tag_negative(metapath)] = F.sigmoid(e_neg_logits.detach())

        loss = torch.true_divide(loss, max(len(edge_index_dict) * 2, 1))
        return loss, edge_pred_dict

    def embedding_activation(self, embeddings):
        if self.activation == "sigmoid":
            return F.sigmoid(embeddings)
        elif self.activation == "tanh":
            return F.tanh(embeddings)
        elif self.activation == "relu":
            return F.relu(embeddings)
        else:
            return embeddings

    def attn_activation(self, alpha, metapath_id):
        if isinstance(self.alpha_activation, torch.Tensor):
            return self.alpha_activation[metapath_id] * alpha
        elif isinstance(self.alpha_activation, nn.Module):
            return self.alpha_activation.forward(alpha)
        else:
            return alpha

    def get_head_relations(self, head_node_type, to_str=False) -> list:
        relations = [".".join(metapath) if to_str and isinstance(metapath, tuple) else metapath for metapath in
                     self.metapaths if
                     metapath[0] == head_node_type]
        return relations

    def num_head_relations(self, node_type) -> int:
        """
        Return the number of metapaths with head node type equals to :param node_type: and plus one for none-selection.
        :param node_type (str):
        :return:
        """
        relations = self.get_head_relations(node_type)
        return len(relations) + 1

    def save_relation_weights(self, beta, global_node_idx):
        # Only save relation weights if beta has weights for all node_types in the global_node_idx batch
        if len(beta) < len(self.node_types): return

        self._betas = {}
        self._beta_avg = {}
        self._beta_std = {}
        for node_type in beta:
            with torch.no_grad():
                self._betas[node_type] = pd.DataFrame(beta[node_type].squeeze(-1).cpu().numpy(),
                                                      columns=self.get_head_relations(node_type, True) + [node_type, ],
                                                      index=global_node_idx[node_type].cpu().numpy())

                _beta_avg = np.around(beta[node_type].mean(dim=0).squeeze(-1).cpu().numpy(), decimals=3)
                _beta_std = np.around(beta[node_type].std(dim=0).squeeze(-1).cpu().numpy(), decimals=2)
                self._beta_avg[node_type] = {metapath: _beta_avg[i] for i, metapath in
                                             enumerate(self.get_head_relations(node_type, True) + [node_type])}
                self._beta_std[node_type] = {metapath: _beta_std[i] for i, metapath in
                                             enumerate(self.get_head_relations(node_type, True) + [node_type])}

    def save_attn_weights(self, node_type, attn_weights, node_idx):
        if not hasattr(self, "_betas"):
            self._betas = {}
        if not hasattr(self, "_beta_avg"):
            self._beta_avg = {}
        if not hasattr(self, "_beta_std"):
            self._beta_std = {}

        betas = attn_weights.sum(1)
        with torch.no_grad():
            self._betas[node_type] = pd.DataFrame(betas.cpu().numpy(),
                                                  columns=self.get_head_relations(node_type, True) + [node_type, ],
                                                  index=node_idx.cpu().numpy())

            _beta_avg = np.around(betas.mean(dim=0).cpu().numpy(), decimals=3)
            _beta_std = np.around(betas.std(dim=0).cpu().numpy(), decimals=2)
            self._beta_avg[node_type] = {metapath: _beta_avg[i] for i, metapath in
                                         enumerate(self.get_head_relations(node_type, True) + [node_type])}
            self._beta_std[node_type] = {metapath: _beta_std[i] for i, metapath in
                                         enumerate(self.get_head_relations(node_type, True) + [node_type])}

    def get_relation_weights(self):
        """
        Get the mean and std of relation attention weights for all nodes
        :return:
        """
        return {(metapath if "." in metapath or len(metapath) > 1 else node_type): (avg, std) \
                for node_type in self._beta_avg for (metapath, avg), (relation_b, std) in
                zip(self._beta_avg[node_type].items(), self._beta_std[node_type].items())}

def tag_negative(metapath):
    if isinstance(metapath, tuple):
        return metapath + ("neg",)
    elif isinstance(metapath, str):
        return metapath + "_neg"
    else:
        return "neg"

def untag_negative(metapath):
    if isinstance(metapath, tuple) and metapath[-1] == "neg":
        return metapath[:-1]
    elif isinstance(metapath, str):
        return metapath.strip("_neg")
    else:
        return metapath

def is_negative(metapath):
    if isinstance(metapath, tuple) and metapath[-1] == "neg":
        return True
    elif isinstance(metapath, str) and "_neg" in metapath:
        return True
    else:
        return False

def adamic_adar(indexA, valueA, indexB, valueB, m, k, n, coalesced=False, sampling=True):
    A = SparseTensor(row=indexA[0], col=indexA[1], value=valueA,
                     sparse_sizes=(m, k), is_sorted=not coalesced)
    B = SparseTensor(row=indexB[0], col=indexB[1], value=valueB,
                     sparse_sizes=(k, n), is_sorted=not coalesced)

    deg_A = A.storage.colcount()
    deg_B = B.storage.rowcount()
    deg_normalized = 1.0 / (deg_A + deg_B).to(torch.float)
    deg_normalized[deg_normalized == float('inf')] = 0.0

    D = SparseTensor(row=torch.arange(deg_normalized.size(0), device=valueA.device),
                     col=torch.arange(deg_normalized.size(0), device=valueA.device),
                     value=deg_normalized.type_as(valueA),
                     sparse_sizes=(deg_normalized.size(0), deg_normalized.size(0)))

    out = A @ D @ B
    row, col, values = out.coo()

    num_samples = min(int(valueA.numel()), int(valueB.numel()), values.numel())
    if sampling and values.numel() > num_samples:
        idx = torch.multinomial(values, num_samples=num_samples,
                                replacement=False)
        row, col, values = row[idx], col[idx], values[idx]

    return torch.stack([row, col], dim=0), values

def negative_sample(edge_index, M: int, N: int, n_sample_per_edge: int):
    num_neg_samples = edge_index.size(1) * n_sample_per_edge
    num_neg_samples = int(min(num_neg_samples, M * N - edge_index.size(1)))
    rng = range(M * N)
    idx = (edge_index[0] * N + edge_index[1]).to('cpu')  # idx = N * i + j

    perm = torch.tensor(random.sample(rng, num_neg_samples))
    mask = torch.from_numpy(np.isin(perm, idx)).to(torch.bool)
    rest = mask.nonzero().view(-1)
    while rest.numel() > 0:  # pragma: no cover
        tmp = torch.tensor(random.sample(rng, rest.size(0)))
        mask = torch.from_numpy(np.isin(tmp, idx)).to(torch.bool)
        perm[rest] = tmp
        rest = rest[mask.nonzero().view(-1)]

    row = perm // N
    col = perm % N
    neg_edge_index = torch.stack([row, col], dim=0).long()

    return neg_edge_index.to(edge_index.device)


 # Method description
 # Columns "Name", "Ticket", "Fare" and "Cabin" dropped. Missing data in "Age" column filled with mean age value.
 # Missing data row containing "NaN" dropped. Categorical data columns preprocessed with OneHotEncoder.
 # Continuous data columns preprocessed with StandardScaler.
 # FeatureUnion was applied and data set was fitted with a logistic regression classifier with hyper parameter tuning using GridSearchCV.
 # Accuracy score of 0.8090 and roc_auc_score of 0.8521 was obtained from model training with 20% data.
 # Tested several algorithms: logistic regression, linearsvc, KNN and RandomForestClassifier. Latter is found to be the best.

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import os
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score, accuracy_score
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier

class FeatureSelector(BaseEstimator, TransformerMixin):
    """ To select dataframe columns for Pipeline"""
    # Class Constructor
    def __init__(self, feature_names):
        self.feature_names = feature_names

    # Return self nothing else to do here
    def fit(self, X, y=None):
        return self

    # Method that describes what we need this transformer to do
    def transform(self, X, y=None):
        if  self.feature_names:
            return X[self.feature_names]

dir = os.getcwd()
ext = "Titanic"
path = os.path.join(dir, ext)
os.chdir(path)

df = pd.read_csv("train.csv", header=0, index_col="PassengerId")
df = df.drop(['Name', 'Ticket', 'Fare', 'Cabin'], axis=1)
df["Age"].fillna(df["Age"].mean(), inplace=True)    # Replace null data with mean
df.dropna(axis=0, how='any', inplace=True) # Drop null data row which contain null set in "Embarked"

Y = df["Survived"]
X = df.drop(["Survived"], axis=1)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=42, shuffle=True, test_size=0.1) # Split train.csv into 20% test set

categoricalFeatures = ["Pclass", "Sex", "Embarked"]
continuousFeatures = ["Age", "SibSp", "Parch"]

categoricalPipeline = Pipeline(steps=[
                                     ("categoricalSelector", FeatureSelector(feature_names=categoricalFeatures)),
                                     ("oneHotEncoder", OneHotEncoder(drop="first", sparse=False, dtype=np.int)) # Labelencoder is already included in OneHotEncoder with new update
                                     ])

continuousPipeline = Pipeline(steps=[
                                    ("continuousSelector", FeatureSelector(feature_names=continuousFeatures)),
                                    ('scaler', StandardScaler())
                                    ])

unionPipeline = FeatureUnion(transformer_list=[
                                                ("continuousPipeline", continuousPipeline),
                                                ("categoricalPipeline", categoricalPipeline)
                                                ])

mainPipeline = Pipeline(steps=[
                                ("mainPipeline", unionPipeline),
                                # ("model", LogisticRegression(penalty='l2', C=1.0, solver='liblinear', l1_ratio=None))
                                # ("model", LinearSVC(C=1.0, penalty="l2"))
                                ("model", RandomForestClassifier(n_estimators=10, random_state=42))
                                # ("model", KNeighborsClassifier(n_neighbors=5, algorithm="auto"))
                                ])

# paramsLogReg = {'model__C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]} # For logistic regression and liner SVC
paramsRFC = {"model__n_estimators": [100, 200, 300],
            "model__max_depth": [i for i in range(5, 100, 5)]
            }
# paramsKnn = {'model__n_neighbors': [i for i in range(1, 31, 1)]}
clf = GridSearchCV(mainPipeline, param_grid=paramsRFC, cv=3, verbose=1)
clf.fit(X_train, Y_train)
score = clf.score(X_test, Y_test)
Y_pred = clf.predict(X_test)
print(score)

# # ROC curve for logistic regression
# Y_pred_prob = clf.predict_proba(X_test)[:, 1]
# fpr, tpr, thresholds = roc_curve(Y_test, Y_pred_prob)
# fig, ax = plt.subplots(1, 1, sharex=False, sharey=False)
# sns.set_style("whitegrid")
# ax.plot(fpr, tpr, label='Logistic Regression')
# ax.plot([0, 1], [0, 1], 'k--')
# ax.set_xlabel("False Positive Rate")
# ax.set_ylabel("True Positive Rate")
# ax.set_title("Logistic Regression ROC Curve")

# # Area under ROC curve
# rocAreascore = roc_auc_score(Y_test, Y_pred_prob)
# print(rocAreascore)

# plt.show()

################################################ Predict using actual test data ################################################
# Perform prediction using actual test data
dfTest = pd.read_csv("test.csv", header=0, index_col="PassengerId")
dfTest = dfTest.drop(['Name', 'Ticket', 'Fare', 'Cabin'], axis=1)
dfTest.fillna(dfTest["Age"].mean(), inplace=True)

Y_testFinal = clf.predict(dfTest) # Does not need to manually preprocess test data separately. clf.predict puts test data through similar pipeline

# Output test result for Kaggle submission
dfTest["Survived"] = Y_testFinal
dfTest = dfTest.filter(items=["Survived"], axis=1)
dfTest.to_csv("TitanicSubmission.csv", sep=",")
# ################################################               End                 ################################################


#!/usr/bin/env python3

"""
./day5-lunch-4.py all.csv <gene of interest> K4me1.tab K4me3.tab K9me3.tab

"""
#output.tab
# # name - name field from bed, which should be unique
#    size - size of bed (sum of exon sizes
#    covered - # bases within exons covered by bigWig
#    sum - sum of values over all bases covered
#    mean0 - average over bases with non-covered bases counting as zeroes
#    mean - average over just covered bases

# use sum - sum of values over all bases covered as predictor to predict the gene expression FBtr0302347

import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm
import scipy

df = pd.read_csv(sys.argv[1])

goi= df.loc[:, "gene_name"] == sys.argv[2]

fpkm_val=df.loc[goi,"male_10"] #print out the t_name with FPKM under same gene name
toi_name=df.loc[goi,"t_name"]

#print (toi_name)
#print (fpkm_val)

fpkm_list=[]
toi_list=[]

for i, fpkm in enumerate(fpkm_val):
    fpkm_list.append(fpkm)

for q, toi_id in enumerate(toi_name):
    toi_list.append(toi_id)

#print (fpkm_list)
#print (toi_list)

def score_hm(name):
    cover_list=[]
    for i, line in enumerate(name):
        fields = line.rstrip("\n").split("\t")
        cover=fields[3]
        hm_toi=fields[0]
        if hm_toi in toi_list:
            cover_list.append(cover)
    return cover_list #have a list of selected transcript name

cover_list_K41m= score_hm(open(sys.argv[3]))
cover_list_K43m= score_hm(open(sys.argv[4]))
cover_list_K93m= score_hm(open(sys.argv[5]))

#print (len(fpkm_list), len(cover_list_K41m), len(cover_list_K43m), len(cover_list_K93m))
#combine all these lists into dictionary and make a new df
new_df=pd.DataFrame({"FPKM_Sxl":fpkm_list, "K4me1": cover_list_K41m, "K4me3": cover_list_K43m, "K9me3": cover_list_K93m})
#print (new_df)
model = sm.formula.ols(formula = "FPKM_Sxl ~ K4me1 + K4me3 + K9me3", data = new_df)

ols_results = model.fit()

print (ols_results.summary())


import json
f= open("survey.json","r")
x = json.load(f)
f.close()

sum_ages = 0
lst_ages =[]
for i in x:
    temp = i["How many siblings do you have? "]
    if temp.isnumeric():
        age = int(temp)
        sum_ages += age
        lst_ages.append(age)
avg = sum_ages / len(x)
print(avg)
print(lst_ages)


# coding:utf-8
from flask import redirect, url_for, flash, render_template, request
from flask_login import current_user, login_required

from app import db
from app.admin import admin
from app.admin.forms import PostForm, TagForm, AboutForm
from app.decorators import admin_required, permission_required
from app.models import Post, Tag, Comment, Permission

def delete(id, Model):
    model_instance = Model.query.get_or_404(id)
    db.session.delete(model_instance)
    db.session.commit()

@admin.route('/', methods=['GET', 'POST'])
@admin_required
def index():
    return redirect(url_for('admin.posts_manage'))

@admin.route('/post/write', methods=['GET', 'POST'])
@admin_required
def write_post():
    form = PostForm()
    if form.validate_on_submit():
        post = Post(title=form.title.data, body=form.body.data,
                    author=current_user._get_current_object())
        for tag in form.tags.data:
            post.tags.append(tag)
        db.session.add(post)
        db.session.commit()
        flash(u'文章添加成功', 'success')
        return redirect(url_for('.posts_manage'))
    return render_template('admin/write_post.html', form=form)

@admin.route('/post/<int:id>/edit', methods=['GET', 'POST'])
@admin_required
def edit_post(id):
    post = Post.query.get_or_404(id)
    form = PostForm()
    if form.validate_on_submit():
        post.title = form.title.data
        post.body = form.body.data
        for tag in post.tags.all():
            post.tags.remove(tag)
        for tag in form.tags.data:
            post.tags.append(tag)
        db.session.add(post)
        db.session.commit()
        flash(u'文章内容已修改！', 'success')
        return redirect(url_for('.posts_manage'))

    form.title.data = post.title
    form.body.data = post.body
    form.tags.data = post.tags.all()
    return render_template('admin/edit_post.html', form=form, post=post)

@admin.route('/post/<int:id>/delete', methods=['GET', 'POST'])
@admin_required
def delete_post(id):
    delete(id, Post)
    flash(u'已删除！', 'success')
    return redirect(url_for('.posts_manage'))

@admin.route('/posts/manage', methods=['GET', 'POST'])
@admin_required
def posts_manage():
    page = request.args.get('page', 1, type=int)
    pagination = Post.query.order_by(Post.timestamp.desc()).paginate(
        page, per_page=20, error_out=False)
    page_posts = pagination.items
    tags = Tag.query.all()
    return render_template('admin/posts_manage.html',
                           pagination=pagination,
                           page_posts=page_posts,
                           tags=tags)

@admin.route('/posts/delete', methods=['GET', 'POST'])
@admin_required
def delete_posts():
    post_ids = request.form.getlist("do_delete")
    count = 0
    for post_id in post_ids:
        delete(post_id, Post)
        count += 1
    flash(u'共删除%d篇文章' % count, 'success')
    return redirect(url_for('.posts_manage'))

@admin.route('tag/add', methods=['GET', 'POST'])
@admin_required
def add_tag():
    form = TagForm()
    if form.validate_on_submit():
        tag = Tag(name=form.name.data)
        db.session.add(tag)
        db.session.commit()
        flash(u'分类添加成功', 'success')
        return redirect(url_for('.tags_manage'))

@admin.route('/tag/<int:id>/edit', methods=['GET', 'POST'])
@admin_required
def edit_tag(id):
    tag = Tag.query.get_or_404(id)
    form = TagForm()
    if form.validate_on_submit():
        tag.name = form.name.data
        db.session.add(tag)
        db.session.commit()
        flash(u'分类名称已修改', 'success')
        return redirect(url_for('.tags_manage'))
    form.name.data = tag.name
    return render_template('admin/edit_tag.html', form=form, tag=tag)

@admin.route('/delete-tag/<int:id>', methods=['GET', 'POST'])
@admin_required
def delete_tag(id):
    delete(id, Tag)
    flash(u'已删除', 'success')
    return redirect(url_for('.tags_manage'))

@admin.route('/tags/manage', methods=['GET', 'POST'])
@admin_required
def tags_manage():
    form = TagForm()
    page = request.args.get('page', 1, type=int)
    pagination = Tag.query.paginate(
        page, per_page=20, error_out=False)
    page_tags = pagination.items
    return render_template('admin/tags_manage.html',
                           pagination=pagination,
                           page_tags=page_tags,
                           form=form)

@admin.route('/tags/delete', methods=['GET', 'POST'])
@admin_required
def delete_tags():
    count = 0
    for id in request.form.getlist("do_delete"):
        delete(id, Tag)
        count += 1
    flash(u'共删除%d个标签' % count, 'success')
    return redirect(url_for('.tags_manage'))

@admin.route('/tag/<name>', methods=['GET', 'POST'])
@admin_required
def tag_posts(name):
    page = request.args.get('page', 1, type=int)
    tag = Tag.query.filter_by(name=name).first_or_404()
    pagination = tag.posts.order_by(Post.timestamp.desc()).paginate(
        page, per_page=20, error_out=False)
    page_posts = pagination.items
    tags = Tag.query.all()
    return render_template('admin/tag_posts.html',
                           tag=tag,
                           pagination=pagination,
                           page_posts=page_posts,
                           tags=tags)

@admin.route('/comments/manage', methods=['GET', 'POST'])
@admin_required
def comments_manage():
    page = request.args.get('page', 1, type=int)
    pagination = Comment.query.order_by(Comment.timestamp.desc()).paginate(
        page, per_page=20, error_out=False)
    page_comments = pagination.items
    return render_template('admin/comments_manage.html',
                           pagination=pagination,
                           page_comments=page_comments)

@admin.route('/comments/delete', methods=['GET', 'POST'])
@admin_required
def delete_comments():
    count = 0
    for id in request.form.getlist("do_delete"):
        delete(id, Comment)
        count += 1
    flash(u'共删除%d条评论' % count, 'success')
    return redirect(url_for('.comments_manage'))

@admin.route('/comment/<int:id>/delete', methods=['GET', 'POST'])
@admin_required
def delete_comment(id):
    delete(id, Comment)
    flash(u'评论已删除', 'success')
    return redirect(url_for('.comments_manage'))

@admin.route('/comments/manage/enable/<int:id>')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def comment_enable(id):
    comment = Comment.query.get_or_404(id)
    comment.disabled = False
    db.session.add(comment)
    return redirect(url_for('.comments_manage',
                            page=request.args.get('page', 1, type=int)))

@admin.route('/moderate/disable/<int:id>')
@login_required
@permission_required(Permission.MODERATE_COMMENTS)
def comment_disable(id):
    comment = Comment.query.get_or_404(id)
    comment.disabled = True
    db.session.add(comment)
    return redirect(url_for('.comments_manage',
                            page=request.args.get('page', 1, type=int)))

@admin.route('/about/write', methods=['GET', 'POST'])
@admin_required
def about_write():
    form = AboutForm()
    if form.validate_on_submit():
        post = Post(title='about', body=form.content.data,
                    author=current_user._get_current_object())
        post.is_about = True
        db.session.add(post)
        db.session.commit()
        flash(u'文章添加成功', 'success')
        return redirect(url_for('.posts_manage'))
    return render_template('admin/about_write.html', form=form)

@admin.route('/about/edit', methods=['GET', 'POST'])
@admin_required
def about_edit():
    post = Post.query.filter_by(is_about=True).first_or_404()
    form = AboutForm()
    if form.validate_on_submit():
        post.body = form.content.data
        db.session.add(post)
        db.session.commit()
        return redirect(url_for('.posts_manage'))
    form.content.data = post.body
    return render_template('admin/about_edit.html', form=form)


"""
Vigenère and derived ciphers
"""

import string

from Ciphers.types import Action
from letter_values import get_letter_for_value, letter_to_value
from utils import clean_word

def use_vigenere(msg: str, keyword: str, chars_to_keep: str = ' ', action: Action = Action.DECRYPT) -> str:
    """
    Encrypt or decrypt a message using Vigenère cipher

    https://www.braingle.com/brainteasers/codes/vigenere.php

    :param msg: Message to encrypt or decrypt
    :param keyword: Keyword to encrypt or decrypt with
    :param chars_to_keep: Sequence of characters that are left unchanged, by default only space to keep
                          words intact.
    :param action: Flag indicating whether to encrypt or decrypt the message

    >>> use_vigenere('Dlgo wj d wiqooxh', 'keyword')
    THIS IS A MESSAGE
    """

    msg = clean_word(msg, string.ascii_uppercase + chars_to_keep)
    keyword = clean_word(keyword)

    result = ''
    cursor_in_keyword = 0
    for letter_msg in msg:
        if letter_msg in chars_to_keep:
            result += letter_msg
            continue

        letter_key = keyword[cursor_in_keyword % len(keyword)]
        value_msg = letter_to_value[letter_msg]
        value_key = letter_to_value[letter_key] - 1  # In Vigenère, an A in the key means no rotation
        if action == Action.ENCRYPT:
            value_result = value_msg + value_key
        else:
            value_result = value_msg - value_key

        result += get_letter_for_value(value_result)
        cursor_in_keyword += 1

    return result

def use_autokey(msg: str, keyword: str, chars_to_keep: str = ' ', action: Action = Action.DECRYPT) -> str:
    """
    Encrypt or decrypt a message using Autokey cipher

    https://www.braingle.com/brainteasers/codes/autokey.php

    Note: Also known as autoclave

    >>> use_autokey('This is a secret message', 'zebra', action=Action.ENCRYPT)
    'SLJJ IR E TVCQIU DERWBXE'
    """

    if action == Action.ENCRYPT:
        # For encryption, we can simply use Vigenère with the keyword prefixing the message
        return use_vigenere(msg, keyword + msg, chars_to_keep, action)

    result = ''
    msg = clean_word(msg, allowed_chars=string.ascii_uppercase + chars_to_keep)
    keyword = clean_word(keyword)

    cursor_in_keyword = 0
    for letter_msg in msg:
        if letter_msg in chars_to_keep:
            result += letter_msg
            continue

        letter_key = keyword[cursor_in_keyword]
        value_msg = letter_to_value[letter_msg]
        value_key = letter_to_value[letter_key] - 1  # In Vinegère, an A in the key means no rotation
        value_result = value_msg - value_key

        letter_result = get_letter_for_value(value_result)
        result += letter_result
        keyword += letter_result
        cursor_in_keyword += 1

    return result

if __name__ == '__main__':
    # Examples to use Vinegère
    print(use_vigenere('goed gedaan', 'cache', action=Action.ENCRYPT))
    print(use_vigenere('iogk kgdchr', 'cache'))
    print(use_vigenere('This is a message!', 'keyword', action=Action.ENCRYPT))
    print(use_vigenere('Dlgo wj d wiqooxh!', 'keyword', action=Action.DECRYPT))
    print('----')

    print(use_autokey('This is a secret message', 'zebra', action=Action.ENCRYPT))
    print('----')

    msg = 'BSAJDWQPMRIXKXXFIBGEIBYILKNKFCXRJCWYHDZECFUHHRPMULOHTIRXVDRFXRJICIQGEELBHEINDQRRMRZGHIXJHS'
    key = 'doorgedraaid'
    for i in range(len(msg)):
        new_msg = msg[i:] + msg[:i]
        for j in range(len(key)):
            new_key = key[j:] + key[:j]
            decrypted = use_autokey(new_msg, new_key)
            if 'BAND' in decrypted and 'EERSTE' in decrypted:
                print(decrypted)


import serial
import socket
import sys
import os
from _thread import *
import serial.serialutil

last_dev = os.popen('ls /dev/ttyUSB* | tail -n 1').read()
last_dev=last_dev.replace("\n", "")
print(last_dev);
ser = serial.Serial(last_dev, 9600) # here you have to write your port. If you dont know how to find it just write ls -l /dev/tty.* in your terminal (i'm using mac)
#sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#server_address = ('192.168.137.105', 10005)

HOST = ''   # Symbolic name, meaning all available interfaces
PORT = 10101 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ('Socket created');
conn1=None

#print ('connecting')
#sock.connect(server_address)
#print ('connected')
#Bind socket to local host and port
try:

    s.bind((HOST, PORT))
except socket.error as msg:
    if msg.errno == 98:
        print ('Port is already in use');
    else:
        print ('Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]);
        sys.exit()

print ('Socket bind complete');

#Start listening on socket
s.listen(10)
print ('Socket now listening');

#Function for handling connections. This will be used to create threads
def clientthread():
    #Sending message to connected client
    #message='Welcome to the server. Type something and hit enter\n'
    #conn.send(bytes(message.encode('utf-8'))) #send only takes string

    #infinite loop so that function do not terminate and thread do not end.
    while True:
        try:
            print("loop starts");
            global ser
            response = ser.readline()
            response=response.replace(b'\x02', b'')
            print("reading data")
            if(conn1):
                conn1.send(response)

            print (response);
        except serial.SerialException:
            #x=os.system("ls /dev/ttyUSB*")
            #print(x);
            #if(x==0):
            try:
                print("ok");
                last_dev = os.popen('ls /dev/ttyUSB* | tail -n 1').read()
                last_dev=last_dev.replace("\n", "")
                print(last_dev);
                ser = serial.Serial(last_dev, 9600)
            except:
                print("");
        except FileNotFoundError:
            print("");
        except KeyboardInterrupt:
            print("");
        except:
            print("");

    #ser.close()
    print("out of loop1");
    #came out of loop
    #conn1.close()

#now keep talking with the client
clientNo=0
start_new_thread(clientthread ,())
while 1:
   #wait to accept a connection - blocking call
    conn, addr = s.accept()
    if(conn1):
        conn1.close()

    conn1=conn
    clientNo=clientNo+1
    print ('Connected with ' + addr[0] + ':' + str(addr[1]));
    #start new thread takes 1st argument as a function name to be run, second is the tuple of arguments to the function.

s.close()


import json
import os
import shlex
import subprocess
import time

class CommandGenerator:
    def __init__(self, command_template, replace_dict={"": [""]}, handle_responses=lambda responses: responses):
        self.command_template = command_template
        self.replace_dict = replace_dict
        self.handle_responses = handle_responses

    def generate_commands(self):
        commands = [self.command_template]
        for keyword, replacements in self.replace_dict.items():
            commands = [command.replace(keyword, replacement) for command in commands for replacement in replacements]
        return commands

    def run_subprocess_and_get_results(self,command, max_length=1024, timeout=1):
        print(">> Running: ", str(command)[ :max_length])
        args = shlex.split(command) ## while err??
        process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        (out, err, rc) = *process.communicate(), process.returncode
        time.sleep(timeout)
        print(">> Results: ", str((out, err, rc))[ :max_length])
        if rc != 0:
            print(">> Command failed!")
            if auto_y or input("Retry command (y)?: ") == 'y':
                return self.run_subprocess_and_get_results(command)
        return {'command': command, 'stdout': out, 'stderr': err, 'returncode': rc}

    def generate_responses(self, commands):
        responses = []
        for command in commands:
            responses += [self.run_subprocess_and_get_results(command)]
        return responses

    def process_responses(self):
        commands = self.generate_commands()
        responses = self.generate_responses(commands)
        print("Printing responses:\n\n", "\n\n".join(map(lambda response: str(response), responses)))
        return self.handle_responses(responses)

def get_quests():
    get_template = "curl -v '<API_URL>' -H 'Accept: application/json, text/plain, */*' --insecure"
    get_replace_dict = {
        "<API_URL>": ["https://manila1.cpaas.awsiondev.infor.com:18010/coleman/api/quest"],
    }
    def get_handler(responses):
        print("Processing responses from: get commands.")
        try:
            quests = json.loads(responses[0]['stdout'].decode('utf-8'))
            print("Finished looking for {} quests: {}.".format(len(quests), [quest['name'] for quest in quests]))
        except Exception as e:
            print(e)
            print(responses[0]['stderr'].decode('utf-8'))
        return quests
    if input("Run get tasks (y)? ") == 'y':
        get_generator = CommandGenerator(command_template=get_template, replace_dict=get_replace_dict, handle_responses=get_handler)
        quests = get_generator.process_responses()
        return quests

def save_quests(quests):
    save_template = "curl -v '<API_URL>/<DS_ID>?render=true' -H 'Accept: application/json, text/plain, */*' --insecure"
    save_replace_dict = {
        "<API_URL>": ["https://manila1.cpaas.awsiondev.infor.com:18010/coleman/api/quest"],
        "<DS_ID>": [quest['id'] for quest in quests],
    }
    def save_handler(responses):
        print("Processing responses from: save commands.")
        if not os.path.exists(backup_directory): os.makedirs(backup_directory)
        for result in responses:
            try:
                saved_quest =  json.loads(result['stdout'].decode('utf-8'))['quest']
                backup_file = os.path.join(backup_directory, "backup_{}_{}.json".format(saved_quest["name"], saved_quest["id"]))
                with open(backup_file, 'wb') as out_file:
                    out_file.write(result['stdout'])
                print("Successfully saved: {}".format(backup_file))
            except Exception as e:
                print(e)
                print(result['stderr'].decode('utf-8'))
        return "Finished saving backups to: /{}/.".format(backup_directory)
    if input("Run save tasks (y)? ") == 'y':
        save_generator = CommandGenerator(command_template=save_template, replace_dict=save_replace_dict, handle_responses=save_handler)
        print("Backup status: {}".format(save_generator.process_responses()))

def delete_quests(quests):
    delete_template = "curl -v '<API_URL>/<DS_ID>' -X DELETE -H 'Accept: application/json, text/plain, */*' --insecure"
    delete_replace_dict = {
        "<API_URL>": ["https://manila1.cpaas.awsiondev.infor.com:18010/coleman/api/quest"],
        #"<DS_ID>": [quest['id'] for quest in quests],
        #"<DS_ID>": [quest['id'] for quest in quests if len(quest['activities']) < 2],
    }
    if input("Run delete tasks (y)? ") == 'y':
        delete_generator = CommandGenerator(command_template=delete_template, replace_dict=delete_replace_dict)
        print("Delete status: {}".format(delete_generator.process_responses()))

def update_quests(quests):
    update_template = "curl -v '<API_URL>' -X POST -H 'Content-Type: application/json' -H 'Accept: application/json, text/plain, */*' --data-binary '<QUEST_JSON>' --insecure"
    update_replace_dict = {
        "<API_URL>": ["https://manila1.cpaas.awsiondev.infor.com:18010/coleman/api/quest"],
        "<QUEST_JSON>": [json.dumps({"quest": quest}) for quest in quests],
    }
    if input("Run update tasks (y)? ") == 'y':
        update_generator = CommandGenerator(command_template=update_template, replace_dict=update_replace_dict)
        responses = update_generator.process_responses()
        print("Update status: {}".format(responses))

def dynamo_scan_keys(region_name, table_name):
    scan_template = """aws dynamodb scan \
--region '<REGION_NAME>' \
--table-name '<TABLE_NAME>' \
--attributes-to-get 'id'
    """
    scan_replace_dict = {
        "<REGION_NAME>": [region_name],
        "<TABLE_NAME>": [table_name],
    }
    def scan_handler(responses):
        load_scan_result = json.loads(responses[0]['stdout'].decode('utf-8'))
        return load_scan_result['Items']
    if auto_y or input("Run scan tasks (y)? ") == 'y':
        scan_generator = CommandGenerator(command_template=scan_template, replace_dict=scan_replace_dict, handle_responses=scan_handler)
        keys = scan_generator.process_responses()
        return keys

def dynamo_add_string_field(region_name, table_name, keys, field_name, dummy_string):
    add_template = """
aws dynamodb update-item \
--region '<REGION_NAME>' \
--table-name '<TABLE_NAME>' \
--key '<KEY>' \
--update-expression 'SET <FIELD_NAME> = :nf' \
--expression-attribute-values '{ ":nf": { "S": "<DUMMY_STRING>" }}'
    """
    add_replace_dict = {
        "<REGION_NAME>": [region_name],
        "<TABLE_NAME>": [table_name],
        "<KEY>": [json.dumps(key) for key in keys],
        "<FIELD_NAME>": [field_name],
        "<DUMMY_STRING>": [dummy_string],
    }
    if auto_y or input("Run add tasks (y)? ") == 'y':
        add_generator = CommandGenerator(command_template=add_template, replace_dict=add_replace_dict)
        responses = add_generator.process_responses()
        print("Add status: {}".format(responses))
        return responses

def dynamo_delete_string_field(region_name, table_name, keys, field_name):
    delete_template = """
aws dynamodb update-item \
--region '<REGION_NAME>' \
--table-name '<TABLE_NAME>' \
--key '<KEY>' \
--update-expression 'REMOVE <FIELD_NAME>'
    """
    delete_replace_dict = {
        "<REGION_NAME>": [region_name],
        "<TABLE_NAME>": [table_name],
        "<KEY>": [json.dumps(key) for key in keys],
        "<FIELD_NAME>": [field_name],
    }
    if input("Run delete tasks (y)? ") == 'y':
        delete_generator = CommandGenerator(command_template=delete_template, replace_dict=delete_replace_dict)
        responses = delete_generator.process_responses()
        print("Delete status: {}".format(responses))
        return responses

""" Notes:
For automating AWS commands: you must have AWS CLI and proper credentials set up in your machine
For automating API calls via cURL: Copy a cURL sample from Chrome > F12 > Network > locate particular request > Right-click > Copy > Copy as cURL (bash)
"""

if __name__ == '__main__':
    ##########################
    ###  Modify execution  ###
    ##########################
    auto_y = True
    region_name = "eu-west-1"
    table_name = "coleman_execution_context"

    keys = dynamo_scan_keys(region_name, table_name)
    """
    new_fields = {
        "createdBy": "Jay Ryan Ramos",
        "createdOn": "2018-09-07T03:52:55.152+0000",
        "lastUpdatedBy": "Jay Ryan Ramos",
        "lastUpdatedOn": "2018-09-07T03:52:55.152+0000",
    }

    add_responses = []
    for field, value in new_fields.items():
        add_responses += [dynamo_add_string_field(region_name, table_name, keys, field, value)]
    """
    for delete_field in ["failedActivity", "worklog"]:
        dynamo_delete_string_field(region_name, table_name, keys, delete_field)

    ##########################
    ###  Modify execution  ###
    ##########################

    print("Entering Interactive mode: Input Ctrl + Z to exit.")
    import code; code.interact(local={**locals(), **globals()})
    print("Done!")


# -*- coding: utf-8 -*-
import scrapy
from vulinfo.items import VulinfoItem

class VulspiderSpider(scrapy.Spider):
    name = 'vulspider'
    allowed_domains = ['www.umei.cc']
    start_urls = ['https://www.umei.cc/meinvtupian/xingganmeinv/1.htm']

    def parse(self, response):

        #本页地址列表
        localurls=response.xpath('//a[@class="TypeBigPics"]/@href').getall()
        for localurl in localurls:

            yield scrapy.Request(url=localurl, callback=self.parse)
        #下一页地址
        localnextpage=response.xpath('//div[@class="NewPages"]/ul/li/a/@href').getall()[-2]
        yield scrapy.Request(url=response.urljoin(localnextpage),callback=self.parse)

        #各个标题的item和url
        nextpage=response.xpath('//div[@class="NewPages"]/ul/li/a/@href').getall().pop()
        if nextpage =='#':
            pass
        else:

            yield scrapy.Request(url=response.urljoin(nextpage),callback=self.parse)

        item=VulinfoItem()
        item['image_urls']=response.xpath('//p[@align="center"]/a/img/@src').getall()
        item['images']=response.xpath('//p[@align="center"]/a/img/@alt').getall()
        item['image_paths']=response.xpath('//p[@align="center"]/a/img/@alt').get()

        yield item


import pandas as pd
from scipy.io.arff import loadarff
import numpy as np
import random
import matplotlib.pyplot as plt
import sys

def get_max_features(iris_df, class_names, features_to_search):
    iris_df_ax = iris_df.max()
    min_arr = np.array([iris_df_ax.sepallength, iris_df_ax.sepalwidth, iris_df_ax.petallength, iris_df_ax.petalwidth])
    return min_arr

def get_min_features(iris_df, class_names, features_to_search):
    iris_df_min = iris_df.min()
    min_arr = np.array([iris_df_min.sepallength, iris_df_min.sepalwidth, iris_df_min.petallength, iris_df_min.petalwidth])
    return min_arr

def get_min_2d_numpy_arr(numpy_arr):

    min_sepallength = None
    min_sepalwidth = None
    min_petallength = None
    min_petalwidth = None

    for row in numpy_arr:
        if(min_sepallength == None or row[0] < min_sepallength):
            min_sepallength = row[0]

        if (min_sepalwidth == None or row[1] < min_sepalwidth):
            min_sepalwidth = row[1]

        if (min_petallength == None or row[2] < min_petallength):
            min_petallength = row[2]

        if (min_petalwidth == None or row[3] < min_petalwidth):
            min_petalwidth = row[3]

    min_arr = [min_sepallength, min_sepalwidth, min_petallength, min_petalwidth]
    return min_arr
def get_max_2d_numpy_arr(numpy_arr):

    max_sepallength = None
    max_sepalwidth = None
    max_petallength = None
    max_petalwidth = None

    for row in numpy_arr:
        if(max_sepallength == None or row[0] > max_sepallength):
            max_sepallength = row[0]

        if (max_sepalwidth == None or row[1] > max_sepalwidth):
            max_sepalwidth = row[1]

        if (max_petallength == None or row[2] > max_petallength):
            max_petallength = row[2]

        if (max_petalwidth == None or row[3] > max_petalwidth):
            max_petalwidth = row[3]

    max_arr = [max_sepallength, max_sepalwidth, max_petallength, max_petalwidth]
    return max_arr

def create_synthetic_data(classname):

    features_to_search = ['sepallength', 'sepalwidth', 'petallength', 'petalwidth']
    class_names = [b'Iris-setosa', b'Iris-versicolor', b'Iris-virginica']

    # load data
    raw_data = loadarff('iris.arff')
    iris_df = pd.DataFrame(raw_data[0])

    # min and max array where each it follows in same index pattern as features_to_search array
    min_arr = get_min_features(iris_df, class_names, features_to_search)
    max_arr = get_max_features(iris_df, class_names, features_to_search)

    # From my group's Blackboard thread where Stuart Woodburry suggested this one line code to generate random 100
    # we want 100 new data for each feature so a 4 by 100 array
    oldRandomMat = [[(random.uniform(0, 1) * max_arr[i] - min_arr[i]) + min_arr[i] for i in range(0, len(features_to_search))] for j in range(0, 100)]
    oldRandomMat = np.array(oldRandomMat)

    # get first fifty instances so we can calculate a covariance
    first_fifty = iris_df.loc[(iris_df['class'] == classname)].head(50)
    all_col_except_class = first_fifty[features_to_search]
    numpy_arr_first_fifty = all_col_except_class.values
    # To get a proper covariance matrix we need to transpose so we get a 4 x 4 array
    numpy_arr_first_fifty = numpy_arr_first_fifty.T
    numpy_cov = np.cov(numpy_arr_first_fifty)

    # multiply the random Matrix with the covariance
    rnd_data = np.matmul(numpy_cov, oldRandomMat.T)
    randomMat = rnd_data.T

    # get the min and max of the random matrix of each feature so we should get a 1 by 4 array
    min_arr_randomMat = get_min_2d_numpy_arr(randomMat)
    max_arr_randomMat = get_max_2d_numpy_arr(randomMat)

    # sepal length processing
    X = randomMat.T[0]
    P_min = min_arr_randomMat[0]
    P_max = max_arr_randomMat[0]
    a = sepal_length_min = min_arr[0]
    b = sepal_length_max = max_arr[0]
    synthetic_sepal_length = ((X - P_min) / (P_max - P_min)) * (b - a) + a
    iris_synthetic = synthetic_sepal_length

    # sepal width processing
    X = randomMat.T[1]
    P_min = min_arr_randomMat[1]
    P_max = max_arr_randomMat[1]
    a = sepal_width_min = min_arr[1]
    b = sepal_width_max = max_arr[1]
    synthetic_sepal_width = ((X - P_min) / (P_max - P_min)) * (b - a) + a
    iris_synthetic = np.vstack((iris_synthetic, synthetic_sepal_width))

    # petal length processing
    X = randomMat.T[2]
    P_min = min_arr_randomMat[2]
    P_max = max_arr_randomMat[2]
    a = petal_length_min = min_arr[2]
    b = petal_length_max = max_arr[2]
    synthetic_petal_length = ((X - P_min) / (P_max - P_min)) * (b - a) + a
    iris_synthetic = np.vstack((iris_synthetic, synthetic_petal_length))

    # petal width processing
    X = randomMat.T[3]
    P_min = min_arr_randomMat[3]
    P_max = max_arr_randomMat[3]
    a = petal_width_min = min_arr[3]
    b = petal_width_max = max_arr[3]
    synthetic_petal_width = ((X - P_min) / (P_max - P_min)) * (b - a) + a
    # iris_synthetic[3] = synthetic_petal_width
    iris_synthetic = np.vstack((iris_synthetic, synthetic_petal_width))

    # let's the get the original data values
    orig_data = iris_df[features_to_search].values

    # get the mean of the original data set for each feature
    orig_mean = [np.mean(orig_data[:, i]) for i in range(0, len(features_to_search))]
    # get the mean of the just generated synth data set for each feature
    synth_mean = [np.mean(iris_synthetic[:, i]) for i in range(0, len(features_to_search))]

    # subtract the two
    mu = np.subtract(synth_mean, orig_mean)

    # let's make the mu_100
    mu_100 = np.multiply(np.ones((1,100)), mu[0])[0]
    mu_100 = np.vstack((mu_100, np.multiply(np.ones((1,100)), mu[1])[0]))
    mu_100 = np.vstack((mu_100, np.multiply(np.ones((1,100)), mu[2])[0]))
    mu_100 = np.vstack((mu_100, np.multiply(np.ones((1,100)), mu[3])[0]))

    # subtract the mu_100 from synthetic data
    final_iris_synth = np.subtract(iris_synthetic, mu_100)

    # scatter plot x - column 0, y - column 1, shown with marker o
    plt.plot(final_iris_synth[:,3], final_iris_synth[:,0], 'o', label='data', color = "blue")
    plt.plot(orig_data[:,3], orig_data[:,0], 'o', label='data', color = "red")
    # create legend in case you have more than one series
    plt.legend()
    plt.title(classname)
    plt.show()

    return final_iris_synth

def problem3():

    # I don't understand why this isn't working
    # I followed the professors steps in office hours and I thought
    # I had the reasoning down but I'm not getting the right results
    # class_names = [b'Iris-setosa', b'Iris-versicolor', b'Iris-virginica']

    setosa_synth = create_synthetic_data(b'Iris-setosa')
    versicolor_synth = create_synthetic_data(b'Iris-versicolor')
    virginica_synth = create_synthetic_data(b'Iris-virginica')

    final_synth = setosa_synth
    final_synth = np.hstack((final_synth, versicolor_synth))
    final_synth = np.hstack((final_synth, virginica_synth))

    # Final data of size 300, 4
    final_synth = final_synth.T

    # dump to file
    np.set_printoptions(threshold=sys.maxsize)
    with open('Problem3_synth_result.txt', 'w') as prob3_synth_file_output:
        prob3_synth_file_output.write(np.array2string(final_synth, suppress_small=False))
        prob3_synth_file_output.close()

def main():
    problem3()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- cording: utf-8 -*-

from flask import Flask, render_template, send_from_directory
import logging

app = Flask(__name__, static_folder='../frontend/dist/static', template_folder='../frontend/dist')
app.config.from_pyfile('../config.py')
app.logger.setLevel(logging.DEBUG)

@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def index(path):
    return render_template('index.html')

@app.route('/media/<path:path>')
def send_media(path):
    print(path)
    return send_from_directory('../media', path)

from backend.apis import character, photo, voice

app.register_blueprint(character.app, url_prefix='/api/character')
app.register_blueprint(photo.app, url_prefix='/api/photo')
app.register_blueprint(voice.app, url_prefix='/api/voice')