import xml.etree.ElementTree as ET
from xml.etree.ElementTree import ParseError
import argparse
import json
import time
import geojson
from geojson import Feature, Point, FeatureCollection, Polygon
import re
import os
'''from geographiclib.geodesic import Geodesic
import math'''

def writeLogs(line):
    print(line)
    file='logs_'+time.strftime("%Y%m%d")
    f=open(file,'a')
    if f.mode=="a":
        f.write(time.strftime("%Y%m%d%H%M%S")+' '+line+"\n")
    return

def savejson(data, file_path):
    complete_path=file_path + '_'+time.strftime("%Y%m%d")+".json"
    writejson(data,complete_path)
    return

def savegeojson(data, file_path):
    complete_path=file_path + '_'+time.strftime("%Y%m%d")+".geojson"
    writejson(data,complete_path)
    return

def writejson(data,file_path):
    str_data = open(file_path,'w')
    json.dump(data, str_data, indent=1)
    writeLogs("data saved in: "+file_path+"\n")
    return

def lookup(complex_element,key):
    for element in complex_element:
        if element.tag == key:
            return element.text
    #in case one of the namespace is used, we should look for the key + namespace
    for ns in AIXM_NAMESPACE.values():
        for element in complex_element:
            if element.tag == "{"+ns+"}"+key:
                return element.text
    return ''

def lookupattrib(complex_element,key,keyattribute):
    for element in complex_element:
        if element.tag == key:
            return element.attrib[keyattribute]
    #in case one of the namespace is used, we should look for the key + namespace
    for ns in AIXM_NAMESPACE.values():
        for element in complex_element:
            if element.tag == "{"+ns+"}"+key:
                if keyattribute in element.attrib:
                    return element.attrib[keyattribute]
                for ns in AIXM_NAMESPACE.values():#in case one of the namespace is used, we should look for the keyattribute + namespace
                    if "{"+ns+"}"+keyattribute in element.attrib:
                        return element.attrib["{"+ns+"}"+keyattribute]
    return ''

def findAllcheck(tree,path):
    try:
        elements = tree.findall(path,AIXM_NAMESPACE)
    except:
        writeLogs("ERROR: impossible to parse the path "+path+ " with the namespaces "+AIXM_NAMESPACE)
        return None
    if elements== []:
        writeLogs("there is no Element  with the tag "+path)

    return elements

def extractFeatureAIXM(fxml,featureType): #return all the feature of this feature type as a liste
    elements=[]
    elementTree = ET.parse(fxml)
    elements = findAllcheck(elementTree,'aixm-message:hasMember/aixm:'+featureType)
    return elements

def checkLastVersion(v):
    if len(v) == 1:
        file_path = "./"
        file_name = v
    else:
        p = os.path.split(v)
        file_path = p[0]
        file_name = p[1]

    print("path: ",file_path,"name : ",file_name)

    file_list = os.listdir(file_path)
    lastUpdateName=''
    lastUpdate=0

    lname=len(file_name)
    for file in file_list:
        if file[0:lname]==file_name:
            try:
                fdate=int(getFileDate(file))
            except:
                writeLogs("ERROR: No date in the name of:"+ file)
                continue
            if fdate>lastUpdate:
                lastUpdate=fdate
                lastUpdateName=file
    if lastUpdate!=0:
        writeLogs('INFO: File used : '+ lastUpdateName +" in :"+file_path)
        return lastUpdateName
    writeLogs('INFO: no file found with the following name: '+file_name+ "in:"+file_path)
    return

def getFileDate(name):
    fulldate=re.search(r'\d+',name).group()
    return fulldate[0:8]

def getcolor(elem,type): # this function return the defined color that should be applied, or grey (#CBD2D0) if none
    if elem.get('color')!=None:return elem.get('color')
    if type in ['AIRSPACE','Airspace']: return '#CBD2D0'
    if type in ['DESIGNATEDPOINT','DesignatedPoint']: return '#549CE5'
    if type in ['NAVAID','Navaid']: return '#AAB2BA'
    return ''

def getActivity(Airspace): #check the activity Type
    activities=[]
    activations=findAllcheck(Airspace,'aixm:activation/aixm:AirspaceActivation')
    for activation in activations:
        activity=lookup(activation,'activity')
        if activity!='':activities.append(activity)
    return activities

def getElement(feature,allElements,elementsToAdd,doc):
    if len(elementsToAdd)==0:return [] #no element to be added to this chart
    elementForChart=[]
    for element in allElements:
        id=lookup(element, 'identifier')
        allElementTS=findAllcheck(element,'aixm:timeSlice/aixm:'+feature+'TimeSlice')
        for elementTS in allElementTS:
            designator=lookup(elementTS,'designator')
            type=lookup(elementTS,'type')
            for elementToAdd in elementsToAdd:
                if elementToAdd['type'] == type or elementToAdd['type'] =='': #if type='', it means that any type should be taken into consideration
                    if elementToAdd.get('activity')!=None:#for airspaces, an activity type could be define
                        if elementToAdd['activity'] not in getActivity(elementTS):
                            continue#if the activity does not correspond to the define one, ignore this element
                    if isinstance(elementToAdd['name'],str) and elementToAdd['name'] in designator: #if only one string is defined, we check if the designator contain this string
                        elementForChart.append({"name":designator,"featureType":feature,"type":type,'referencedoc':doc,'ref_uid':id,'color':getcolor(elementToAdd,feature)})
                    if isinstance(elementToAdd['name'],list):
                        for elem in elementToAdd['name']:
                            if elem in designator: #if a liste of name is defined, we check if the designator contain any of this string
                                elementForChart.append({"name":designator,"featureType":feature,"type":type,'referencedoc':doc,'ref_uid':id,'color':getcolor(elementToAdd,feature)})
    writeLogs(str(len(elementForChart))+" elements of type "+feature+" added to "+args.sfolayer+".json")
    return elementForChart

def getRoutesID(routes,routeNames):#return a dictionary with for each route id, a route name
    ids={}
    for name in routeNames:
        for route in routes:
            id=lookup(route, 'identifier')
            allrouteTS=findAllcheck(route,'aixm:timeSlice/aixm:RouteTimeSlice')
            for routeTS in allrouteTS:
                if name == lookup(routeTS,'name'):
                    ids[id]=name
    return ids
def showMe(runwayAxisIDs):
    for ra in runwayAxisIDs:
        writeLogs("Key : {} , first_dir : {}, second_dir : {}".format(ra['id'],ra['first_dir'],ra['second_dir']))
    exit()

def getRunwayDirectionID(runwaydirectionsxml,ref):
    for rwydir in runwaydirectionsxml:
        id=lookup(rwydir, 'identifier')
        allrwydirTS=findAllcheck(rwydir, 'aixm:timeSlice/aixm:RunwayDirectionTimeSlice')
        for rwydirTS in allrwydirTS:
            allrwyext=findAllcheck(rwydirTS, 'aixm:extension/cae:RunwayDirectionExtension')
            for rwyext in allrwyext:
                featureCode=lookup(rwyext, 'featureCode')
                if featureCode == ref:
                    return id
    return ''

def getRunwayCentrelinePoint(runwaycentrelinepointsxml,runwaydirid):
    result=[]
    for centrelinePoint in runwaycentrelinepointsxml:
        id=lookup(centrelinePoint, 'identifier')
        allpointTS=findAllcheck(centrelinePoint, 'aixm:timeSlice/aixm:RunwayCentrelinePointTimeSlice')
        for pointTS in allpointTS:
            onrwy=lookupattrib(pointTS,'onRunway',"href")
            role=lookup(pointTS, 'role')
            if runwaydirid in onrwy and role in RWYCENTRELINEPTROLE:
                allpositions=findAllcheck(pointTS, 'aixm:location/aixm:ElevatedPoint')
                for position in allpositions:
                    pos=lookup(position, 'pos')
                    result=[id,pos]
                    return result
    return []

def getSecondAxisPoint(startpos,refpos,len):
    pos=''
    # writeLogs("start position :"+startpos)
    # writeLogs("reference position :"+refpos)
    # start point
    startcoord=startpos.split(" ")
    startlat=float(startcoord[0])
    startlong=float(startcoord[1])
    # reference point
    refcoord=refpos.split(" ")
    reflat=float(refcoord[0])
    reflong=float(refcoord[1])

    geod=Geodesic.WGS84 # define the WGS84 ellipsoid
    # get distance between start and ref points
    distance=geod.Inverse(reflat, reflong, startlat, startlong)
    # get ratio
    ratio=(distance['s12']+(len*1.852e3))/distance['s12']
    # compute second axis point
    l=geod.InverseLine(reflat, reflong, startlat, startlong)
    g=l.Position(ratio * l.s13)
    pos="{:.15f} {:.15f}".format(g['lat2'],g['lon2'])
    return pos

def getRunwayAxisPoints(runwaydirectionsxml,runwaycentrelinepointsxml,elementsToAdd,doc):
    if len(elementsToAdd)==0:return
    elementForChart=[]
    for elementToAdd in elementsToAdd:
        if isinstance(elementToAdd['airport'],str) and isinstance(elementToAdd['axis'], str) and isinstance(elementToAdd['length'], int):
            directions=elementToAdd['axis'].split("_")
        if len(directions) != 2:
            continue
        # get runway directions IDs
        first_rwyref=directions[0]+" "+elementToAdd['airport']
        first_rwydirid=getRunwayDirectionID(runwaydirectionsxml,first_rwyref)
        second_rwyref=directions[1]+" "+elementToAdd['airport']
        second_rwydirid=getRunwayDirectionID(runwaydirectionsxml,second_rwyref)
        # get runway centreline points
        first_rwyref=elementToAdd['airport']+"_"+directions[0]
        second_rwyref=elementToAdd['airport']+"_"+directions[1]
        first_point=getRunwayCentrelinePoint(runwaycentrelinepointsxml,first_rwydirid)
        third_point=getRunwayCentrelinePoint(runwaycentrelinepointsxml,second_rwydirid)
        # compute coordinates
        second_point=getSecondAxisPoint(third_point[1],first_point[1],elementToAdd['length'])
        fourth_point=getSecondAxisPoint(first_point[1],third_point[1],elementToAdd['length'])

        first_points={'name':'START_POINT','featureType':'RunwayCentreLinePoint','referencedoc':doc,'ref_uid':first_point[0]},{'name':'END_POINT','featureType':'Point','coordinates':[fourth_point]}
        elementForChart.append({'name':first_rwyref,'featureType':'RunwayAxis','referencedoc':'','ref_uid':'','color':'#9B59B6','Points':first_points})
        second_points={'name':'START_POINT','featureType':'RunwayCentreLinePoint','referencedoc':doc,'ref_uid':third_point[0]},{'name':'END_POINT','featureType':'Point','coordinates':[second_point]}
        elementForChart.append({'name':second_rwyref,'featureType':'RunwayAxis','referencedoc':'','ref_uid':'','color':'#9B59B6','Points':second_points})
    writeLogs(str(len(elementForChart))+" elements of type RunwayAxis added to "+args.sfolayer+".json")
    return elementForChart

def getPointRefID(segment,startorend):
    point=segment.find('aixm:'+startorend+'/aixm:EnRouteSegmentPoint',AIXM_NAMESPACE)
    pointid=lookupattrib(point,'pointChoice_fixDesignatedPoint',"href")
    if pointid!='': #the start or end of the segment is a designatedpoint
     return {'type':"DesignatedPoint",'id':pointid[9:]} #remove "urn:uuid:" from the id
    pointid=lookupattrib(point,'pointChoice_navaidSystem','href')
    if pointid!='': #the start or end of the segment is a Navaid
     return {'type':"Navaid",'id':pointid[9:]}
    return ''

def getSegmentRefRoute(segmentsxml,routeIDs):
    segmentsforchart=[]
    for segment in segmentsxml:
        id=lookup(segment,'identifier')
        allsegmentTS=findAllcheck(segment,'aixm:timeSlice/aixm:RouteSegmentTimeSlice')
        for segmentTS in allsegmentTS:
            routeref=lookupattrib(segmentTS,'routeFormed',"href")
            for routeid in routeIDs:
                if routeid in routeref: #this route segment belong to a route that we need to include in the chart
                    startpoint=getPointRefID(segmentTS,"start")#ID of the designated point starting the segment
                    endpoint=getPointRefID(segmentTS,"end")#ID of the designated point ending the segment
                    segmentsforchart.append({'id':id,'route name':routeIDs[routeid],'refstart':startpoint,'refend':endpoint})
                continue
    return segmentsforchart

def getPointName(point,pointsxml,navaidsxml):
    if point['type']=="DesignatedPoint":
        for dpoint in pointsxml:
            if point['id']==lookup(dpoint,'identifier'):
                pointTS=dpoint.find('aixm:timeSlice/aixm:DesignatedPointTimeSlice',AIXM_NAMESPACE)
    if point['type']=="Navaid":
        for navaid in navaidsxml:
            if point['id']==lookup(navaid,'identifier'):
                pointTS=navaid.find('aixm:timeSlice/aixm:NavaidTimeSlice',AIXM_NAMESPACE)
    return lookup(pointTS,'designator')

def addSegment(elementForChart,segment,startname,endname,docSegment,docPoint):
    segmentname=segment['route name']+' ('+startname+' - '+endname+')'
    startend=[{'name':startname,'featureType':segment['refstart']['type'],'referencedoc':docPoint,'ref_uid':segment['refstart']['id']},{'name':endname,'featureType':segment['refend']['type'],'referencedoc':docPoint,'ref_uid':segment['refend']['id']}]
    elementForChart.append({"name":segmentname,"featureType":'RouteSegment','referencedoc':docSegment,'ref_uid':segment['id'],'color':ROUTE_COLOR,"Points":startend})
    return elementForChart

def getRouteSegment(routesxml,routesegmentsxml,pointsxml,navaidsxml,routetoadd,docSegment,docPoint):
    if len(routetoadd)==0:return []
    elementForChart=[]
    routesID=getRoutesID(routesxml,routetoadd)#list of all the ID of the route to be displayed on the chart
    segmentsRef=getSegmentRefRoute(routesegmentsxml,routesID)#liste of the segment referencing those routes ID
    for segment in segmentsRef:
        startname=getPointName(segment['refstart'],pointsxml,navaidsxml)
        endname=getPointName(segment['refend'],pointsxml,navaidsxml)
        elementForChart=addSegment(elementForChart,segment,startname,endname,docSegment,docPoint)
    writeLogs(str(len(elementForChart))+" elements of type route segment added to "+args.sfolayer+".json")

    return elementForChart

def chartDefinition(airspaces,navaids,points,routesegments,routes,chartIn,chartConf):
    writeLogs("starting to collect the element to add to the .json for: "+chartIn['NAME'])
    airspaceForChart=getElement("Airspace",airspaces,chartIn['AIRSPACE'],'Airspace_NoRefGeoborder.xml')
    designatedPointForChart=getElement("DesignatedPoint",points,chartIn['POINT'],'DesignatedPoint_Navaid.xml')
    navaidForChart=getElement("Navaid",navaids,chartIn['NAVAID'],'DesignatedPoint_Navaid.xml')
    routesegmentForChart=getRouteSegment(routes,routesegments,points,navaids,chartIn['ROUTE'],'RouteSegment.xml','DesignatedPoint_Navaid.xml')
    runwaycentrelinepointForChart=getRunwayAxisPoints(runwaydirections,runwaycentrelinepoints,chartIn['RWYAXIS'],'RunwayCentrelinePoint.xml')

    elements=airspaceForChart+navaidForChart+designatedPointForChart+routesegmentForChart+runwaycentrelinepointForChart#list of elements contained in chartConf
    chartConf["sfo Layers"].append({"chartname":chartIn['NAME'], "elements":elements})
    return

def readGeojson(fgeojson):
    data=open(fgeojson,'r')
    return geojson.load(data)

def getFeatureType(geojsontype):#the type is indicated in the Type data as ""<feauretype>TYPE"
    try:
        type=re.search('(.*)%s' % ("TYPE"),geojsontype).group(1)
    except:
        writeLogs("Can't find type in: "+ geojsontype)
        return
    return type

def getFeatureSubType(geojsonfc):
    #the subtype is the last string,some time indicated into ()
    try:
        fc=geojsonfc.split(' ')
        subtype=str(fc[len(fc)-1])
        if subtype[0]=="(":subtype=subtype[1:] #remove the () if any
        if subtype[-1]==")":subtype=subtype[:-1]
    except:
        writeLogs("Can't find subtype in: "+ geojsonfc)
        return ''
    return subtype

def getFeatureName(geojsonfc):
    #the name is the first string in the feature code
    try:
        fc=geojsonfc.split(' ')
        name=fc[0]
    except:
        writeLogs("Can't find name in: "+ geojsonfc)
        return '',''
    return name

def insertGeojson(out,geometry, type,subtype,name,id,color):
    properties={"uid":id,"feature type":type,"type":subtype,"name":name,"color":color}
    feature=Feature(geometry=geometry,properties=properties)
    out["features"].append(Feature(geometry=geometry,properties=properties))
    return

def getFeatureGeojson(elementsDict,geojsonIn,geojsonOut,chartname):
    nb=0#count the number of element added
    if len(elementsDict)==0:return #no element to be added to this geojson
    featurescol=readGeojson(geojsonIn)
    for feature in featurescol["features"]:
        featureCode=feature['properties']['featureCode']
        featureType=getFeatureType(feature['properties']['dataType'])
        if featureType in ["AIRSPACE","DESIGNATEDPOINT","NAVAID"]:
            featureSubType=getFeatureSubType(featureCode)
        else: continue #in case the feature is another type (for exemple DME or GEOBORDDER) we do not add it
        featureName=getFeatureName(featureCode)
        featureID=feature['properties']["identifier"]['value']

        for element in elementsDict: #check if the feature is part of the list of element to be added in the chart
            if element['type'] == featureSubType or element['type'] =='': #if type='', it means that any type should be taken into consideration
                if isinstance(element['name'],str):#if the name is a string, just check if it is contained in the feauture code
                    if element['name'] in featureName:
                        insertGeojson(geojsonOut,feature['geometry'],featureType,featureSubType,featureName,featureID,getcolor(element,featureType))
                        nb+=1
                    continue
                if isinstance(element['name'],list):#if the name is a list of name, I need to check if any of them is contained into the feature featureCode
                    for name in element['name']:
                        if name in featureName:
                            insertGeojson(geojsonOut,feature['geometry'],featureType,featureSubType,featureName,featureID,getcolor(element,featureType))
                            nb+=1
                        continue
                    continue
                else:
                    writeLogs("The attribute name of "+element[name]+" should be a string or a list")
    writeLogs(str(nb)+" elements of type "+featureType+" added to "+chartname+".geojson")
    return

def getRouteGeojson(routesDict,geojsonIn,geojsonOut,chartname):
    nb=0#count the number of element added
    if len(routesDict)==0:return #no element to be added to this geojson
    featurescol=readGeojson(geojsonIn)
    for feature in featurescol["features"]:
        featureCode=feature['properties']['featureCode']
        featureRouteName=getFeatureName(featureCode)
        featureType=getFeatureType(feature['properties']['dataType'])
        featureID=feature['properties']["identifier"]['value']

        for route in routesDict: #check if the feature is part of the list of element to be added in the chart
            if route == featureRouteName:
                insertGeojson(geojsonOut,feature['geometry'],featureType,'',featureCode,featureID,ROUTE_COLOR)
                nb+=1
    writeLogs(str(nb)+" elements of type "+featureType+" added to "+chartname+".geojson")
    return

def chartGeojson(airspaceGeojson,navaidGeojson,designatedpointGeojson,RouteSegmentGesojson,chartIn):
    writeLogs("Start to collect the element to create: "+chartIn['NAME']+".geojson")
    chartGeojson=FeatureCollection([]) #geojson for this chart
    getFeatureGeojson(chartIn['AIRSPACE'],airspaceGeojson,chartGeojson,chartIn['NAME'])
    getFeatureGeojson(chartIn['POINT'],designatedpointGeojson,chartGeojson,chartIn['NAME'])
    getFeatureGeojson(chartIn['NAVAID'],navaidGeojson,chartGeojson,chartIn['NAME'])
    getRouteGeojson(chartIn['ROUTE'],RouteSegmentGesojson,chartGeojson,chartIn['NAME'])
    savegeojson(chartGeojson,chartIn['NAME'])
    return

#########GLOBAL VARIABLE########################################
AIXM_NAMESPACE={'aixm-message':'http://www.aixm.aero/schema/5.1/message','aixm':"http://www.aixm.aero/schema/5.1",'gml':"http://www.opengis.net/gml/3.2",'xlink':"http://www.w3.org/1999/xlink"}
ROUTE_COLOR='#3D5B74'

######### ZRH Lower Chart ##############
ZRH_LOWER={}
ZRH_LOWER['NAME']="ZRH Lower Chart" #name that will appears on the App
ZRH_LOWER['NAVAID']=[{"type":"VOR","name":''},{"type":"DME","name":''},{"type":"NDB","name":''},{"type":"VOR_DME","name":''},{"type":"VORTAC","name":''}] #Navaid that will be displayed
ZRH_LOWER['POINT']=[{"type":"ICAO","name":''}]#designated point to be displayed. '' means all
ZRH_LOWER['ROUTE']=["L613","W112","Z119","Z83"]#route that should be displayed
ZRH_LOWER['AIRSPACE']=[{"type":"SECTOR","name":["NORTH","EAST","WEST","SOUTH"]}] #airspace to be displayed. No name means all Airspace of this type

######### ZRH Upper Chart ##############
ZRH_UPPER={}
ZRH_UPPER['NAME']="ZRH Upper Chart" #name that will appears on the App
ZRH_UPPER['NAVAID']=[{"type":"VOR","name":''},{"type":"DME","name":''},{"type":"NDB","name":''},{"type":"VOR_DME","name":''},{"type":"VORTAC","name":''}] #Navaid that will be displayed
ZRH_UPPER['POINT']=[{"type":"ICAO","name":''}]#designated point to be displayed. '' means all
ZRH_UPPER['ROUTE']=["UL613","UP131","Z119","UZ6113","UZ630"]#route that should be displayed
ZRH_UPPER['AIRSPACE']=[{"type":"SECTOR","name":["M2","M1"],"color":"#79B8A9"}] #airspace to be displayed. No name means all Airspace of this type

######### GVA Lower Chart ##############
GVA_LOWER={}
GVA_LOWER['NAME']="GVA Lower Chart" #name that will appears on the App
GVA_LOWER['NAVAID']=[{"type":"VOR","name":''},{"type":"DME","name":''},{"type":"NDB","name":''},{"type":"VOR_DME","name":''},{"type":"VORTAC","name":''}] #Navaid that will be displayed
GVA_LOWER['POINT']=[{"type":"ICAO","name":''}]#designated point to be displayed. '' means all
GVA_LOWER['ROUTE']=[]#route that should be displayed
GVA_LOWER['AIRSPACE']=[{"type":"SECTOR","name":["L1"],"color":"#79B8A9"},{"type":"AWY","name":''},{"type":"CTR","name":''},{"type":"TMA","name":"LS"}] #airspace to be displayed. No name means all Airspace of this type

######### GVA Upper Chart ##############
GVA_UPPER={}
GVA_UPPER['NAME']="GVA Upper Chart" #name that will appears on the App
GVA_UPPER['NAVAID']=[{"type":"VOR","name":''},{"type":"DME","name":''},{"type":"NDB","name":''},{"type":"VOR_DME","name":''},{"type":"VORTAC","name":''}] #Navaid that will be displayed
GVA_UPPER['POINT']=[{"type":"ICAO","name":''}]#designated point to be displayed. '' means all
GVA_UPPER['ROUTE']=['UN852','UN853','UL153','UN869','UN871','UM729','UM135','UM730','Y24','UP860','UT45','UL612','UM975']#route that should be displayed
GVA_UPPER['AIRSPACE']=[{"type":"SECTOR","name":["L1"],"color":"#79B8A9"},{"type":"AWY","name":''},{"type":"CTR","name":''},{"type":"TMA","name":"LS"} ] #airspace to be displayed. No name means all Airspace of this type

######### TWR/APP Chart ##############
TWR_APP={}
TWR_APP['NAME']="TWR APP Chart" #name that will appears on the App
TWR_APP['NAVAID']=[{"type":"VOR","name":''},{"type":"DME","name":''},{"type":"NDB","name":''},{"type":"VOR_DME","name":''},{"type":"VORTAC","name":''}] #Navaid that will be displayed
TWR_APP['POINT']=[{"type":"ICAO","name":''},{"type":"OTHER:VFR_REP","name":'','color':"#67E175"}]#designated point to be displayed. '' means all
TWR_APP['ROUTE']=[]#route that should be displayed
TWR_APP['AIRSPACE']=[{"type":"SECTOR","name":'LSZH',"color":"#79B8A9"},{"type":"AWY","name":''},{"type":"CTR","name":''},{"type":"TMA","name":"LS"},{"type":"R","name":'','color':'#909491','activity':'GLIDING'} ] #airspace to be displayed. No name means all Airspace of this type
TWR_APP['RWYAXIS']=[{"airport":"LSGG","axis":"04_22","length":20},{"airport":"LSZH","axis":"10_28","length":20},{"airport":"LSZH","axis":"16_34","length":20},{"airport":"LSZH","axis":"14_32","length":20},{"airport":"LSMD","axis":"11_29","length":10}]

##### list of charts#######
CHART_LIST=[ZRH_LOWER,ZRH_UPPER,GVA_LOWER,GVA_UPPER,TWR_APP]
################################################################
parser = argparse.ArgumentParser()
parser.add_argument('--airspace-path', type=str,  help='path of Airspace file without version nor extension', dest='airspace', default='./Airspace')
parser.add_argument('--routesegment-path', type=str,  help='path of RouteSegment file without version nor extension', dest='routesegment', default='./RouteSegment')
parser.add_argument('--navaid-designatedpoint-path', type=str, help='path of Navaid_DesignatedPoint file without version nor extension', dest='designatedPointNavaid', default='./DesignatedPoint_Navaid')
parser.add_argument('--sfo-layer-path', type=str, help='path of sfos_layer file without version nor extension', dest='sfolayer', default='./SFO_layer')
parser.add_argument('--runwaycentrelinepoint-path', type=str, help='path of RunwayCentrelinePoint file without version nor extension', dest='runwaycentrelinepoint', default='./RunwayCentrePoint')

args = parser.parse_args()
args = parser.parse_args()

#input
routesegmentxml=checkLastVersion(args.routesegment)
airspacexml=checkLastVersion(args.airspace)
pointxml=checkLastVersion(args.designatedPointNavaid)
runwaycentrelinepointxml=checkLastVersion(args.runwaycentrelinepoint)

print('\n')

airspaceGeojson="Airspace.geojson"
designatedpointGeojson="DesignatedPoint.geojson"
navaidGeojson="Navaid.geojson"
RouteSegmentGesojson="RouteSegment.geojson"
#output
#fsfolayer=checkLastVersion(args.sfolayer)

#extract elements from the xml
airspaces=extractFeatureAIXM(airspacexml,"Airspace")
navaids=extractFeatureAIXM(pointxml,"Navaid")
points=extractFeatureAIXM(pointxml,"DesignatedPoint")
routesegments=extractFeatureAIXM(routesegmentxml, "RouteSegment")
routes=extractFeatureAIXM(routesegmentxml, "Route")
runwaydirections=extractFeatureAIXM(runwaycentrelinepointxml, "RunwayDirection")
runwaycentrelinepoints=extractFeatureAIXM(runwaycentrelinepointxml, "RunwayCentrelinePoint")

sfolayer={}
sfolayer["sfo Layers"]=[]
'''for chartdef in CHART_LIST:

    chartGeojson(airspaceGeojson,navaidGeojson,designatedpointGeojson,RouteSegmentGesojson,chartdef)'''

for chartdef in CHART_LIST:

    chartDefinition(airspaces,navaids,points,routesegments,routes,chartdef,sfolayer)
    print('\n')

savejson(sfolayer,args.sfolayer)
writeLogs("File Updated")
writeLogs("Done!")


from config import (
    BORDERS_TABLE as borders_table,
    OSM_TABLE as osm_table
)
from countries_division import country_initial_levels
from subregions import (
    get_subregions_info,
    update_border_mwm_size_estimation,
)

class CountryStructureException(Exception):
    pass

def _clear_borders(conn):
    with conn.cursor() as cursor:
        cursor.execute(f"DELETE FROM {borders_table}")
    conn.commit()

def _find_subregions(conn, osm_ids, next_level, regions):
    """Return subregions of level 'next_level' for regions with osm_ids."""
    subregion_ids = []
    for osm_id in osm_ids:
        more_subregions = get_subregions_info(conn, osm_id, borders_table,
                                              next_level, need_cities=False)
        for subregion_id, subregion_data in more_subregions.items():
            region_data = regions.setdefault(subregion_id, {})
            region_data['name'] = subregion_data['name']
            region_data['mwm_size_est'] = subregion_data['mwm_size_est']
            region_data['parent_id'] = osm_id
            subregion_ids.append(subregion_id)
    return subregion_ids

def _create_regions(conn, osm_ids, regions):
    if not osm_ids:
        return
    osm_ids = list(osm_ids)  # to ensure order
    sql_values = ','.join(
            f'({osm_id},'
            '%s,'
            f"{regions[osm_id].get('parent_id', 'NULL')},"
            f"{regions[osm_id].get('mwm_size_est', 'NULL')},"
            f'(SELECT way FROM {osm_table} WHERE osm_id={osm_id}),'
            'now())'
            for osm_id in osm_ids
    )
    with conn.cursor() as cursor:
        cursor.execute(f"""
            INSERT INTO {borders_table} (id, name, parent_id, mwm_size_est,
                                 geom, modified)
            VALUES {sql_values}
            """, tuple(regions[osm_id]['name'] for osm_id in osm_ids)
        )

def _make_country_structure(conn, country_osm_id):
    regions = {}  # osm_id: { 'name': name,
                  #           'mwm_size_est': size,
                  #           'parent_id': parent_id }

    country_name = get_osm_border_name_by_osm_id(conn, country_osm_id)
    country_data = regions.setdefault(country_osm_id, {})
    country_data['name'] = country_name
    # TODO: country_data['mwm_size_est'] = ...

    _create_regions(conn, [country_osm_id], regions)

    if country_initial_levels.get(country_name):
        admin_levels = country_initial_levels[country_name]
        prev_admin_levels = [2] + admin_levels[:-1]
        prev_region_ids = [country_osm_id]

        for admin_level, prev_level in zip(admin_levels, prev_admin_levels):
            if not prev_region_ids:
                raise CountryStructureException(
                        f"Empty prev_region_ids at {country_name}, "
                        f"AL={admin_level}, prev-AL={prev_level}"
                )
            subregion_ids = _find_subregions(conn, prev_region_ids,
                                             admin_level, regions)
            _create_regions(conn, subregion_ids, regions)
            prev_region_ids = subregion_ids
    warning = None
    if len(regions) == 1:
        try:
            update_border_mwm_size_estimation(conn, country_osm_id)
        except Exception as e:
            warning = str(e)
    return warning

def create_countries_initial_structure(conn):
    _clear_borders(conn)
    with conn.cursor() as cursor:
        # TODO: process overlapping countries, like Ukraine and Russia with common Crimea
        cursor.execute(f"""
            SELECT osm_id, name
            FROM {osm_table}
            WHERE admin_level = 2 and name != 'Ukraine'
            """
        )
        warnings = []
        for rec in cursor:
            warning = _make_country_structure(conn, rec[0])
            if warning:
                warnings.append(warning)
    conn.commit()
    return warnings

def get_osm_border_name_by_osm_id(conn, osm_id):
    with conn.cursor() as cursor:
        cursor.execute(f"""
            SELECT name FROM {osm_table}
            WHERE osm_id = %s
            """, (osm_id,))
        rec = cursor.fetchone()
        if not rec:
            raise CountryStructureException(
                f'Not found region with osm_id="{osm_id}"'
            )
        return rec[0]

def _get_country_osm_id_by_name(conn, name):
    with conn.cursor() as cursor:
        cursor.execute(f"""
            SELECT osm_id FROM {osm_table}
            WHERE admin_level = 2 AND name = %s
            """, (name,))
        row_count = cursor.rowcount
        if row_count > 1:
            raise CountryStructureException(f'More than one country "{name}"')
        rec = cursor.fetchone()
        if not rec:
            raise CountryStructureException(f'Not found country "{name}"')
        return int(rec[0])


from collections import deque
from functools import lru_cache
from typing import List

class Node:
    def __init__(self, name):
        self.children = []
        self.name = name

    def addChild(self, name):
        self.children.append(Node(name))
        return self

    def depthFirstSearch(self, array):
        # Write your code here.
        array.append(self.name)
        for child in self.children:
            child.depthFirstSearch(array=array)
        return array

    def breadthFirstSearch(self, array):
        # Write your code here.
        pass

class Graph:
    adj = []

    def addEdge(self, u, v, isDirected):
        self.adj[u].append(v)
        if not isDirected:
            self.adj[v].append(u)

    def dfsUtil(self, source, destination, count):
        if source == destination:
            return count
        else:

            for u in self.adj[source]:
                count += 1
                self.dfsUtil(u, destination=destination, count=count)
        return count

    def validPath(self, n: int, edges: List[List[int]], source: int, destination: int) -> bool:
        W = len(edges)

        a = [1]
        for _ in range(n):
            self.adj.append([])
        count = 0
        for i in range(n):
            self.addEdge(edges[i][0], edges[i][1], True)
        count = self.dfsUtil(source, destination, count)
        return count == 0
def allPathsSourceTarget( graph: List[List[int]]) -> List[List[int]]:

    target = len(graph) - 1
    results = []

    def backtrack(currNode, path):
        # if we reach the target, no need to explore further.
        if currNode == target:
            results.append(list(path))
            return
        # explore the neighbor nodes one after another.
        for nextNode in graph[currNode]:
            path.append(nextNode)
            backtrack(nextNode, path)
            path.pop()
    # kick of the backtracking, starting from the source node (0).
    path = deque([0])
    backtrack(0, path)

    return results

"""
    For DP Solution
    nextNode∈neighbors(currNode),allPathsToTarget(currNode)={currNode+allPathsToTarget(nextNode)}
"""
def allPathsSourceTargetV2( graph: List[List[int]]) -> List[List[int]]:

    target = len(graph) - 1

    # apply the memoization
    @lru_cache(maxsize=None)
    def allPathsToTarget(currNode):
        if currNode == target:
            return [[target]]

        results = []
        for nextNode in graph[currNode]:
            for path in allPathsToTarget(nextNode):
                results.append([currNode] + path)

        return results

    return allPathsToTarget(0)

def dfsUtil(array,source,visited):
    visited[source]=True
    i = 0
    target = 0
    while i!=source:
        print(visited)
        target+=array[i]
        visited[i]=True
        i+=1

    print()

    return False

def hasSingleCycle(array):
    jumps = 0
    position = 0

    while True:
        position += array[position]
        position %= len(array)
        print(position,end=' ')
        jumps += 1

        if position == 0 or array[position] == 0 or jumps > len(array):
            break
    print()
    return jumps == len(array)

def hasSingleCycleV2(array):
    # Write your code here.
    visited = [False for _ in range(len(array))]
    n = len(array)
    for i in range(n):
        visited = [False for _ in range(len(array))]
        if not dfsUtil(array,i,visited):
            return False
    return True

def hasSingleCycleV4(array):
    n = len(array)
    jumps = 0
    current_position=0
    while True:
        current_position+=array[current_position]
        current_position=current_position%n
        jumps+=1
        if current_position == 0 or array[current_position] == 0 or jumps > len(array):
            break

    return jumps == n

def hasSingleCycleV5(array):
    correct_cycles = 0
    jumps = 0
    n = len(array)
    for i in range(n):
        vertex = i
        jumps=0
        for j in range(n):
            jumps+= array[j]
            jumps = jumps % n
            if jumps == i:
                print(jumps)
                correct_cycles+=1
                break

    print(correct_cycles)
    return correct_cycles == n-1

def maximumScore(nums: List[int], multipliers: List[int]) -> int:
    pointer1 = 0
    n = len(nums)
    pointer2 = len(nums)-1
    m = len(multipliers)
    dp = [[0]*(m+1) for _ in range(m+1)]
    for i in range(m-1,-1,-1):
        for left in range(i,-1,-1):
            mult = multipliers[i]
            right = n-1-(i-left)
            dp[i][left]=max(mult*nums[left]+dp[i+1][left+1],mult*nums[right]+dp[i+1][left])
        print(dp)
    return dp[0][0]
#!/bin/python3

#
# Complete the 'maximumSum' function below.
#
# The function is expected to return a LONG_INTEGER.
# The function accepts following parameters:
#  1. LONG_INTEGER_ARRAY a
#  2. LONG_INTEGER m
#

def maximumSum(a, m):
    max_so_far=0;total=0
    current = 0;left=0
    for i in range(len(a)):

        if a[i]%m==m-1:
            return m-1
        if max_so_far==m-1:
            return max_so_far
        max_so_far+=a[i]

        if total<=max_so_far:
            total=max_so_far
        if total%m==m-1:
            print("Foo",total)
            return m-1
        if max_so_far>m:
            left = i
            while max_so_far>m and left>=0:
                max_so_far-=a[left]
                left-=1
        print(left)

        i=left
        print("Max",max_so_far,end=' ')
    print(total)
    return total%m

def maximumScoreV2( nums: List[int], multipliers: List[int]) -> int:
        # lru_cache from functools automatically memoizes the function
        @lru_cache(2000)
        def dp(i, left):
            # Base case
            if i == m:
                return 0

            mult = multipliers[i]
            right = n - 1 - (i - left)

            # Recurrence relation
            return max(mult * nums[left] + dp(i + 1, left + 1),
                       mult * nums[right] + dp(i + 1, left))

        n, m = len(nums), len(multipliers)
        return dp(0, 0)

if __name__ == '__main__':
    a = [2,3,1,-4,-4,2]
    print(hasSingleCycleV5(a))

    nums1 = [1,2,3]
    multipliers = [3,2,1]
    print(maximumScore(nums1,multipliers))
    a = [1,5,3,22,30,2,5]
    print(maximumSum(a,2))


import collections

class Solution:
    def findRotateSteps(self, ring, key):
        """
        :type ring: str
        :type key: str
        :rtype: int
        """
        dic = collections.defaultdict(list)
        for i, c in enumerate(ring):
            dic[c].append(i)
        n = len(ring)
        lst = [(0, 0)]
        for c in key:
            nxt_lst = []
            for v in dic[c]:
                min_step = float('inf')
                for step, u in lst:
                    step += min((v-u)%n, (u-v)%n)
                    if step < min_step:
                        min_step = step
                nxt_lst.append((min_step+1, v))
            lst = nxt_lst
        return min(lst)[0]

def test(ring, key):
    print(ring, key)
    r = Solution().findRotateSteps(ring, key)
    print(r)

ring = 'GODDING'
key = 'GD'
test(ring, key)
ring = 'ABRRCBD'
key = 'BCD'
test(ring, key)


# -*- coding: utf-8 -*-
#from test3 import Test
import math
import time
import datetime
from operator import itemgetter

from multiprocessing import Pool, Manager

from django.core.management.base import BaseCommand
from django.db.models import Q, Max
from django import db

def handle(kwargs):
    print(kwargs)

def handle2():
    queue = Manager().Queue(maxsize=4)
    queue.put(0)  # 触发程序开始

    per_num = 2
    all_record = [1, 2, 3, 4, 5, 6, 7, 8]
    count = 8
    cnt = int(math.ceil(count / per_num))

    pool = Pool(processes=4)
    arg_list = []
    for _ in range(cnt):
        arg_list.append({'qq': 1212, 'qqe': 13})
    pool.map(handle, arg_list)
    pool.close()
    pool.join()
    end_time = time.time()
    print("end at: ", end_time)
    print('total use {} s.'.format(end_time))
    print('Done!')

handle2()


import numpy as np  #导入矩阵操作函数库
import cv2
import os
from xlrd import open_workbook
import matplotlib.pyplot as plt
from sklearn import metrics
from sklearn.decomposition import PCA
from time import time

import warnings

warnings.filterwarnings('ignore')

#忽略一些版本不兼容等警告
warnings.filterwarnings("ignore")

sign=0
y = []
x = []

is_color = 1   #是否输入彩色图，为0则为灰度图
acc_res = []
# recall_res=[]
# hunxiao_res=[]

# #############################################################################
# 读取数据集

with open('messidor_features.artff.txt', 'r') as f:
    lines = f.readlines()[25:]
    for line in lines:
        l = [float(i) for i in line.split(',')]
        x.append(l[:-1])
        y.append(l[-1])

x = np.array(x)
y = np.array(y)
n_classes = 2
n_samples = x.shape[0]
n_features = x.shape[1]

print("Total dataset size:")
print("n_samples: %d" % n_samples)
print("n_features: %d" % n_features)
print("n_classes: %d" % n_classes)

# #############################################################################
# 数据集划分
# 交叉验证

from sklearn.model_selection import KFold
kf = KFold(n_splits=10,shuffle=True)
i=0
for i, (train_index, test_index) in enumerate(kf.split(x)):
    print('train_index{}'.format(i))
    X_train, y_train = x[train_index], y[train_index]
    X_test, y_test = x[test_index], y[test_index]
    print("训练样本上采样...")

    #贝叶斯主代码
    from sklearn.naive_bayes import MultinomialNB
    clf =  MultinomialNB()
    X_train += np.abs(X_train.min())

    from sklearn.naive_bayes import GaussianNB
    clf = GaussianNB()
    clf.fit(X_train, y_train)
    print(clf)

    # 测试evaluate y_test == y_pred rate
    y_pred = clf.predict(X_test)
    y_pred2 = clf.predict(X_train)
    acc = np.sum(y_test == y_pred) / X_test.shape[0]
    acc2 = np.sum(y_train == y_pred2) / X_train.shape[0]
    print('X_test:', X_test.shape[0])
    print('test acc: %.3f' % acc)
    print('X_train:', X_train.shape[0])
    print('train acc: %.3f' % acc2)
    acc_res.append(acc)

    print(clf.score(X_test, y_test))                         #预测准确率
    print(metrics.classification_report(y_test,y_pred)) #包含准确率，召回率等信息表
    print(metrics.confusion_matrix(y_test,y_pred))      #混淆矩阵
print(sum(acc_res)/len(acc_res))

# #精确度：precision，正确预测为正的，占全部预测为正的比例，TP / (TP+FP)
# 召回率：recall，正确预测为正的，占全部实际为正的比例，TP / (TP+FN)
# F1-score：精确率和召回率的调和平均数，2 * precision*recall / (precision+recall)

分两类 多项式贝叶斯.py


# region Imports

import os
import subprocess
import pathlib
import sys
import argparse
import json
from dotenv import load_dotenv

from Utilities.Arguments import args
from Utilities.Logger import Logger
from Subscription import Subscription
from Repositories import REPOSITORIES
from Destinations import DESTINATIONS

# endregion

class Chrysalis:
	"""
	The entry point for Chrysalis.

	Attributes:
		subscriptions (dict): Decoded subscription settings.
	"""

	# region Attributes

	subscriptions = []

	# endregion

	# region Constructors

	def __init__(self):
		load_dotenv()

		self.load_subscriptions()

	# endregion

	# region Functions

	def load_subscriptions(self):
		"""
		Reads in subscriptions.json and decodes all the settings
		into Subscription objects.
		"""

		with open('src/subscriptions.json', 'r') as myfile:
			subscription_encoded=myfile.read()

		subscriptions_decoded = json.loads(subscription_encoded)

		self.subscriptions = []

		for sub in subscriptions_decoded:
			self.subscriptions.append(Subscription(dict_config = sub))

	def process_subscription(self, subscription: Subscription):
		"""
		Runs youtube-dl and the post-processing for the given subscription.

		Parameters:
			subscription (Subscription): The subscription to process.
		"""

		if not subscription.enabled:
			return

		Logger.log(r'Chrysalis', r'Processing "{}"...'.format(subscription.name))

		self.setup_staging_directory(subscription)

		if subscription.logging and subscription.logging.path:
			pathlib.Path(subscription.logging.path).parent.mkdir(parents=True, exist_ok=True)

		command = self.construct_command(subscription)

		subprocess.run(command, shell=True)

	def setup_staging_directory(self, subscription: Subscription) -> str:
		"""
		Constructs and creates the staging directory for the given subscription.

		Parameters:
			subscription (Subscription): The subscription to process.

		Returns:
			str: The path to the staging directory.
		"""

		pathlib.Path(subscription.staging_directory).mkdir(parents=True, exist_ok=True)

		return subscription.staging_directory

	def construct_command(self, subscription: Subscription) -> str:
		"""
		Builds the youtube-dl command for the given subscription.

		Args:
			subscription (Subscription): The subscription to process.

		Returns:
			str: The youtube-dl command with all desired arguments.
		"""

		command =  r'youtube-dl'

		# Add the youtube-dl config path.
		if subscription.youtubedl_config.config:
			config_path = os.path.join(os.getenv('youtubedl_config_directory'), subscription.youtubedl_config.config)
			command += r' --config-location "{}"'.format(config_path)

		# Add the metadata-from-title pattern.
		if subscription.youtubedl_config.metadata_format:
			command += r' --metadata-from-title "{}"'.format(subscription.youtubedl_config.metadata_format)

		# Add the output pattern.
		if subscription.youtubedl_config.output_format:
			output_format = subscription.staging_directory + '/staging_area/' + subscription.youtubedl_config.output_format
			command += r' -o "{}"'.format(output_format)

		# Add the path to the video ID archive.
		if subscription.youtubedl_config.archive:
			archive_path = os.path.join(subscription.staging_directory, subscription.youtubedl_config.archive)
			command += r' --download-archive "{}"'.format(archive_path)

		# Add any extra arguments this sub has.
		if subscription.youtubedl_config.extra_commands:
			command += " " + subscription.youtubedl_config.extra_commands

		# Add the subscription URL.
		command += r' "{}"'.format(subscription.url)

		# Construct the post-processing call back into
		# Chrysalis to be run after each successful download.
		if subscription.post_processing:
			command += ' --exec \'"{}" "{}" --postprocess {{}} --subscription "{}"\''.format(
				sys.executable,
				__file__,
				subscription.name
			)

		# Construct the stdout redirect to the log file.
		if subscription.logging.path:
			command += r' {} "{}"'.format(
				'>>' if subscription.logging.append == True else '>',
				subscription.logging.path
			)

		Logger.log(r'Chrysalis', r'Command to be run: [{}]'.format(command))

		return command

	def postprocess(self, file: str, subscription: Subscription) -> str:
		"""
		Runs the post-processing for the given youtube-dl output file.

		Args:
			file (str): Absolute path to the youtube-dl output file.
			subscription (Subscription): The settings to process the file under.

		Returns:
			str: The absolute path to the folder where all the files were moved.
		"""

		from PostProcessor import PostProcessor

		Logger.log(r'Crysalis', r'Starting PostProcessor for {}'.format(file), 1)

		postprocessor = PostProcessor(
			file = file,
			settings = subscription
		)

		postprocessor.run()

		Logger.tabs -= 1

	def run(self) -> int:
		"""
		Entry point for the Chrysalis process.

		Returns:
			int: Status code.
		"""

		if args.postprocess is not None:
			subs = [item for item in self.subscriptions if item.name == args.subscription]

			subscription = subs[0] if subs else None

			if not subscription:
				return -1

			self.postprocess(args.postprocess, subscription)
		else:
			for subscription in self.subscriptions:
				self.process_subscription(subscription)

	# endregion

Chrysalis().run()


"""DataUpdateCoordinator for Plugwise."""
from datetime import timedelta

import async_timeout
from plugwise import Smile
from plugwise.exceptions import XMLDataMissingError

from homeassistant.core import HomeAssistant
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator, UpdateFailed

from .const import DEFAULT_SCAN_INTERVAL, DEFAULT_TIMEOUT, DOMAIN, LOGGER

class PlugwiseDataUpdateCoordinator(DataUpdateCoordinator[bool]):
    """Class to manage fetching Plugwise data from single endpoint."""

    def __init__(self, hass: HomeAssistant, api: Smile) -> None:
        """Initialize the coordinator."""
        super().__init__(
            hass,
            LOGGER,
            name=api.smile_name or DOMAIN,
            update_interval=DEFAULT_SCAN_INTERVAL.get(
                str(api.smile_type), timedelta(seconds=60)
            ),
        )
        self.api = api

    async def _async_update_data(self) -> bool:
        """Fetch data from Plugwise."""
        try:
            async with async_timeout.timeout(DEFAULT_TIMEOUT):
                await self.api.full_update_device()
        except XMLDataMissingError as err:
            raise UpdateFailed("Smile update failed") from err
        return True


# coding=utf-8
import os

import xlrd
import logging
from openpyxl import load_workbook
from copy import copy

logging.basicConfig(filename='mylog.txt', format="%(asctime)s : %(message)s",
                    level=logging.DEBUG)
# 读数据时，计算各指标值
sum_target_col_list = {'日期': '', '利用率平均值': 0, 'Fallback流程成功率': 0, '发送PDU Session Establishment Accept消息数': 0,
                       '接收PDU Session Establishment Request消息数': 0,
                       'N2模式二次寻呼响应次数': 0, 'N2模式寻呼请求次数': 0, 'N2模式一次寻呼响应次数': 0,
                       'AMF注册态最大用户数': 0, 'PGW-C通过GTP S5/S8/S2a/S2b创建的平均在线Session数': 0, 'SMF 5G 实时PDU会话数': 0, }

# 获取到各指标所在列（value），用于写数据时定位到具体cell
data_target_col_list = {'日期': '', '今日活动用户数(5G)': 0, '今日活动用户数(5G+4G)': 0, '昨日活动用户数(5G)': 0, '昨日活动用户数(5G+4G)': 0,
                        '新增日活数(5G)': 0, '新增日活数(5G+4G)': 0, 'SA寻呼成功率': 0, 'SA会话建立成功率': 0, 'EPS FB接通率': 0,
                        'AMF CPU负荷': 0, 'SMF CPU负荷': 0}

VMDic = {'AMF_CPU_Rate': 0, 'AMF_Count': 0, 'SMF_CPU_Rate': 0, 'SMF_Count': 0}

# 获取当前路径下所有xls和xlsx文件
def XLSFileList():
    filelist = []
    for root, dirs, files in os.walk(".", topdown=False):
        for name in files:
            str = os.path.join(root, name)
            if str.split('.')[-1] == 'xls' or str.split('.')[-1] == 'xlsx':
                filelist.append(str)
    return filelist

# 读取配置文件
def readConfig():
    try:
        confDic = {}
        file = open(os.getcwd() + '\\config.txt', "rb")
        for line in file.readlines():
            if 'region' in line.decode():
                confDic['region'] = line.decode().split('=')[1].strip('\n\t\r')
    except Exception as err:
        confDic['region'] = '广东'
        logging.error('readConfig function error : %s', err)
    return confDic

# 计算总和
def MaxNum(rown, coln, num_rows, worksheet):
    datalist = []
    for r in range(rown, num_rows):
        try:
            datalist.append(int(worksheet.cell_value(r, coln)))
        except Exception as err:
            logging.error('MaxNum function error : %s', err)
            continue
    return max(datalist)

# 计算平均数
def AverageNum(rown, coln, num_rows, worksheet):
    total = 0
    num = 0
    res = 0
    for r in range(rown, num_rows):
        try:
            total += int(worksheet.cell_value(r, coln))
            num = num + 1
        except Exception as err:
            logging.error('addTotal function error : %s', err)
            continue
    if num != 0:
        res = total / num
    return res

# 计算总和
def TotalNum(rown, coln, num_rows, worksheet):
    total = 0
    for r in range(rown, num_rows):
        try:
            total += int(worksheet.cell_value(r, coln))
        except Exception as err:
            logging.error('addTotal function error : %s', err)
            continue
    return total

# 计算平均值（取除0值）
def AverageNumWithoutZero(rown, coln, num_rows, worksheet):
    total = 0
    num = 0
    res = 0
    for r in range(rown, num_rows):
        try:
            if float(worksheet.cell_value(r, coln)) != 0:
                total += float(worksheet.cell_value(r, coln))
                num = num + 1
        except Exception as err:
            logging.error('addTotal function error : %s', err)
            continue
    if num != 0:
        res = total / num
    return res

# 读取模板文件，获取到要填的单元格，将计算的数据写入到目标单元格
def ReadAndAnalysis(XLSPath):
    logging.info('analytical %s start.', XLSPath)
    workbook = xlrd.open_workbook(XLSPath)  # 打开一个workbook
    worksheet = workbook.sheet_by_name(u'Sheet1')  # 定位到目标sheet
    num_rows = worksheet.nrows  # 获取该sheet中的有效行数
    num_cols = worksheet.ncols  # 获取列表的有效列数
    logging.info('begin analytical %s.', XLSPath)
    if '保存时间' in str(worksheet.cell_value(1, 0)):
        sum_target_col_list['日期'] = '.'.join(str(worksheet.cell_value(1, 0)).split(' ')[1].split('-')[1:3])

    # 解析xls
    row = 0
    col = 0
    flag = 0
    Nrow = 0
    Ncol = 0
    Nflag = 0
    title = ''
    for rown in range(0, num_rows):
        for coln in range(num_cols):
            # 解析指定列
            for tclo in sum_target_col_list:
                if tclo in str(worksheet.cell_value(rown, coln)) and coln > 2:
                    if 'AMF注册态最大用户数' == tclo or '实时PDU会话数' in tclo or '创建的平均在线Session数' in tclo:
                        Nrow = rown + 1
                        UserNumList = []
                        while Nrow < num_rows:
                            UserNumList.append(
                                int(worksheet.cell_value(Nrow, coln)) + int(worksheet.cell_value(Nrow + 1, coln)))
                            Nrow = Nrow + 2
                        if len(UserNumList):
                            UserNumList.sort(reverse=True)
                            sum_target_col_list[tclo] = UserNumList[0]
                        break
                    if 'N2模式二次寻呼响应次数' in tclo or 'N2模式寻呼请求次数' in tclo or 'N2模式一次寻呼响应次数' in tclo:
                        sum_target_col_list[tclo] = AverageNum(rown + 1, coln, num_rows, worksheet)
                        break
                    if 'PDU Session Establishment' in tclo:
                        sum_target_col_list[tclo] = TotalNum(rown + 1, coln, num_rows, worksheet)
                        break
                    if 'Fallback流程成功率' in tclo:
                        sum_target_col_list[tclo] = AverageNumWithoutZero(rown + 1, coln, num_rows, worksheet)
                        break
                    if '利用率平均值' in tclo:
                        flag = 1
                        row = rown + 1
                        col = coln
                        break
    if flag == 1:
        for r in range(row, num_rows):
            if 'SMF' in str(worksheet.cell_value(r, col - 2)):
                VMDic['SMF_CPU_Rate'] = VMDic['SMF_CPU_Rate'] + float(worksheet.cell_value(r, col))
                VMDic['SMF_Count'] = VMDic['SMF_Count'] + 1
            if 'AMF' in str(worksheet.cell_value(r, col - 2)):
                VMDic['AMF_CPU_Rate'] = VMDic['AMF_CPU_Rate'] + float(worksheet.cell_value(r, col))
                VMDic['AMF_Count'] = VMDic['AMF_Count'] + 1
        # print(UserNumList)
    logging.info('analytical complete.', )

def ReadAndWrite(XLSPath):
    # 打开一个workbook
    workbook = xlrd.open_workbook(XLSPath)
    # 抓取所有sheet页的名称
    worksheets = workbook.sheet_names()
    # 定位到目标sheet
    worksheet = workbook.sheet_by_name(worksheets[0])
    # 获取该sheet中的有效行数
    num_rows = worksheet.nrows
    # 获取列表的有效列数
    num_cols = worksheet.ncols

    # 打开excel
    wb = load_workbook(XLSPath)
    # 获取工作表
    for name in wb.sheetnames:
        if 'Sheet1' in name:
            sheet = wb[name]

    logging.info('write %s start.', XLSPath)
    for row in range(num_rows):
        CopyCellFont(sheet, row + 1, num_cols + 1)
    sheet.cell(1, num_cols + 1).value = data_target_col_list['日期']
    sheet.cell(2, num_cols + 1).value = data_target_col_list['今日活动用户数(5G)']
    sheet.cell(3, num_cols + 1).value = data_target_col_list['今日活动用户数(5G+4G)']
    sheet.cell(4, num_cols + 1).value = sheet.cell(2, num_cols).value
    sheet.cell(5, num_cols + 1).value = sheet.cell(3, num_cols).value
    sheet.cell(6, num_cols + 1).value = str(data_target_col_list['今日活动用户数(5G)'] - sheet.cell(2, num_cols).value)
    sheet.cell(7, num_cols + 1).value = str(data_target_col_list['今日活动用户数(5G+4G)'] - sheet.cell(3, num_cols).value)
    if data_target_col_list['SA寻呼成功率'] != 0:
        sheet.cell(8, num_cols + 1).value = data_target_col_list['SA寻呼成功率']
    if data_target_col_list['SA会话建立成功率'] != 0:
        sheet.cell(9, num_cols + 1).value = data_target_col_list['SA会话建立成功率']
    sheet.cell(10, num_cols + 1).value = '*(' + str(data_target_col_list['EPS FB接通率']) + ')'
    sheet.cell(11, num_cols + 1).value = str(data_target_col_list['AMF CPU负荷']) + '/' + str(
        data_target_col_list['SMF CPU负荷'])

    wb.save(XLSPath)
    logging.info('write %s end.', XLSPath)

def CopyCellFont(sheet, writeRow, writeCol):
    sheet.cell(writeRow, writeCol).font = copy(sheet.cell(writeRow, 6).font)
    sheet.cell(writeRow, writeCol).border = copy(sheet.cell(writeRow, 6).border)
    sheet.cell(writeRow, writeCol).fill = copy(sheet.cell(writeRow, 6).fill)
    sheet.cell(writeRow, writeCol).number_format = copy(sheet.cell(writeRow, 6).number_format)
    sheet.cell(writeRow, writeCol).protection = copy(sheet.cell(writeRow, 6).protection)
    sheet.cell(writeRow, writeCol).alignment = copy(sheet.cell(writeRow, 6).alignment)

def XLSReadAndAnalysis():
    logging.info('begin to read file')
    for xfl in XLSFileList():
        ReadAndAnalysis(xfl)
    # for i in sum_target_col_list:
    #     print(i, sum_target_col_list[i])

def XLSReadAndWrite():
    logging.info('begin to write file')
    for xfl in XLSFileList():
        if '指标汇总' in xfl:
            ReadAndWrite(xfl)

def ConvertData():
    logging.info('begin to ConvertData')
    data_target_col_list['日期'] = sum_target_col_list['日期']
    data_target_col_list['今日活动用户数(5G)'] = sum_target_col_list['AMF注册态最大用户数']
    data_target_col_list['今日活动用户数(5G+4G)'] = sum_target_col_list['PGW-C通过GTP S5/S8/S2a/S2b创建的平均在线Session数'] + \
                                             sum_target_col_list['SMF 5G 实时PDU会话数']
    if sum_target_col_list['N2模式寻呼请求次数'] != 0:
        data_target_col_list['SA寻呼成功率'] = str(round(
            (sum_target_col_list['N2模式二次寻呼响应次数'] + sum_target_col_list['N2模式一次寻呼响应次数']) / \
            sum_target_col_list['N2模式寻呼请求次数'] * 100, 2)) + '%'
    if sum_target_col_list['接收PDU Session Establishment Request消息数'] != 0:
        data_target_col_list['SA会话建立成功率'] = str(round(
            sum_target_col_list['发送PDU Session Establishment Accept消息数'] / sum_target_col_list[
                '接收PDU Session Establishment Request消息数'] * 100, 2)) + '%'
    data_target_col_list['EPS FB接通率'] = str(round(sum_target_col_list['Fallback流程成功率'], 2)) + '%'
    if VMDic['AMF_Count'] != 0:
        data_target_col_list['AMF CPU负荷'] = str(round(VMDic['AMF_CPU_Rate'] / VMDic['AMF_Count'], 2)) + '%'
    if VMDic['SMF_Count'] != 0:
        data_target_col_list['SMF CPU负荷'] = str(round(VMDic['SMF_CPU_Rate'] / VMDic['SMF_Count'], 2)) + '%'
    # for i in data_target_col_list:
    #     print(i, ' ', data_target_col_list[i])
    logging.info('end ConvertData')

def main():
    # 解析xls文件到list，用于后续数据处理数据源
    logging.info('welcome to SA Commercial Support world.')
    try:
        XLSReadAndAnalysis()
        ConvertData()
        XLSReadAndWrite()
    except Exception as err:
        logging.error('error: %s', err)

    logging.info("end SA Commercial Support  world")

if __name__ == '__main__':
    main()


"""
P4Join algorithm

Paper:
Sehili, Ziad, et al.
"Privacy preserving record linkage with PPJoin."
Datenbanksysteme für Business, Technologie und Web (BTW 2015) (2015).

Implemented by GreatYYX https://github.com/greatyyx
"""
import collections
from itertools import product
from functools import reduce
from typing import List, Tuple, Set
import hashlib
import hmac
from ppjoin.ppjoin_ import ceil

def list_to_vec(l):
    vec = 0
    for idx, e in enumerate(reversed(l)):
        vec |= (e << idx)
    return vec

def vec_to_list(vec, l_len):
    l = [0] * l_len
    idx = l_len - 1
    while vec != 0 and l_len >= 0:
        l[idx] = vec & 1
        vec >>= 1; idx -= 1
    return l

def str_to_byte(s):
    return s.encode('utf-8')

def byte_to_str(b):
    return b.decode('utf-8')

def all_sb_idx(b, vec_len):
    """
    Get set-bit indices
    """
    l = []
    for idx in reversed(range(vec_len)):
        if b & 1:
            l.append(idx)
        b >>= 1
    return list(reversed(l))

def set_bit(b, vec_len, idx):
    return b | 1 << (vec_len - 1 - idx)

def base_hash(key, msg, method):
    return int(hmac.new(key=key, msg=msg, digestmod=method).hexdigest(), 16)

def encode_record(record: List[List[str]], hmac_key: str, vec_len: int, k: int = 2) -> List[int]:
    hmac_key = str_to_byte(hmac_key)
    vec = 0
    for t in record:
        t = str_to_byte(t)
        for i in range(1, k+1):
            set_bit_idx = (
                base_hash(key=hmac_key, msg=t, method=hashlib.sha1) +
                base_hash(key=hmac_key, msg=t, method=hashlib.md5) * i
            ) % vec_len
            vec = set_bit(vec, vec_len, set_bit_idx)
    return vec

def prefix(vec, vec_len, t):
    sb_idx = all_sb_idx(vec, vec_len)
    # prefix_length = ceil((1 - t) * len(sb_idx)) + 1
    prefix_length = len(sb_idx) - ceil(t * len(sb_idx)) + 1
    prefix_length = min(prefix_length, len(sb_idx))
    prefix_sb_idx = sb_idx[:prefix_length]
    prefix_vec = map(lambda x: set_bit(0, vec_len, x), prefix_sb_idx[:])
    return reduce(lambda x, y: x | y, prefix_vec)

def compare(records, vec_len, t, order_map):
    cp = set()
    lmap = collections.defaultdict(set)

    if t == 0:
        return set(filter(lambda x: x[0] != x[1], product(range(len(records)), range(len(records)))))

    for xr_idx, xr in enumerate(records):
        xl = len(all_sb_idx(xr, vec_len))
        for el in list(lmap.keys()):

            if el < xl * t:  # length filter
                del lmap[el]
                continue

            for (yr_idx, yr) in lmap[el]:
                xp = prefix(xr, vec_len, t)
                yp = prefix(yr, vec_len, t)
                if xp & yp == 0:  # prefix filter
                    continue

                yl = len(all_sb_idx(yr, vec_len))
                if positional_filter(xp, yp, xl, yl, t, vec_len):
                    continue

                score = jaccard(xr, yr, vec_len)
                if score >= t:
                    cp.add((xr_idx, yr_idx))

        lmap[xl].add((xr_idx, xr))

    return cp

def positional_filter(xp, yp, xl, yl, t, vec_len):
    overlap = len(all_sb_idx(xp & yp, vec_len))
    sb_idx1 = all_sb_idx(xp, vec_len)
    sb_idx2 = all_sb_idx(yp, vec_len)
    p1, p2 = sb_idx1[-1], sb_idx2[-1]
    diff1, diff2 = 0, 0

    if p1 > p2:
        diff1 = len([sb for sb in sb_idx1 if sb > p2])
    else:
        diff2 = len([sb for sb in sb_idx2 if sb > p1])

    rest = min(xl - len(sb_idx1) + diff1, yl - len(sb_idx2) + diff2)

    return overlap + rest < ceil((xl + yl) * t / (t + 1))

def preprocess(records, vec_len):
    # get all set bits index of records
    records_sb_idx = []
    for vec in records:
        records_sb_idx.append(all_sb_idx(vec, vec_len))

    # get frequency order of index of all set bits
    elements = [e for r in records_sb_idx for e in r]
    order_map = dict(
        (el, i)
        for i, (el, count) in enumerate(sorted(collections.Counter(elements).items(), key=lambda x: (x[1], x[0])))
    )  # (element, order)

    # reorder set bit of all records
    reordered_records = []
    for vec_sb_idx in records_sb_idx:
        vec = 0
        for set_bit_idx in sorted(vec_sb_idx, key=lambda x: order_map[x]):
            vec = set_bit(vec, vec_len, set_bit_idx)
        reordered_records.append(vec)

    # sort reordered records based on cardinality
    argsort = sorted(range(len(reordered_records)), key=lambda i: len(all_sb_idx(reordered_records[i], vec_len)))
    reordered_records.sort(key=lambda r: len(all_sb_idx(r, vec_len)))

    return reordered_records, argsort, order_map

def jaccard(n1, n2, vec_len):
    return 1.0 * len(all_sb_idx(n1 & n2, vec_len)) / len(all_sb_idx(n1 | n2, vec_len))

def join(datasets: List[List[int]], t: float = 0, vec_len: int = 0) -> Set[Tuple[Tuple]]:
    ret = set()
    if not datasets:
        return ret

    dataset = []
    dataset_id_offset = [0]
    for d in datasets:
        dataset += d
        dataset_id_offset.append(len(d) + dataset_id_offset[-1])
    if len(dataset_id_offset) > 1:
        dataset_id_offset = dataset_id_offset[:-1]

    records_sorted, original_order, order_map = preprocess(dataset, vec_len)
    result = compare(records_sorted, vec_len, t, order_map)

    for r in result:
        r1id, r2id = r[0], r[1]
        r1id, r2id = original_order[r1id], original_order[r2id]
        if r1id == r2id:
            continue

        # r1id should <= r2id
        if r1id > r2id:
            r1id, r2id = r2id, r1id
        # find which original datasets the rids belong to
        ds1_offset = next(x for x in reversed(dataset_id_offset) if x <= r1id)
        ds2_offset = next(x for x in reversed(dataset_id_offset) if x <= r2id)
        # both are from one source (except only one dataset is provided)
        if len(dataset_id_offset) > 1 and ds1_offset == ds2_offset:
            continue

        ret.add((
            (dataset_id_offset.index(ds1_offset), r1id - ds1_offset),
            (dataset_id_offset.index(ds2_offset), r2id - ds2_offset)
        ))

    return ret


from functools import reduce
import time
import functools
import random

def triplc(lst):
    return [x*x for x in lst]

def sum(lst):
    return reduce(lambda x,y: x + y,lst)

def rf(l):
    with open('a.txt', 'r') as f:
        while True:
            c = f.read(l)
            if c == '':
                break
            yield c

def has_null_value(lst):
    for x in lst:
        if x == None:
            return False

def tf(x):
    if x:
        print('t')
        return True
    else:
        print('f')
        return False

def has_null_value1(lst):
    t = list(filter(tf, lst))
    if len(t) != len(lst):
        return False

def f(n):
    if n<=1:
        return n
    else:
        return (f(n-1) + f(n-2))

def fib():
    for x in range(10):
        print(f(x))

def c_password(leng,*,l_letters=False,u_letters=False,numbers=False,sp_char=False):
    print(leng, "length:", "###l_letters=", l_letters, "###u_letters=", u_letters, "###numbers=", numbers, "###sp_char=", sp_char)
    k = 0
    l_l = 0
    u_l = 0
    n = 0
    s_ch = 0

    l_lc = ''
    u_lc = ''
    n_c = ''
    s_chc = ''

    spchar = []

    kd = [l_letters,u_letters,numbers,sp_char]
    for x in kd:
        if x:
            k += 1
    if l_letters:
        l_l = random.randint(1, leng)
    if u_letters:
        u_l = random.randint(1, leng-l_l)
    if numbers:
        n = random.randint(1, leng-(l_l+u_l))
    if sp_char:
        s_ch = leng-(l_l+u_l+n)
    print(l_l,u_l,n,s_ch)

    while l_l:
        l_lc += chr(random.randint(97, 122))
        l_l -= 1

    while u_l:
        l_lc += chr(random.randint(65, 90))
        l_l -= 1

    while n:
        n_c += str(random.randint(0, 9))
        n -= 1

    for x in "~!@#$%^&*()":
        spchar.append(x)

    while s_ch:
        r = random.randint(0, 9)
        s_chc += spchar[r]
        s_ch -= 1

    return l_lc + u_lc + n_c + s_chc

mul_by_two = functools.partial(mul, 2)

def timing(func):
    @functools.wraps(func)
    def wrapper(*args, **kw):
        time_start=time.time()
        rs = func(*args, **kw)

        print((time.time() - time_start))

    return wrapper

@timing
def add(x,y,z):
        print("gaga")
        return x+y+z


import argparse
import os, sys, stat

def change_permissions(files_to_convert):
    """Set permissions for files using os.chmod

    Parameters
    ----------
    files_to_convert : list like
        List of files to change permissions of.
    """
    for f in files_to_convert:
        os.chmod(f, stat.S_IREAD|
                    stat.S_IWRITE|
                    stat.S_IEXEC|
                    stat.S_IRGRP|
                    stat.S_IWGRP|
                    stat.S_IXGRP)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--files', default=None, dest='files',
                        nargs='+', help='Path of files')
    args = parser.parse_args()

    change_permissions(args.files)

if __name__ == '__main__':
    main()


import datetime
import os
import csv
import pdb

# use pdb.set_trace() to set a breakpoint

'''
finish is mandatory
'''
# checks if mandatory

def is_mandatory(input):
    print("temporary")

# checks if numeric only
def is_numeric(input):
    if input.isnumeric() is True:
        return True
    else:
        return False

# checks if provided length is true
def is_valid_length(input, length):
    # len() outputs an integer so need to cast length to int
    if (len(str(input)) == int(length)):
        return True
    else:
        return False

# checks if less than or equal to specified length
def is_less_than_eq_to_length(input, length):
    # len() outputs an integer so need to cast length to int
    if (len(str(input)) <= int(length)):
        return True
    else:
        return False

# checks to see if there are no nulls, empty string, all spaces, or all zeros
def no_missing_values(input):
    output = True
    # check for all zeros nulls, or white spaces
    if input is None:
        output = False
    elif input is '':
        output = False
    elif input.isspace():
        output = False
    elif input.isnumeric():
        if int(input) == 0:
            output = False
    return output

# def trim function no leading and trailing spaces
def trim(input):
    return input.strip()

# check if valid aleph year (1980 - Present)

def is_valid_aleph_year(year):
    current_year = datetime.datetime.now().year
    if year not in range(1980, current_year+1):
        return False
    else:
        return True

def no_leading_space(input):
    if input[0].isspace():
        return False
    else:
        return True

# need to make sure null passes missing value data quality
def is_valid_aleph_date(string_date):
    string_date_valid = datetime.datetime.strptime(
        string_date, '%Y%m%d').strftime('%Y%m%d')
    if string_date == string_date_valid and is_valid_aleph_year(string_date):
        return True
    else:
        return False

# # checks if hour is valid HHMM format
def is_valid_hour(hhmm):
    hhmm_form = ('{0:04d}'.format(hhmm))
    if int(str(hhmm_form)[0:2]) in range(00,25):
        return True
    else:
        return False
            #^^looking at z35-event-hour goes from xxnn to nnnn, not sure how to parse this for our check

# function create a python dict from a 2 column CSV lookup table in the lookup_tables directory

def create_dict_from_csv(csv_file):
    csv_file_path = os.path.join('lookup_tables', csv_file)
    with open(csv_file_path, mode='r') as f:
        reader = csv.DictReader(f, fieldnames=('code', 'description'))
        csv_dict = {}
        for row in reader:
            csv_dict[row['code']] = row['description']
        return csv_dict

#^^^^ will we make this an absolute path to lookup_tables?
#^^^^ will this only work on 2 column CSVs? Any situation where this would need to be extensible?

def create_dict_from_csv_redux(csv_file):
#    csv_file_path = os.path.join('lookup_tables', csv_file)
    with open(csv_file, mode='r') as f:
        reader = csv.DictReader(f, fieldnames=('code', 'description'))
        csv_dict = {}
        for row in reader:
            csv_dict[row['code']] = row['description']
        return csv_dict


import json
import os
import sys

from collections import defaultdict, OrderedDict
from typing import DefaultDict, List

def make_data_dict(filename, in_dir):
    entry_set : DefaultDict[str, list()] = defaultdict(lambda: list())

    with open(in_dir + filename, "r", encoding='utf8') as f:
        data_list = f.readlines()

        for line in data_list:
            entry = json.loads(line)

            if entry["label"] != "UNDEFINED":
              entry_set[entry["label"]].append(entry)

    return entry_set

def map_chemprot_labels_to_custom_labels(cpl_set):
    '''
    Maps ChemProt relation-labels to Custom relation-labels
    cid   Custom class                  cpr     ChemProt
    -------------------------------------------------------------------------------------
    0   NOT                             10      NOT
    1   PART-OF                         1       PART-OF
    2   INTERACTOR                      2       REGULATOR
                                        2       DIRECT-REGULATOR
                                        2       INDIRECT-REGULATOR
                                        5       AGONIST
                                        7       MODULATOR
                                        8       CO-FACTOR
                                        9       SUBSTRATE
    3   REGULATOR-POSITIVE              3       UPREGULATOR
                                        3       ACTIVATOR
                                        3       INDIRECT-UPREGULATOR
                                        5       AGONIST-ACTIVATOR
                                        7       MODULATOR-ACTIVATOR
    4   REGULATOR-NEGATIVE              4       DOWNREGULATOR
                                        4       INHIBITOR
                                        4       INDIRECT-DOWNREGULATOR
                                        6       ANTAGONIST
                                        7       MODULATOR-INHIBITOR
                                        5       AGONIST-INHIBITOR
    5   OTHER                                   all labels not included above
    '''

    cl_to_cid = {"NOT":                 "0",
                 "PART-OF":             "1",
                 "INTERACTOR":          "2",
                 "REGULATOR-POSITIVE":  "3",
                 "REGULATOR-NEGATIVE":  "4",

                 "OTHER":               "5"
                 }

    cpl_to_cl = {"PART-OF":                 "PART-OF",

                 "REGULATOR":               "INTERACTOR",
                 "DIRECT-REGULATOR":        "INTERACTOR",
                 "INDIRECT-REGULATOR":      "INTERACTOR",
                 "MODULATOR":               "INTERACTOR",
                 "COFACTOR":                "INTERACTOR",
                 "SUBSTRATE":               "INTERACTOR",
                 "SUBSTRATE_PRODUCT-OF":    "INTERACTOR",
                 "PRODUCT-OF":              "INTERACTOR",
                 "AGONIST":                 "INTERACTOR",

                 "UPREGULATOR":             "REGULATOR-POSITIVE",
                 "ACTIVATOR":               "REGULATOR-POSITIVE",
                 "INDIRECT-UPREGULATOR":    "REGULATOR-POSITIVE",
                 "AGONIST-ACTIVATOR":       "REGULATOR-POSITIVE",
                 "MODULATOR-ACTIVATOR":     "REGULATOR-POSITIVE",

                 "DOWNREGULATOR":           "REGULATOR-NEGATIVE",
                 "INHIBITOR":               "REGULATOR-NEGATIVE",
                 "INDIRECT-DOWNREGULATOR":  "REGULATOR-NEGATIVE",
                 "ANTAGONIST":              "REGULATOR-NEGATIVE",
                 "MODULATOR-INHIBITOR":     "REGULATOR-NEGATIVE",
                 "AGONIST-INHIBITOR":       "REGULATOR-NEGATIVE",

                 "NOT":                     "NOT",

                 "UNDEFINED":               "OTHER"
                 }

    cl_set: DefaultDict[str, list()] = defaultdict(lambda: list())

    for label in cpl_set:
        for entry in cpl_set[label]:
            chemprot_label = entry["label"]
            custom_label = cpl_to_cl[chemprot_label]
            custom_id = cl_to_cid[custom_label]

            entry["cid"] = custom_id
            entry["custom_label"] = custom_label

            cl_set[custom_label].append(entry)

    counter = {
        "PART-OF": 0,
        "INTERACTOR": 0,
        "REGULATOR-POSITIVE": 0,
        "REGULATOR-NEGATIVE": 0,
        "NOT": 0,
        "OTHER": 0
    }

    return cl_set

def write_files(entry_set, filename, out_dir):
  result_set = list()

  statistics: DefaultDict[str, defaultdict()] = defaultdict(lambda: defaultdict(int))
  master_set: DefaultDict[str, list()] = defaultdict(lambda: list())

  for custom_label in entry_set:
      for i, entry in enumerate(entry_set[custom_label]):
          master_set[custom_label].append(entry)

      label_list = master_set[custom_label]
      result_set += label_list

      length = len(label_list)
      dataset = os.path.splitext(filename)[0]

      statistics[dataset][custom_label]   = length
      statistics[dataset]["total"]       += length

  with open(out_dir + filename, "w", encoding='utf8') as out_file:

      for entry in result_set[:-1]:
          out_file.write(json.dumps(entry, ensure_ascii=False) + "\n")

      out_file.write(json.dumps(result_set[-1], ensure_ascii=False))

  with open(out_dir + "statistics.txt", "a", encoding='utf8') as stats:

      for set in sorted(statistics):
        stats.write(set + "_set\n" + "-"*50 + "\n")
        stats.write("label" + " "*25 + "count" + "\t" + "%\n")
        stats.write("- "*25 + "\n")
        stats.write("total" + " "*25 + str(statistics[set]["total"]) + "\t" +
                    str(round(100*statistics[set]["total"]/statistics[set]["total"], 2)) + "\n")

        for custom_label in statistics[set]:
            if custom_label == "total":
                continue

            stats.write(custom_label + " "*(30-len(custom_label)) + str(statistics[set][custom_label]) + "\t" +
                        str(round(100*statistics[set][custom_label]/statistics[set]["total"], 2)) + "\n")
        stats.write("-"*50 + "\n\n")

# Pass directory with processed files and output directory
def run(input_path, output_path):
  io_files = [("chemprot_training_processed.txt", "chemprot_train.txt"),
  ("chemprot_development_processed.txt", "chemprot_dev.txt")]

  for in_file, out_file in io_files:
    entry_set = make_data_dict(in_file, input_path)
    custom_entry_set = map_chemprot_labels_to_custom_labels(entry_set)
    write_files(custom_entry_set, out_file, output_path)


# 12_01_multiLinear02.py
import numpy as np
from sklearn.model_selection import train_test_split
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense

filename = 'multiLinear02.csv'
data = np.loadtxt(filename, delimiter=',')

table_col = data.shape[1]
y_column = 1
x_column = table_col - y_column

x = data[:, 0:x_column]
y = data[:, x_column:]
print(type(x))
print(y)

seed = 0
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=seed)

model = Sequential()
model.add(Dense(input_dim=x_column, units=y_column, activation='linear'))
model.compile(loss='mse', optimizer='adam')
model.fit(x_train, y_train, epochs=5000, batch_size=100, verbose=0)
print(model.get_weights())
prediction = model.predict(x_test)

for idx in range(len(y_test)):
    label = y_test[idx] # 정답 데이터
    pred = prediction[idx] # 예측치
    print(f'real : {label}, prediction : {pred}')