from ctypes.wintypes import HPALETTE
import json
from urllib import request, response
from django.http import HttpResponse, JsonResponse
from django.shortcuts import render
from django.views import View
from django.views.decorators.csrf import csrf_exempt
from django.utils.decorators import method_decorator
from .calculation import *

from .models import Geometria

# Create your views here.

class Vista(View):
    @method_decorator(csrf_exempt)
    def dispatch(self, request, *args, **kwargs):
        return super().dispatch(request, *args, **kwargs)

    def get(self,request):

        listGet=list(Geometria.objects.values())
        #print(listGet)
        thisdict = {
            "type": "FeatureCollection",
        }
        miLista=[]
        for a in listGet:
            miLista.append({
                "type": "Feature",
                "properties":{
                    "color":a["class_color"],
                    "info":a["class_info"],
                    "strokeWeight":a["class_strokeWeight"]},
                "geometry":{
                    "type":a["class_type"],
                    "coordinates":json.loads(a["class_coordinates"])
                    }
                }
            ),
        thisdict["features"]=miLista
        return JsonResponse(thisdict)

    def post(self,request):
        dictEntrada = json.loads(request.body)

        ##############################
        # tipo de bomba
        ##############################
        r_destruccion = 0
        r_radiacion = 0
        a = "big bomb" # dictEntrada['properties']['bomba']
        if(a == "big bomb"):
            r_destruccion = 2
            r_radiacion = 8
        elif (a =="medium bomb"):
            r_destruccion = 10
            r_radiacion = 20

        ##############################
        # agregar radio de destrcucion
        ##############################
        circulo = genera_circulo_2(
            float(dictEntrada["latitud"]),
            float(dictEntrada["longitud"]),
            r_destruccion)

        Geometria.objects.create(
            class_info = 'generado por punto del cliente',
            class_color = 'red',
            class_coordinates = circulo,
            class_type = 'Polygon'
            )
        ##############################
        # agregar radio de radiacion
        ##############################
        circulo = genera_circulo_2(
            float(dictEntrada["latitud"]),
            float(dictEntrada["longitud"]),
            r_radiacion)

        Geometria.objects.create(
            class_info = 'generado por punto del cliente',
            class_color = 'yellow',
            class_coordinates = circulo,
            class_type = 'Polygon'
            )

        ##############################
        # agregar punto ingresado por usuario
        ##############################
        Geometria.objects.create(
            class_info = "si info",# dictEntrada['properties']['info'],
            class_color = "blue", # dictEntrada['properties']['color'],
            class_coordinates = [ float(dictEntrada["longitud"]), float(dictEntrada["latitud"]) ],
            #dictEntrada['geometry']['coordinates'],
            class_type = "Point"#dictEntrada['geometry']['type']
        )

        ##############################
        # respuesta al cliente
        ##############################
        respuesta = {
	        "type": "Feature",
            "properties": {
		        "mensaje": "Tudu ven! "
	        }
        }
        print(type(JsonResponse(dictEntrada)))
        return JsonResponse(dictEntrada)

    def delete(self,request):
        Geometria.objects.filter(id=7).delete()
        respuesta = {
	        "type": "Feature",
            "properties": {
		        "mensaje": "se borro! "
	        }
        }
        return JsonResponse(respuesta)

    def put(self,request, *args, **kwargs):
        muer = 0
        hola=json.loads(request.body)
        #print(hola)
        if(len(Geometria.objects.filter(id="1")) == 0): # punto
            Geometria.objects.create(
                id=1,
                class_type="Point",
                class_coordinates=[-77.0282400,-12.0431800])
        elif(len(Geometria.objects.filter(id="2")) == 0): # destruccion
            Geometria.objects.create(
                id=2,
                class_type="Polygon",
                class_color="red",
                class_coordinates=[[ [-77.0282400,-12.0431800], [-87.0282400,-12.0431800], [-87.0282400,-22.0431800], [-77.0282400,-12.0431800] ]],
                class_strokeWeight =1 )
        elif(len(Geometria.objects.filter(id="3")) == 0): # radiacion
            Geometria.objects.create(
                id=3,
                class_type="Polygon",
                class_color="yellow",
                class_coordinates=[[ [-77.0282400,-12.0431800], [-87.0282400,-12.0431800], [-87.0282400,-22.0431800], [-77.0282400,-12.0431800] ]],
                class_strokeWeight = 1)
        elif(len(Geometria.objects.filter(id="4")) == 0): # interseccion
            Geometria.objects.create(
                id=4,
                class_type="Polygon",
                class_color="green",
                class_coordinates=[[ [-77.0282400,-12.0431800], [-87.0282400,-12.0431800], [-87.0282400,-22.0431800], [-77.0282400,-12.0431800] ]],
                class_strokeWeight = 4)
        else:
            if(hola["properties"]["bomba"] == "termo"):
                radio_radiacion=10
                radio_destruccion=4
            elif (hola["properties"]["bomba"] == "neutro"):
                radio_radiacion=20
                radio_destruccion=10
            else:
                radio_radiacion=2
                radio_destruccion=1
            ##############################
            # actualiza punto
            ##############################
            b = float(hola['geometry']['coordinates'][0])
            a = float(hola['geometry']['coordinates'][1])
            Geometria.objects.filter(id=1).update(
                class_coordinates=[ b,a ]
                )
            ##############################
            # actualiza circulo de destruccion
            ##############################
            circulo_d = genera_circulo_2(a,b,radio_destruccion)
            Geometria.objects.filter(id=2).update(
                class_coordinates=circulo_d
                )
            ##############################
            # actualiza circulo de radiacion
            ##############################
            circulo_r = genera_circulo_2(a,b,radio_radiacion)
            Geometria.objects.filter(id=3).update(
                class_coordinates=circulo_r
                )
            ##############################
            # calculando interseccion
            ##############################
            geo = json.loads(Geometria.objects.filter(id=5).get().class_coordinates)
            p_list_1 = conversor_poligonoGoogle_a_poligonoShapely(geo[0])
            p_list_2 = conversor_poligonoGoogle_a_poligonoShapely(circulo_d[0])
            p_1 = lista_a_poligono(p_list_1)
            p_2 = lista_a_poligono(p_list_2)
            inter = interseccion(p_1,p_2)
            mi_lista = poligono_a_lista_shapely(inter)
            salida = conversor_poligonoShapely_a_poligonoGoogle(mi_lista)
            Geometria.objects.filter(id=4).update(
                class_coordinates=[salida]
            )
            ##############################
            # calculando muertos
            ##############################
            muer = muertos(32970000,area(inter),area(p_1))

        respuesta = {
	        "type": "Feature",
            "properties": {
		        "mensaje": "se hizo update! ",
                "muertos": muer
	        }
        }
        return JsonResponse(respuesta)


# deseasonlize and normalize the summertime MDA8O3 from 2014 to 2021 in BTH, YRD and PRD regions

import numpy as np
from netCDF4 import Dataset

def moving_avg(data):

    data_avg = np.zeros((92))
    for m in range(0,len(data)-21):
        data_avg[m] = data[m+21]-np.nanmean(data[m:m+21])
    return data_avg

def main():

    file_o3 = '/home/lss/data/O3_meteo/data_extract/vc_sta_std/CNEMC/CNEMC_MDA8_O3.nc'
    dataset = Dataset(file_o3)
    o3_bth = dataset.variables['O3_bth'][:]
    o3_yrd = dataset.variables['O3_yrd'][:]
    o3_prd = dataset.variables['O3_prd'][:]
    dataset.close()
    O3 = np.append(o3_bth,o3_yrd,axis=1)
    O3 = np.append(O3,o3_prd,axis=1)

    O3_avg = np.zeros((8,len(O3[0]),92))
    O3_de = np.zeros((8,len(O3[0]),92))
    for j in range(0,len(O3[0])):
        for i in range(0,len(O3)):
            O3_avg[i,j,:] = moving_avg(O3[i,j,:])
        O3_de[0:4,j] = O3_avg[0:4,j]/np.nanstd(O3_avg[0:4,j])
        index = np.where(O3_de[0:4,j]>np.nanpercentile(O3_de[0:4,j],90))
        O3_de[4:,j] = O3_avg[4:,j]/np.nanstd(O3_avg[4:,j])
        index = np.where(O3_de[4:,j]>np.nanpercentile(O3_de[4:,j],90))

    file_output = '/home/lss/data/O3_meteo/data_extract/vc_sta_std/CNEMC/CNEMC_ND_MDA8_O3.nc'
    f_w = Dataset(file_output,'w',format='NETCDF4')
    f_w.createDimension('year',8)
    f_w.createDimension('days',92)
    f_w.createDimension('BTH',len(o3_bth[0]))
    f_w.createDimension('YRD',len(o3_yrd[0]))
    f_w.createDimension('PRD',len(o3_prd[0]))
    f_w.createVariable('O3_bth',np.float32,('year','BTH','days'))
    f_w.createVariable('O3_yrd',np.float32,('year','YRD','days'))
    f_w.createVariable('O3_prd',np.float32,('year','PRD','days'))
    f_w.variables['O3_bth'][:] = O3_de[:,0:len(o3_bth[0]),:]
    f_w.variables['O3_yrd'][:] = O3_de[:,len(o3_bth[0]):len(o3_bth[0])+len(o3_yrd[0]),:]
    f_w.variables['O3_prd'][:] = O3_de[:,len(o3_bth[0])+len(o3_yrd[0]):len(o3_bth[0])+len(o3_yrd[0])+len(o3_prd[0]),:]
    f_w.close()

if __name__=='__main__':
    main()


from discord.colour import Colour
from discord.ext import commands
from discord import Embed
from discord.ext import commands

class Utils(commands.Cog):
    def __init__(self,bot):
        self.bot=bot
        bot.remove_command('help')

    @commands.command(aliases=["ajuda"])
    async def help(self,ctx,argument=None):
        embed = Embed(colour=Colour.blurple())
        if argument is None:
            embed.title = "Comandos"
            embed.description = ".ajuda\n.moeda\n.adicionar\n.remover\n.listar\n\nUtilizar .ajuda <nomedocomando> para saber mais sobre."
        elif argument in ("help","ajuda"):
            embed.title = "Ajuda"
            embed.description = "Lista os comandos do Bot. Caso receba o nome do comando como argumento, retorna uma descrição do comando"
        elif argument in ("moeda","coin","preco","preço"):
            embed.title = "Moeda"
            embed.description = "Recebe o nome de uma moeda e retorna o valor atualizado em dólares (USD) e em reais (BRL)."
        elif argument in ("adicionar","add"):
            embed.title = "Adicionar"
            embed.description = "Recebe o nome de uma moeda e adiciona ela à sua lista pessoal de favoritos."
        elif argument in ("remover","rm"):
            embed.title = "Remover"
            embed.description = "Recebe o nome de uma moeda e remove ela da sua lista de favoritos."
        elif argument in ("listar","ls","lista"):
            embed.title = "Listar"
            embed.description = "Retorna os valores atualizado de cada moeda da sua lista de favoritos em dólares (USD) e em reais (BRL)."
        else:
           embed.title = "Comando não encontrado!"
        await ctx.send(embed=embed)

    @commands.command()
    async def ping(self,ctx):
        await ctx.send("pong")

def setup(bot):
    bot.add_cog(Utils(bot))


import simplejson as json # requires simplejson v2.5.0 or newer
import os
import shutil
import sys

# Global variables! Hard-coded params!
# That's how you know this is such a great script. :-)
gSupportedLocales = ['ar', 'ca', 'de', 'el', 'en-US', 'es', 'fr', 'it', 'pt-BR', 'ru', 'tr', 'zh-TW']
gDefaultLocale = 'en-US'

def propertyOrder(key):
  order = (
    'name',
    'description',
    'launch_path',
    'developer',
    'permissions',
    'locales',
    'default_locale'
  )
  if key[0] in order:
    return order.index(key[0])
  else:
    return key[0] # alphabetic order

# The expected file structure is as follows:
# /
#     [app]/
#         [lang]/
#             manifest.webapp
#             locale/
#                 [*].properties
#     ...

# inject localized name/descriptions into JSON manifests
def mergeManifests(inputDir, outputDir, application):
  manifestPath = os.path.join(outputDir, application, 'manifest.webapp')
  print(manifestPath)

  # load JSON manifest
  dest = open(manifestPath, 'r')
  data = json.load(dest)
  data['locales'] = {}
  data['default_locale'] = gDefaultLocale
  dest.close()

  # fill the 'locales' property
  for lang in gSupportedLocales:
    sourcePath = os.path.join(inputDir, application, lang, 'manifest.properties')
    source = open(sourcePath, 'r');

    # parse name/description in the properties file -- FIXME:
    # we expect to find 'name' on the first line, 'description' on the 2nd
    desc = {}
    lines = source.readlines()
    desc['name'] = lines[0].replace('name=', '').replace('\n', '')
    desc['description'] = lines[1].replace('description=', '').replace('\n', '')
    data['locales'][lang] = desc

    dest = open(manifestPath, 'wb')
    dest.write(json.dumps(data, indent = 2, separators=(',', ': '), \
        item_sort_key = propertyOrder))
    dest.close()

# concatenate files in the 'locale' directory (if any)
def mergeProperties(inputDir, outputDir, application):
  localeDir = os.path.join(inputDir, application, gDefaultLocale, 'locale')
  if os.path.isdir(localeDir):
    for resource in os.listdir(localeDir):

      resourcePath = os.path.join(outputDir, application, 'locale', resource)
      print(resourcePath)

      dest = open(resourcePath, 'wb')
      for lang in gSupportedLocales:
        dest.write('[' + lang + ']\n')
        sourcePath = os.path.join(inputDir, application, lang, 'locale', resource)
        shutil.copyfileobj(open(sourcePath, 'rb'), dest)

# import l10n data for all Gaia apps
def main():
  if not len(sys.argv) == 2:
    print('Usage: ' + sys.argv[0] + ' [applicationDirectory]')
    exit()

  inputDir = os.path.realpath(os.path.dirname(sys.argv[0]))
  inputDir = os.path.join(inputDir, 'apps');
  outputDir = os.path.realpath(sys.argv[1])
  print('Importing manifests and properties...')
  print('  from: ' + inputDir)
  print('  into: ' + outputDir)

  for application in os.listdir(inputDir):
    mergeManifests(inputDir, outputDir, application)
    #mergeProperties(inputDir, outputDir, application)

# startup
if __name__ == "__main__":
  main()


import numpy as np
import pandas as pd
import xarray as xr

from common import xarray_extras, utils
from softio.common import GEO_REGIONS, GEO_REGION, REGION_DIM
from softio.postprocess import emission_contrib_postprocess

TOP_HEIGHT = 'top_height'
BOTTOM_HEIGHT = 'bottom_height'
DELTA_HEIGHT = 'delta_height'

HEIGHT_DIM = 'height'

def get_height_ds(height_coord):
    height_dim, = height_coord.dims
    top_height = height_coord.astype('f8')
    bottom_height = xr.DataArray(np.concatenate(([0.], top_height[:-1])),
                                 coords=dict(height=height_coord), dims=[height_dim])
    height_ds = xr.Dataset({TOP_HEIGHT: top_height,
                            BOTTOM_HEIGHT: bottom_height,
                            DELTA_HEIGHT: top_height - bottom_height})
    return height_ds

def get_lon_lat_region_mask(region_da):
    """
    Produces a region_mask bool DataArray based on region_da (e.g. as stored in the variable GFED_regions_0_5deg in
    /o3p/iagos/softio/EMISSIONS/GFED3_fire_regions.nc). It is gridded with lon-lat coordinates as region_da
    and 'region' coordinates with values 0, 1, ..., 14. An extra non-dimensional coordinate 'geo_region'
    with region codes is provided.
    :param region_da: xarray DataArray gridded with lon-lat dimensions with values being region codes: 1-14 or 0.
    :return: xarray DataArray region_mask with bool values; gridded with lon-lat dimensions of region_da and
    """
    geo_regions = list(GEO_REGIONS)
    try:
        geo_regions.remove('TOTAL')
    except ValueError:
        pass
    geo_regions.insert(0, 'RESI')
    geo_region_by_id = dict(enumerate(geo_regions, start=0))
    region_da = region_da.astype('i4')
    lon_dim, lat_dim = xarray_extras.get_lon_lat_label(region_da)
    da = xr.DataArray(
        False,
        coords={REGION_DIM: np.array(list(geo_region_by_id.keys()), dtype='i4'),
                lon_dim: region_da[lon_dim],
                lat_dim: region_da[lat_dim]},
        dims=[REGION_DIM, lon_dim, lat_dim],
        name='region_mask'
    )
    da = da.assign_coords({GEO_REGION: (REGION_DIM, list(geo_region_by_id.values()))})
    da.loc[{REGION_DIM: region_da}] = True
    return da

# prepare region_mask
default_region_ds = xr.open_dataset('/o3p/iagos/softio/EMISSIONS/GFED3_fire_regions.nc').squeeze(drop=True)
default_region_mask = get_lon_lat_region_mask(default_region_ds.GFED_regions_0_5deg)

def regrid_time(da, target_time_coords):
    """
    Regrid the time coordinate of da to target_time_coords.
    Time coordinates are assumed to indicate the beginning of a time period.
    da must have time coordinates more coarse than target_time_coords
    :param da: xarray.DataArray
    :param target_time_coords: xarray.DataArray
    :return: xarray.DataArray
    """
    return da.sel({'time': target_time_coords}, method='ffill').assign_coords({'time': target_time_coords})

def get_emission_contrib_by_region(residence_time, emission, injection_rate, region_mask=None, postprocess=True,
                                   output_time_granularity=None):
    """
    NOTE: The lon-lat grid of residence_time must be the same or more coarse than that of emission and injection_rate.
    Time periods of residence_time must be the same or finer than time periods of emission (and injection rate).
    Time periods of residence_time, emission and injection_rate are represented in coordinates of 'time' dimension and
    refer to beginning time of a time period.
    Emission and injection rate must have the same coordinates in dimensions they share.
    :param residence_time: xarray DataArray containing lon-lat dimensions and height dimension; in units s m3 kg-1
    :param emission: xarray DataArray containing lon-lat dimensions and having units kg m-2 s-1
    :param injection_rate: xarray DataArray containing height dimension; in units m-1
    :param region_mask: a boolean xarray DataArray with lon-lat and region dimensions; if None (default),
    uses 'default_region_mask'
    :return: an xarray DataArray containing region dimension; in ppb
    """
    if region_mask is None:
        region_mask = default_region_mask
    res_time_units = residence_time.attrs.get('units')
    if res_time_units != 's m3 kg-1':
        raise ValueError(f'residence_time units must be s m3 kg-1; got {res_time_units}')

    # lon-lat dimension labels
    lon_dim, lat_dim = xarray_extras.get_lon_lat_label(residence_time)

    # regrid data to match a lon-lat grid of residence_time
    emission = xarray_extras.regrid_lon_lat(emission, target_resol_ds=residence_time, method='mean',
                                            longitude_circular=True, keep_attrs=True)
    try:
        xarray_extras.get_lon_lat_label(injection_rate)
        regrid_injection_rate = True
    except ValueError:
        regrid_injection_rate = False
    if regrid_injection_rate:
        injection_rate = xarray_extras.regrid_lon_lat(injection_rate, target_resol_ds=residence_time, method='mean',
                                                      longitude_circular=True, keep_attrs=True)
    region_mask = xarray_extras.regrid_lon_lat(region_mask.astype('f8'), target_resol_ds=residence_time,
                                               method='nearest', longitude_circular=True, keep_attrs=True)

    rt_time_periods = residence_time['time']
    rt_dt = abs(utils.unique(np.unique(rt_time_periods.diff('time'))))

    # apply output_time_granularity to emission and injection_rate, if set
    if output_time_granularity is not None:
        out_dt = pd.Timedelta(output_time_granularity)
        if out_dt <= pd.Timedelta(0):
            raise ValueError(f'output_time_granularity must be positive; '
                             f'got output_time_granularity={output_time_granularity}, which gives out_dt={out_dt}')
        if out_dt > pd.Timedelta('24H'):
            raise ValueError(f'output_time_granularity cannot exceed 24h; '
                             f'got output_time_granularity={output_time_granularity}, which gives out_dt={out_dt}')
        if pd.Timedelta('24H') % out_dt != pd.Timedelta(0):
            raise ValueError(f'output_time_granularity must be an integer fraction of 24h; '
                             f'got output_time_granularity={output_time_granularity}, which gives out_dt={out_dt}')
        if out_dt % rt_dt != pd.Timedelta(0):
            raise ValueError(f'output_time_granularity must be a multiple of temporal resolution of residence time={rt_dt} '
                             f'got output_time_granularity={output_time_granularity}, which gives out_dt={out_dt}')

        # regrid time coord of emission and injection_rate so that they have out_dt granularity
        if 'time' not in emission.dims:
            # add 'time' dimension to emission
            emission = emission.expand_dims({'time': np.datetime64('1900-01-01')})

        time_periods_with_out_dt = pd.date_range(start=rt_time_periods.min().values,
                                                 end=rt_time_periods.max().values + np.timedelta64(24, 'h'),
                                                 freq=output_time_granularity,
                                                 normalize=True)
        emission = regrid_time(emission, time_periods_with_out_dt)
        if 'time' in injection_rate.dims:
            injection_rate = regrid_time(injection_rate, time_periods_with_out_dt)

    # coarsen time dimension of residence_time to match that of emission or output_time_granularity, if set
    if 'time' in emission.dims:
        emission_time = emission['time']\
            .sel({'time': rt_time_periods}, method='ffill')\
            .assign_coords({'time': rt_time_periods})
        residence_time = residence_time.assign_coords({'emission_time': emission_time})
        residence_time = residence_time.groupby('emission_time').sum('time').rename({'emission_time': 'time'})
    else:
        residence_time = residence_time.sum('time')

    # compute contribution
    contribution = ((residence_time * injection_rate).sum(HEIGHT_DIM)
                    * emission * region_mask).sum([lon_dim, lat_dim]) * 1e9
    # must load DaskArray to numpy array, because otherwise
    # contrib_da.loc[{GEO_REGION: 'TOTAL'}] = contrib_da.sum(GEO_REGION)
    # is not available in emission_contrib_postprocess
    contribution = contribution.load()

    # setup metadata
    contribution.name = 'emission_contrib'
    contribution.attrs = dict(
        standard_name='mixing_ratio',
        long_name='Emission contribution from region',
        units='1e-9',
    )

    if postprocess:
        contribution = emission_contrib_postprocess(contribution)
    return contribution

def get_injection_rate(inj_height, height_coord):
    """
    Calculates injection rate (in m-1) based on injection height.
    The integral of the injection rate wrt height (in m) is 1.
    :param inj_height: xarray DataArray or a scalar with injection height (in m above the ground).
    :param height_coord: xarray DataArray with height of upper layers of vertical grid cells
    :return: xarray DataArray
    """
    height_ds = get_height_ds(height_coord)
    first_height = height_ds[TOP_HEIGHT][0].item()
    inj_height_corrected = np.maximum(inj_height, first_height)
    inj_coeff = (inj_height_corrected - height_ds[BOTTOM_HEIGHT]) / height_ds[DELTA_HEIGHT]
    inj_coeff = inj_coeff.clip(0., 1.)
    return inj_coeff / inj_height_corrected


import csv, json, xlrd

def csvParse(csvfile):
    # Open the CSV
    f = open( csvfile, 'rU' )
        # Change each fieldname to the appropriate field name.
    reader = csv.DictReader( f, fieldnames = ( "Protocol","Source Port","Destination Port","Source IP", "Destination IP", "Source MAC", "Destination MAC", "segment ID", "Frame name", "Frame ID", "segment name", "packet ID", "Receivers"))
    framenames = []
    store = []
    # Store frame names in a list
    for row in reader:
        frame= {"FrameName":row["Frame name"],
        "FrameID": row["Frame ID"],
        "protocol": row["Protocol"],
        "segments":[]}
        if row["Frame name"] not in framenames:
            framenames.append(row["Frame name"])
            store.append(frame)

    # Create Objects for Frames, segments and packets
    segment = {"segmentName":""}
    for frame in store:
        f = open( csvfile, 'rU' )
        reader = csv.DictReader( f, fieldnames = ( "Protocol","Source Port","Destination Port","Source IP", "Destination IP", "Source MAC", "Destination MAC", "segment ID","Frame name", "Frame ID", "segment name", "packet ID", "Receivers"))
        for row in reader:
            if frame["FrameName"] == row["Frame name"]:
                if segment["segmentName"] != row["segment name"]:
                    segment = {
                        "segmentID":row["segment ID"],
                        "segmentName": row["segment name"],
                        "srcPort":row["Source Port"],
                        "destPort":row["Destination Port"],
                        "packets":[{
                            "packetID":row["packet ID"],
                            "Receivers":row["Receivers"],
                            "destIP":row["Destination IP"],
                            "destMAC":row["Destination MAC"],
                            "srcIP": row["Source IP"],
                            "srcMAC":row["Source MAC"]
                            }]
                        }
                    frame["segments"].append(segment)
                else:
                    packet = {
                        "packetID":row["packet ID"],
                        "Receivers":row["Receivers"],
                        "destIP":row["Destination IP"],
                        "destMAC":row["Destination MAC"],
                        "srcIP": row["Source IP"],
                        "srcMAC":row["Source MAC"]
                        }
                    segment["packets"].append(packet)

    # Parse the CSV into JSON
    out = json.dumps(store, indent=4)
    # Save the JSON
    f = open( 'data.json', 'w')
    f.write(out)

def xlsParse(xlsfile):
    store = []  #List of objects to be parsed into json file
    framenames = []
    book = xlrd.open_workbook(xlsfile)
    sh1 = book.sheet_by_index(0)

    for rx in range(1, sh1.nrows):
        if sh1.row(rx)[8].value not in framenames:
            framenames.append(sh1.row(rx)[8].value)
            frame = {"frameName": sh1.row(rx)[8].value,
               "frameID": sh1.row(rx)[9].value,
               "protocol": sh1.row(rx)[0].value,
               "segments":[]
               }
            store.append(frame)

    segment = {"segmentName":""}
    for frame in store:
        for rx in range(1, sh1.nrows):
            if frame["frameName"] == sh1.row(rx)[8].value:
                if segment["segmentName"] != sh1.row(rx)[10].value:
                    segment = {
                    "segmentID":sh1.row(rx)[7].value,
                    "segmentName": sh1.row(rx)[10].value,
                    "srcPort":int(sh1.row(rx)[1].value),
                    "destPort":int(sh1.row(rx)[2].value),
                    "packets":[{
                        "packetID":sh1.row(rx)[11].value,
                        "Receivers":sh1.row(rx)[12].value,
                        "srcMAC": sh1.row(rx)[5].value,
                        "srcIP": sh1.row(rx)[3].value,
                        "destIP":sh1.row(rx)[4].value,
                        "destMAC":sh1.row(rx)[6].value
                        }]
                    }
                    frame["segments"].append(segment)
                else:
                    packet = {
                    "packetID":sh1.row(rx)[11].value,
                    "Receivers":sh1.row(rx)[12].value,
                    "srcMAC": sh1.row(rx)[5].value,
                    "srcIP": sh1.row(rx)[3].value,
                    "destIP":sh1.row(rx)[4].value,
                    "destMAC":sh1.row(rx)[6].value
                    }
                    segment["packets"].append(packet)

    # Parse the XLS into JSON
    out = json.dumps(store, indent=4)
    # Save the JSON
    f = open( 'xlsdata.json', 'w')
    f.write(out)

csvParse('dataset.csv')

xlsParse('dataset.xls')


import pandas as pd
import numpy as np
import pickle
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import PowerTransformer

class GetAndProcessData:

    def get_data(self, path, test_flag):

        with open(path + 'X_data', 'rb') as f: #.pickle
            X = pickle.load(f)  # X dims = [obs, ts, f]
        with open(path + 'y_data', 'rb') as f:
            y = pickle.load(f)

        # remove bad features 22, 57, 65 (negative time etc.):
        X = np.delete(X, [22, 57, 65], 2)

        # for debug:
        #X = np.float32(np.random.rand(4250, 28, 5))
        #y = np.float32(y)
        # X = X[0:1000, :, :]
        # y = y[0:1000, :]

        if test_flag:
            X_test = X
            y_test = y
            return X_test, np.squeeze(y_test)  # un-normalized

        else:
            X_train, X_dev, y_train, y_dev = train_test_split(X, y, test_size=0.30, random_state=42)
            return X_train, np.squeeze(y_train), X_dev, np.squeeze(y_dev)  # un-normalized

    @staticmethod
    def add_const(X):
        X += 1

    def generate_data_for_scaler(self, X_train, objects_write_path):

        # use up to 10k obs for scaling:
        n_obs_for_sclaer = min(X_train.shape[0], 10000)
        random_sampling_mask = np.random.choice(np.arange(X_train.shape[0]), n_obs_for_sclaer, replace=False)

        # calc scaler only based n the last day in the sequence:
        active_features_mask = np.std(X_train[:, -1, :], axis=0) > 0#1.0  # or lower value - 0

        with open(objects_write_path + '/active_features_mask.pkl', 'wb') as file:
            pickle.dump(active_features_mask, file)

        # define the data for scalers calculation (last day in sequence):
        #data_for_scaler = np.squeeze(X_train.astype(np.float64)[:, -1, active_features_mask][random_sampling_mask])
        data_for_scaler = X_train.astype(np.float64)[:, -1, active_features_mask][random_sampling_mask]  #
        # add const to avoid zero-values of some features:
        self.add_const(data_for_scaler)

        return data_for_scaler, active_features_mask

    def calc_scaler(self, data_for_scaler, objects_write_path):

        # define scaler for the Gaussian similarity. It also returns zero-mean + unit variance!:
        scaler = PowerTransformer(method='box-cox', standardize=True, copy=True)  # method='yeo-johnson'
        scaler.fit(data_for_scaler)

        # save scaler:
        with open(objects_write_path + '/scaler.pkl', 'wb') as file:
            pickle.dump(scaler, file)

        return scaler

    def remove_bad_features(self, X, active_features_mask):
        X = X[:, :, active_features_mask]
        return X

    def apply_scaler(self, X, scaler):
        self.add_const(X)
        # scale all days in the sequence:
        for i in range(X.shape[1]):
            X[:, i, :] = scaler.transform(np.squeeze(X[:, i, :]))
        return X

    @staticmethod
    def load_objects(objects_read_path):
        # load scaler and features mask:
        with open(objects_read_path + '/scaler.pkl', 'rb') as file:
            scaler0 = pickle.load(file)
        with open(objects_read_path + '/active_features_mask.pkl', 'rb') as file:
            active_features_mask = pickle.load(file)
        return scaler0, active_features_mask

    @staticmethod
    def do_clipping(X):
        # clip negative data (the scaler box-cox needs positive values)
        X = np.clip(X, a_min=0, a_max=None)
        return X

    def run_all_train(self, data_path, objects_write_path):
        X_train, y_train, X_dev, y_dev = self.get_data(path=data_path, test_flag=False)
        data_for_scaler, active_features_mask = self.generate_data_for_scaler(X_train, objects_write_path)
        scaler = self.calc_scaler(data_for_scaler, objects_write_path)
        X_train = self.remove_bad_features(X_train, active_features_mask)
        X_dev = self.remove_bad_features(X_dev, active_features_mask)
        X_train = self.apply_scaler(X_train, scaler)
        X_dev = self.apply_scaler(X_dev, scaler)
        return X_train, X_dev, y_train, y_dev  # scaled data

    def run_all_test(self, data_path, objects_read_path):
        X_test, y_test = self.get_data(path=data_path, test_flag=True)
        scaler, active_features_mask = self.load_objects(objects_read_path)
        X_test = self.remove_bad_features(X_test, active_features_mask)
        X_test = self.do_clipping(X_test)
        X_test = self.apply_scaler(X_test, scaler)
        return X_test, y_test  # scaled data

if __name__ == '__main__':
    print('start fetching data')
    data_train_path = '../mock_data/train_data/'
    data_test_path = '../mock_data/test_data/'
    objects_path = '../mock_data/data_objects'  # scaler and features mask
    gpd = GetAndProcessData()
    X_train, X_dev, y_train, y_dev = gpd.run_all_train(data_train_path, objects_path)
    X_test, y_test = gpd.run_all_test(data_test_path, objects_path) # use this with test data path
    print('data fetching successful')


from matplotlib import pyplot as plt
from matplotlib import animation
from matplotlib import style
import serial

fig,ax1=plt.subplots(figsize=(12,6))
ax1.set_xlabel("X")
ax1.set_ylabel("Y")
ax1.set_title("CAR'S PATH")
ser = serial.Serial('COM3', 115200)
ax1.plot(50, 50, "bo-")
def plotea (i):
    xx=[]
    yy=[]
    for i in range(50):
        data=ser.readline()
        data=data.splitlines()[0]
        data=data.decode().split(",")
        xx.append(float(data[0]))
        yy.append(float(data[1]))
        print(data[0],data[1])
    ax1.set_ylim([-30, 60])
    ax1.set_xlim([-30,60])

    try:  # Nos permite comprobar si hay un error al ejecutar la siguiente instrucción.
        ax1.plot(xx,yy, "r--")
    except:
        pass

ani = animation.FuncAnimation(fig, plotea, interval = 1) #Creamos animación para que se ejecute la función plotea con un intervalo de 1ms.

plt.show() #Muestra la gráfica.


import sys
import os
import arcpy
import numpy
import datetime
import math
import pandas as pd
import glob
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
from arcpy.sa import *

class Toolbox(object):
    def __init__(self):
        """Define the toolbox (the name of the toolbox is the name of the .pyt file)."""
        self.label = 'Magnetometer Survey Toolbox'
        self.alias = ''

        # List of tool classes associated with this toolbox
        self.tools = [SmoothRawAltitudeTool]

class SmoothRawAltitudeTool(object):
    def __init__(self):
        """Define the tool (tool name is the name of the class)."""
        self.label = 'Smooth Raw Altitude'
        self.description = ''
        self.canRunInBackground = True

    def getParameterInfo(self):
        """Define parameter definitions"""
        param0 = arcpy.Parameter(displayName = 'Input Raw File',
                                 name = 'inputRawFile',
                                 datatype = 'Table',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param1 = arcpy.Parameter(displayName = 'Altitude Max Threshold',
                                 name = 'altMax',
                                 datatype = 'Double',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param2 = arcpy.Parameter(displayName = 'Rolling Stats Window Size',
                                 name = 'window',
                                 datatype = 'Double',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param3 = arcpy.Parameter(displayName = 'Altitude field',
                                 name = 'inputAltfield',
                                 datatype = 'Field',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param4 = arcpy.Parameter(displayName = 'Line field',
                                 name = 'inputLinefield',
                                 datatype = 'Field',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param5 = arcpy.Parameter(displayName = 'Time field',
                                 name = 'inputTimefield',
                                 datatype = 'Field',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param6 = arcpy.Parameter(displayName = 'AltSmth Temp Directory',
                                 name = 'altSmthPathway',
                                 datatype = 'Workspace',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param7 = arcpy.Parameter(displayName = 'Output Pathway',
                                 name = 'outputPathway',
                                 datatype = 'Workspace',
                                 parameterType = 'Required',
                                 direction = 'Input')

        param8 = arcpy.Parameter(displayName = 'Output File Name (tool will append _Altsmth)',
                                 name = 'outputFileName',
                                 datatype = 'String',
                                 parameterType = 'Required',
                                 direction = 'Input')

        # Default values
        param1.value = 20
        param2.value = 5
        param3.value = "Field2"
        param4.value = "Field3"
        param5.value = "Field4"
        param6.value = "outfile"

        param3.parameterDependencies = [param0.name]
        param4.parameterDependencies = [param0.name]
        param5.parameterDependencies = [param0.name]
        return [param0, param1, param2, param3, param4, param5, param6, param7,param8]

    def isLicensed(self):
        """Set whether tool is licensed to execute."""
        try:
            if arcpy.SetProduct('arcinfo') == "NotLicensed":
                raise Exception
        except Exception:
            return False
        return True

    def updateParameters(self, parameters):
        return

    def updateMessages(self, parameters):
        return

    def execute(self, parameters, messages):
        # Set input variables
        inputRawFile = parameters[0].valueAsText
        arcpy.AddMessage('Raw File: {}'.format(inputRawFile))

        altMax=(int(parameters[1].value))
        arcpy.AddMessage('Altitude Maximum Threshold: {}'.format(altMax))

        window=(int(parameters[2].value))
        arcpy.AddMessage('Rolling Stats Window Size: {} points'.format(window))

        inputAltfield=parameters[3].valueAsText

        inputLinefield=parameters[4].valueAsText

        inputTimefield=parameters[5].valueAsText

        altSmthPathway=parameters[6].valueAsText

        outputWorkspace=parameters[7].valueAsText

        outputFileName=parameters[8].valueAsText

        inputTable=pd.read_csv(r'{}'.format(inputRawFile))
        df=pd.DataFrame(inputTable)
        arcpy.AddMessage("Read raw file")

        # plot raw altitude; use this to confirm or adjust max alt threshold
        plt.style.use('classic')
        x=df['{}'.format(inputTimefield)]
        y=df['{}'.format(inputAltfield)]
        fig,ax=plt.subplots()
        ax.plot(x,y, linewidth=1.0)
        ax.get_xaxis().set_visible(False)
        ax.get_yaxis().set_visible(True)
        plt.savefig(r"{}\{}.png".format(altSmthPathway,outputFileName))

        ##iterate a list of unique line names
        lineNameList=df['{}'.format(inputLinefield)].unique()
        arcpy.AddMessage(lineNameList)
        arcpy.AddMessage("Total Number of Lines: {}".format(len(lineNameList)))

        df['{}_dspk'.format(inputAltfield)]=df['{}'.format(inputAltfield)].where(df['{}'.format(inputAltfield)]< altMax)

        ## define a function to perform a rolling stats computation on single line of data
        def rollingStatsperLine(x):
            arcpy.AddMessage("Line Name: {}".format(lineNameList[x]))
            mask=df['{}'.format(inputLinefield)]==lineNameList[x]
            dfx=df[mask].copy()
            dfx['{}_intp'.format(inputAltfield)]=dfx['{}_dspk'.format(inputAltfield)].interpolate(method='linear')
            dfx['{}_smth'.format(inputAltfield)]=dfx['{}_intp'.format(inputAltfield)].rolling(window,center=True).mean()
            dfx.to_csv(r"{}\{}ln{}.txt".format(altSmthPathway,outputFileName,x),float_format='%.2f',index=False)
        ##plot raw channel
            plt.style.use('classic')
            a=dfx['{}'.format(inputTimefield)]
            b=dfx['{}'.format(inputAltfield)]
            fig,ax=plt.subplots()
            ax.plot(a,b, linewidth=1.0)
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(True)
            plt.savefig(r"{}\1_raw_{}_ln{}.png".format(altSmthPathway,outputFileName,x))
        ##plot despike channel result
            plt.style.use('classic')
            c=dfx['{}'.format(inputTimefield)]
            d=dfx['{}_dspk'.format(inputAltfield)]
            fig,ax=plt.subplots()
            ax.plot(c,d, linewidth=1.0)
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(True)
            plt.savefig(r"{}\2_dspk_{}_ln{}.png".format(altSmthPathway,outputFileName,x))
        ##plot smoothed channel result
            plt.style.use('classic')
            e=dfx['{}'.format(inputTimefield)]
            f=dfx['{}_smth'.format(inputAltfield)]
            fig,ax=plt.subplots()
            ax.plot(e,f, linewidth=1.0)
            ax.get_xaxis().set_visible(False)
            ax.get_yaxis().set_visible(True)
            plt.savefig(r"{}\3_smth_{}_ln{}.png".format(altSmthPathway,outputFileName,x))
            return

        ## call the function and run as iterations of the line name list
        x=0
        while x <= len(lineNameList):
            rollingStatsperLine(x)
            x +=1
            if x == len(lineNameList):
                break

        ## read directory of CSV files generated for each df split by line, merge into a single df
        path =(r"{}".format(altSmthPathway))
        all_files = glob.glob(os.path.join(path,"{}*.txt".format(outputFileName)))
        df_from_each_file = (pd.read_csv(f,sep=',')for f in all_files)
        df_merged = pd.concat(df_from_each_file,ignore_index=True)
        df_final=df_merged.dropna()
        arcpy.AddMessage("Merged Resutls")

        ## export merged df as full csv file
        os.chdir(r"{}".format(outputWorkspace))
        df_final.to_csv("{}_Altsmth.txt".format(outputFileName),index=False)

        for f in all_files:
            os.remove(f)

        arcpy.AddMessage("completed altitude smoothing")


import requests
from bs4 import BeautifulSoup
import norm

def isFloat(s):
    if s == None:
        return False
    for x in s:
        if x not in "0,1,2,3,4,5,6,7,8,9,.".split(","):
            return False
    return True

# SCRAPING
NUM_MOVIES = 250
URL = "https://www.metacritic.com/browse/movies/score/userscore/all/filtered?view=condensed&page="
USER_AGENT = {'User-agent': 'Mozilla/5.0'}
BASE_URL = "https://www.metacritic.com"
movies = []

# SQL
TABLE_NAME = "movies_meta"

# SCRAPING FUNCTIONS

def scrapeAll():
    i = 0
    while len(movies) < NUM_MOVIES:
        page = scrape(URL+str(i), USER_AGENT)
        parseHTML(page)
        i += 1

def scrape(url, userAgent):
    return requests.get(url, headers=userAgent)

# PARSING FUNCTIONS

def parseHTML(page):
    soup = BeautifulSoup(page.content, "html.parser")
    table = soup.find("div", class_="title_bump")
    # Each movie's information is in a <tr> element in the HTML
    trs = table.find_all("tr", class_="expand_collapse")
    for tr in trs:
        movie = parseMovie(tr)
        movies.append(movie)
        if len(movies) >= NUM_MOVIES:
            break

def parseMovie(tr):
    rating = tr.find("td", class_="score").find("div").get_text()

    td = tr.find("td", class_="details")
    title = td.find("h3").get_text()
    href = td.find("a", class_="title").get("href")
    movie_url = BASE_URL + href
    genres = scrapeGenres(movie_url)

    justGotDate = 0
    for child in td.children:
        child = str(child)
        if justGotDate == 1:
            justGotDate = 2
        elif justGotDate == 2:
            pipeIndex = child.find("|")
            if pipeIndex == -1:
                mpaa = "Rating Unlisted"
            else:
                mpaa = child[pipeIndex+2:-7]
            break
        if child[:6] == "<span>":
            year = child[-11:-7]
            justGotDate = 1

    if title == None:
        title = "N/A"
    if genres == None:
        genres = "N/A"
    if year == None:
        year = "N/A"
    if rating == None:
        rating = "N/A"
    if mpaa == None:
        mpaa = "N/A"

    # If title includes year at the end, use this as the year and remove from title
    if len(title) > 7:
        if title[-6] == "(" and title[-1] == ")" and title[-5:-1].isdigit():
            year = title[-5:-1]
            title = title[:-7]

    return (year, rating, title, movie_url)

def scrapeGenres(url):
    page = scrape(url, USER_AGENT)
    soup = BeautifulSoup(page.content, "html.parser")
    return ", ".join([x.get_text() for x in soup.find("div", class_="genres").contents[3].findAll("span")])

def scrapeRatings(titles):
    movies = []
    for title in titles:
        try:
            rating, url = scrapeRating(title[0], title[1])
            print(rating)
            if rating != None:
                movies.append((rating, title[1], title[0], url))
        except:
            print("temporary error")
    return movies

def scrapeRating(title, year):
    urlTitle = norm.normUrl(title)
    baseUrl = BASE_URL + "/movie/"
    if title.lower() == "star wars: episode iv - a new hope":
        thisUrl = "https://www.metacritic.com/movie/star-wars-episode-iv---a-new-hope"
    elif title.lower() == "star wars: episode v - the empire strikes back":
        thisUrl = "https://www.metacritic.com/movie/star-wars-episode-v---the-empire-strikes-back"
    elif title.lower() == "star wars: episode vi - return of the jedi":
        thisUrl = "https://www.metacritic.com/movie/star-wars-episode-vi---return-of-the-jedi"
    elif title.lower() == "moonlight" and year == 2016:
        thisUrl = "https://www.metacritic.com/movie/moonlight-2016"
    elif title.lower() == "hamilton" and year == 2020:
        thisUrl = "https://www.metacritic.com/movie/hamilton-2020"
    else:
        thisUrl = baseUrl + "-".join(urlTitle.split())
    print("metaUrl=", thisUrl)
    page = scrape(thisUrl, USER_AGENT)
    # if error message like 403 or 404, return nothing
    if str(page).split()[1][1] == "4":
        thisUrl += "-" + year
        page = scrape(thisUrl, USER_AGENT)
        if str(page).split()[1][1] == "4":
            return None
    soup = BeautifulSoup(page.content, "html.parser")
    try:
        rating = soup.find("div", class_="us_wrapper").find(
            "a", class_="metascore_anchor").find("span").get_text()
    except AttributeError as e:
        print("Error scraping: {}".format(e))
        return None
    if not isFloat(rating):
        return None
    return (rating, thisUrl)

def run():
    scrapeAll()
    return movies


# -*- coding: utf-8 -*-
from ge import Node2Vec
import networkx as nx
import json
import pandas as pd
from hashlib import md5

def clean_friends(row):
    if row == 'None':
        return []
    _friends = set(map(lambda _row: str(_row) + '_u', row.split(',')))
    return list(_friends.intersection(user_ids))

def category2word(row):
    if not row:
        return []
    if row == 'None':
        return []
    words = row.split(',')
    words = list(map(lambda _row: _row.replace('&', ' ').replace('/', ' ').replace('   ', ' ').strip(), words))
    words = [md5(word.encode(encoding='UTF-8')).hexdigest() for word in words if word is not None]
    return words

mode = 'online'  # mode in ['online', 'test'. 'train']
train_set = pd.read_csv('../yelp_data/yelp_train.csv').rename(columns={'stars': 'target'})
eval_set = pd.read_csv('../yelp_data/yelp_val.csv').rename(columns={'stars': 'target'})
test_set = pd.read_csv('../yelp_data/yelp_test_ans.csv').rename(columns={'stars': 'target'})

train_set.user_id = train_set.user_id.map(lambda row: str(row) + '_u')
train_set.business_id = train_set.business_id.map(lambda row: str(row) + '_b')
eval_set.user_id = eval_set.user_id.map(lambda row: str(row) + '_u')
eval_set.business_id = eval_set.business_id.map(lambda row: str(row) + '_b')
test_set.user_id = test_set.user_id.map(lambda row: str(row) + '_u')
test_set.business_id = test_set.business_id.map(lambda row: str(row) + '_b')

if mode == 'online':
    user_ids = sorted(list(set(eval_set.user_id.tolist() + train_set.user_id.tolist() + test_set.user_id.tolist())))
    business_ids = sorted(
        list(set(eval_set.business_id.tolist() + train_set.business_id.tolist() + test_set.business_id.tolist())))
    data = pd.concat([train_set, eval_set, test_set])
elif mode == 'test':
    user_ids = sorted(list(set(eval_set.user_id.tolist() + train_set.user_id.tolist())))
    business_ids = sorted(list(set(eval_set.business_id.tolist() + train_set.business_id.tolist())))
    data = pd.concat([train_set, eval_set])
else:
    user_ids = sorted(list(set(train_set.user_id.tolist())))
    business_ids = sorted(list(set(train_set.business_id.tolist())))
    data = train_set

with open("../yelp_data/business.json", 'r', encoding='utf-8') as f:
    lines = map(json.loads, f.readlines())
business = pd.DataFrame.from_records(lines)
business.business_id = business.business_id.map(lambda row: str(row) + '_b')
business = pd.DataFrame(business[business.business_id.isin(business_ids)]).reset_index(drop=True)

with open("../yelp_data/user.json", 'r', encoding='utf-8') as f:
    lines = map(json.loads, f.readlines())
user = pd.DataFrame.from_records(lines)
user.user_id = user.user_id.map(lambda row: str(row) + '_u')
user = pd.DataFrame(user[user.user_id.isin(user_ids)]).reset_index(drop=True)

user['friend_lst'] = user.friends.map(clean_friends)

friend_pairs = set()
for _ in range(len(user)):
    user_id = user.user_id[_]
    friends = user.friend_lst[_]
    if not friends:
        continue
    for friend in friends:
        friend_pairs.add(tuple(sorted([user_id, friend])))

business.categories = business.categories.map(category2word)
categories = set()
for cats in business.categories.values:
    for cat in cats:
        categories.add(cat)
categories = sorted(list(categories))

business_pairs = set()
for _ in range(len(business)):
    business_id = business.business_id[_]
    cats = business.categories[_]
    if not cats:
        continue
    for cat in cats:
        business_pairs.add(tuple(sorted([business_id, cat])))

pairs = list(business_pairs.union(friend_pairs))

with open('../yelp_data/edges.txt', 'w+') as f:
    for line in data.values:
        f.write(f'{line[0]} {line[1]}\n')
    for pair in pairs:
        f.write(f'{pair[0]} {pair[1]}\n')

G = nx.read_edgelist('../yelp_data/edges.txt',
                     create_using=nx.DiGraph(),
                     nodetype=None,
                     data=[('weight', int)])

model = Node2Vec(G, walk_length=25, num_walks=250, p=0.25, q=4, workers=16)
model.train(embed_size=128, window_size=15, iter=40, workers=16)  # train model
embeddings = model.get_embeddings()

df_embeddings = pd.DataFrame.from_dict(embeddings, orient='index',
                                       columns=[f'E_{_}' for _ in range(128)]).reset_index().rename(
    columns={'index': 'id'})
df_embeddings.to_csv('../yelp_data/embeddings.csv', index=False)


from numpy import log
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
import train_dist
import dist_util
import os, sys

MEASURE_TIME = False
DIST = False

class EarlyStopper:
    def __init__(self,patient):
        self.patient = patient
        self.best = None
        self.count = 0
    def __call__(self,score):
        if self.best is None:
            self.best = score
        elif score > self.best:
            self.best = score
            self.count = 0
        else:
            self.count += 1
        return self.count >= self.patient

def objective(config):

    if not MEASURE_TIME:
        config['kernel_size']+=1 # must be odd
        config['dropout'] = float(config['dropout'])
        config['pos_weight'] = float(config['pos_weight'])
        config['num_blocks'] = int(config['num_blocks'])
        config['initial_filter'] = int(config['initial_filter'])
        config['kernel_size'] = int(config['kernel_size'])
        config['scale_filter'] = float(config['scale_filter'])
        if DIST:
            dist_util.broadcast(config)
            print(f'Rank {dist_util.get_rank()} (master) got config {config}')
        config['exp_name'] = f'hyperopt_{config}'

    if MEASURE_TIME:
        config = {'dropout': 0.1535039249785174, 'initial_filter': 30, 'kernel_size': 5, 'num_blocks': 5, 'pos_weight': 1.0125812944977837, 'scale_filter': 1.1299710421481397} # Best

    earlyStopper = EarlyStopper(10)
    for pr_auc in train_dist.main(config,yeild_prauc = True):
        if earlyStopper(pr_auc):
            if DIST:
                dist_util.broadcast("stop")
                print(f'early stop')
            break
        else:
            if DIST:
                dist_util.broadcast("continue")

    return {
        'loss': - earlyStopper.best,
        'status': STATUS_OK,
    }

def main():
    if DIST:
        dist_util.setup_dist()
    if dist_util.is_master():
        trials = Trials()
        best = fmin(objective,
            space={
                'dropout':hp.uniform('dropout', 0, 0.5),
                'pos_weight':hp.loguniform('pos_weight', 0, 3),
                'num_blocks':hp.quniform('num_blocks', 3, 7, 1),
                'initial_filter':hp.quniform('initial_filter', 10, 30, 1),
                'kernel_size':hp.quniform('kernel_size', 4, 12, 2),
                'scale_filter':hp.loguniform('scale_filter',log(1),log(2)),
                },
            algo=tpe.suggest,
            max_evals=1000,
            trials=trials,
            show_progressbar=False,
            )
    else:
        while True:
            config = dist_util.broadcast()
            #print(f'Rank {dist_util.get_rank()} got config {config}')
            for _ in train_dist.main(config,yeild_prauc = True):
                if dist_util.broadcast() == "stop":
                    break

if __name__ == '__main__':
    main()

Stage 1. Hyperparameter tuning/optim_hyp.py


import caffe_pb2 as cpb
from google.protobuf import text_format
from collections import defaultdict
import tensorflow as tf
import numpy as np
from operator import mul
import logging

class Network:
    # Public functions.
    def __init__(self, proto, inputs=dict(), input_format="NC*"):
        """Create network using structure from Caffe proto file.

        Args:
            proto: The plaintext network prototxt file on disk.
                Type `str`.
            inputs: Dictionary mapping input variable `top` strings to
                Tensors. Use this to connect Tensors directly to the
                network, instead of using `tf.placeholder` type variables
                that are created by default for all layers of type
                `Input`. `tf.placeholder` instances will be created for
                any `top` values missing in the dictionary. Defaults to
                empty `dict`.
            input_format: The order of inputs specified in the prototxt
                file. The TensorFlow program is expected to provide inputs
                in the order `N*C`, where the dimension `N` denotes batch,
                `C` denotes the number of channels, and `*` denotes the
                rest of the dimensions. Valid arguments are `N*C`, `NC*`,
                `*NC`, `CN*`, `C*N`, and `*CN`. The network will permute
                the order of dimensions specified in prototxt file for the
                InputParameter in the order `N*C`. Defaults to `NC*`.
        """
        # Layer names mapped to outputs.
        self._name_outputs = defaultdict(list)
        # Layer top names mapped to outputs.
        self._top_outputs = defaultdict(list)
        # Layer name to list of variables.
        self._name_vars = defaultdict(list)
        # Protobuf LayerParameter messages for each layer, loaded in the
        # order presented in the proto file.
        self._layer_params = []
        # List of variables.
        self._vars = []
        # List of trainable variables.
        self._tvars = []
        # Layer name to (name, var) mapping.
        self._layer_vars = defaultdict(list)
        # The NetParameter protobuf message.
        self._net_param = cpb.NetParameter()
        self._input_format = input_format
        self._inputs = dict(inputs)

        # Load and parse the prototxt file.
        self._parse_proto(proto)
        # Create network, based on the parse.
        self._make_network()

    def load(self, model, sess):
        """Load network variables from NumPy file on disk.

        Args:
            model: The path to the model file on disk.
            sess: The TensorFlow session.
        """
        self._load(model, sess)

    def layer_output(self, layer_name):
        if layer_name not in self._top_outputs:
            raise ValueError("`%s` not defined in network" % layer_name)
        return self._top(layer_name)

    def save(self, path, sess):
        """Save the network weights to disk.

        Saved as compatible NumPy pickle.

        Args:
            path: The path of save file on disk.
            sess: The TensorFlow session.
        """
        m = {}
        for layer_name, name_var_list in self._layer_vars.iteritems():
            m[layer_name] = {}
            for var_name, var in name_var_list:
                m[layer_name][var_name] = var.eval(sess)
        np.save(path, m)

    @property
    def vars(self):
        """The network variables.

        Returns:
            List of all network TensorFlow variables.
        """
        return self._vars

    @property
    def tvars(self):
        """Trainable network variables.

        Returns:
            List of all trainable network TensorFlow variables.
        """
        return self._tvars

    # Layer functions.
    def _add_Input(self, lp):
        """Add input layer placeholder to network.

        Does not support undefined shapes.

        Args:
            lp: Caffe LayerParameter protobuf message
                with type "Input".
        """
        def reorder(dims):
            # Figure out the permutation for the shape to convert to N*C order.
            idx_C = self._input_format.index("C")
            if idx_C == 2:
                idx_C = -1
            idx_N = self._input_format.index("N")
            if idx_N == 2:
                idx_N = -1
            # Reorder.
            new_dim = [dims[idx_N]]
            temp = list(dims)
            temp = [i for j, i in enumerate(dims) if j not in [idx_C, idx_N]]
            new_dim += (temp + [dims[idx_C]])
            return new_dim

        # In case of multiple outputs but one shape specification, repeat.
        if len(lp.top) != len(lp.input_param.shape):
            shapes = lp.input_param.shape * len(lp.top)
        else:
            shapes = lp.input_param.shape
        for (top, shape) in zip(lp.top, shapes):
            # Reorder shape to get `N*C` ordering.
            shape_reordered = tuple(map(int, reorder(shape.dim)))
            # Check if inputs have been supplied by user.
            if top in self._inputs:
                input = self._inputs[top]
                # Make sure user-specified Tensor shape is compatible
                # with proto shape specifications.
                if not input.get_shape().is_compatible_with(shape_reordered):
                    raise ValueError("Shape of supplied `%s` input incompatible"
                                     " with proto specifications" % top)
                output = input
            else:
                output = tf.placeholder(tf.float32, shape_reordered, name=top)
            self._add_output_to_lists(output, lp.name, top)

    def _add_Convolution(self, lp):
        """Add convolution layer to network.

        Note that only 2-D convolutions are currently supported.

        Args:
            lp: Caffe LayerParameter protobuf message
                with type "Convolution".
        """
        filter_size, padding, strides, out_channels = \
            self._parse_ConvolutionParameter(lp.convolution_param)
        # Get the number of chanels in input. Ensure that the number
        # is consistent for each of the bottom layers.
        in_channels = 0
        for bottom in lp.bottom:
            if in_channels == 0:
                in_channels = self._top(bottom).get_shape()[-1]
            if in_channels != self._top(bottom).get_shape()[-1]:
                raise ValueError("bottom layers for %s must have same "
                                 "channel count" % lp.name)
        # Create convolution filter weights.
        with tf.variable_scope(lp.name):
            weights = self._make_vars(
                "weights",
                filter_size + [in_channels, out_channels],
                self._is_training(lp.phase),
                lp.name,
                self._FillerParameter_to_initializer(
                    lp.convolution_param.weight_filler))
            if lp.convolution_param.bias_term:
                biases = self._make_vars(
                    "biases",
                    [out_channels],
                    self._is_training(lp.phase),
                    lp.name,
                    self._FillerParameter_to_initializer(
                        lp.convolution_param.bias_filler))
        # For each bottom layer, create convolutional output
        # feature map.
        for (top, bottom) in zip(lp.top, lp.bottom):
            input = self._top(bottom)
            # Perform 2D convolution.
            with tf.name_scope(self._name_scope_builder(top)):
                # Pad input, if necessary.
                padded_input = tf.pad(input, padding)
                # Convolve.
                output = tf.nn.conv2d(padded_input, weights, strides, "VALID")
                # Add biases, if needed.
                if lp.convolution_param.bias_term:
                    output = tf.nn.bias_add(output, biases)
            # Add to lists.
            self._add_output_to_lists(output, lp.name, top)

    def _add_ReLU(self, lp):
        if len(lp.top) != len(lp.bottom):
            raise ValueError(
                "ReLU layers (%s) must have the same number of "
                "top and bottom layers" % lp.name)
        for top, bottom in zip(lp.top, lp.bottom):
            with tf.name_scope(self._name_scope_builder(top)):
                output = tf.nn.relu(self._top(bottom), name=lp.name)
            self._add_output_to_lists(output, lp.name, top)

    def _add_Pooling(self, lp):
        if len(lp.top) != len(lp.bottom):
            raise ValueError(
                "Pooling layers (%s) must have the same number of "
                "top and bottom layers" % lp.name)

        for top, bottom in zip(lp.top, lp.bottom):
            pp = lp.pooling_param
            # Recover the pooling parameters.
            input = self._top(bottom)
            ksize, padding, strides = self._parse_PoolingParameter(
                pp, input)
            with tf.name_scope(self._name_scope_builder(lp.name)):
                # Pad, if necessary.
                if padding is not None:
                    input = tf.pad(input, padding)
                if pp.pool == cpb.PoolingParameter.MAX:
                    output = tf.nn.max_pool(input, ksize, strides, "VALID")
                elif pp.pool == cpb.PoolingParameter.AVE:
                    output = tf.nn.avg_pool(input, ksize, strides, "VALID")
                else:
                    raise NotImplementedError(
                        "Unsupported pooling: %s" % pp.pool)
            self._add_output_to_lists(output, lp.name, top)

    def _add_InnerProduct(self, lp):
        # TODO(agharwal): Add support for multiple top and bottom layers.
        #                 Users should use Concat and Reshape layers to
        #                 concatenate multiple bottom layers.
        if len(lp.top) > 1:
            raise ValueError(
                "InnerProduct layers (%s) must have exactly one "
                "top layer" % lp.name)

        inputs = []
        with tf.name_scope(self._name_scope_builder(lp.name)):
            for bottom in lp.bottom:
                input = self._top(bottom)
                shape = input.get_shape().as_list()
                # Get the flattened shape, excluding the index
                # correspnding to batch.
                ndims_flat = reduce(mul, shape[1:], 1)  # NOQA
                inputs.append(tf.reshape(input, [shape[0], ndims_flat]))
            # Concatenate flattened inputs.
            if len(inputs) > 1:
                input = tf.concat(1, inputs)
            else:
                input = inputs[0]

        # Prepare weights and biases.
        ipp = lp.inner_product_param
        input_ndims = input.get_shape().as_list()[-1]
        with tf.variable_scope(lp.name):
            weights = self._make_vars(
                "weights",
                [input_ndims, ipp.num_output],
                self._is_training(lp.phase),
                lp.name,
                self._FillerParameter_to_initializer(
                    ipp.weight_filler))
            if ipp.bias_term:
                biases = self._make_vars(
                    "biases",
                    [ipp.num_output],
                    self._is_training(lp.phase),
                    lp.name,
                    self._FillerParameter_to_initializer(
                        ipp.bias_filler))

        # Compute output.
        with tf.name_scope(self._name_scope_builder(lp.name)):
            if ipp.bias_term:
                output = tf.nn.xw_plus_b(input, weights, biases)
            else:
                output = input * weights
        self._add_output_to_lists(output, lp.name, lp.top[0])

    def _add_Dropout(self, lp):
        for top, bottom in zip(lp.top, lp.bottom):
            with tf.name_scope(self._name_scope_builder(top)):
                output = tf.nn.dropout(self._top(bottom),
                                       1.0 - lp.dropout_param.dropout_ratio,
                                       name=lp.name)
            self._add_output_to_lists(output, lp.name, top)

    def _add_Softmax(self, lp):
        if len(lp.top) != len(lp.bottom):
            raise ValueError(
                "Softmax layers (%s) must have the same number of "
                "top and bottom layers" % lp.name)
        for top, bottom in zip(lp.top, lp.bottom):
            with tf.name_scope(self._name_scope_builder(top)):
                output = tf.nn.softmax(self._top(bottom))
            self._add_output_to_lists(output, lp.name, top)

    def _add_Reshape(self, lp):
        # FIXME(agharwal): Possible bug since this is not compensating
        #                  for order of axes in input.
        for top, bottom in zip(lp.top, lp.bottom):
            with tf.name_scope(self._name_scope_builder(top)):
                output = tf.reshape(
                    self._top(bottom),
                    self._ReshapeParameter_to_shape(
                        lp.reshape_param,
                        self._top(bottom).get_shape().as_list()),
                    name=lp.name)
            self._add_output_to_lists(output, lp.name, top)

    def _add_Sigmoid(self, lp):
        if len(lp.top) != len(lp.bottom):
            raise ValueError(
                "Sigmoid layers (%s) must have the same number of "
                "top and bottom layers" % lp.name)
        for top, bottom in zip(lp.top, lp.bottom):
            with tf.name_scope(self._name_scope_builder(top)):
                output = tf.sigmoid(self._top(bottom), name=lp.name)
            self._add_output_to_lists(output, lp.name, top)

    def _add_Concat(self, lp):
        if len(lp.top) != 1:
            raise ValueError(
                "Concat layers (%s) must have exactly one "
                "top layer" % lp.name)
        axis = self._parse_ConcatParameter(lp.concat_param)
        top = lp.top[0]
        bottom_tensors = [self._top(name) for name in lp.bottom]
        with tf.name_scope(self._name_scope_builder(top)):
            output = tf.concat(bottom_tensors, axis, name=lp.name)
        self._add_output_to_lists(output, lp.name, top)

    def _add_Slice(self, lp):
        if len(lp.bottom) != 1:
            raise ValueError(
                "Concat layers (%s) must have exactly one "
                "top layer" % lp.name)
        bottom_tensor = self._top(lp.bottom[0])
        axis, slice_points = self._parse_SliceParameter(lp.slice_param)
        # Recover the total number of axes, and the number of
        # dimensions along `axis`.
        bottom_shape = bottom_tensor.get_shape().as_list()
        num_axes = len(bottom_shape)
        top_layer_names = lp.top
        # Ensure that the number of slices match the number of top layers.
        if len(top_layer_names) != (len(slice_points) + 1):
            raise ValueError(
                "Slice layers (%s) must have exactly one more"
                "top layer than slice points" % lp.name)
        # Make sure the axis index is within range. Note that we need to take
        # into account negative indices too.
        if axis < -num_axes or axis >= num_axes:
            raise ValueError(
                "Slice layer (%s) axis index mismatch %d for %d axes" %
                (lp.name, axis, num_axes))
        start_indices = [0] + slice_points
        end_indices = slice_points + [bottom_shape[axis]]
        for (start_idx, end_idx, top) in zip(start_indices,
                                             end_indices,
                                             top_layer_names):
            begin = [0] * num_axes
            size = bottom_shape
            begin[axis] = start_idx
            size[axis] = end_idx - start_idx
            with tf.name_scope(self._name_scope_builder(top)):
                output = tf.slice(bottom_tensor, begin, size, name=lp.name)
                self._add_output_to_lists(output, lp.name, top)

    # Helper functions.
    def _make_vars(self, name, shape, trainable,
                   layer_name=None, initializer=None):
        """Create variables of given `name` and `shape`.

        Args:
            name: Variable name. Type `str`.
            shape: A 1-D `Tensor` of type `int32` representing
                the shape of the variables.
            trainable: `True` if variables are trainable.
            layer_name: Name of layer for which variables will be used.
                Type `str`. Ignored if set to `None`. Defaults to `None`.
        """
        vars = tf.get_variable(name, shape,
                               trainable=trainable,
                               initializer=initializer)
        self._vars.append(vars)
        if trainable:
            self._tvars.append(vars)
        if layer_name is not None:
            self._layer_vars[layer_name].append((name, vars))
        return vars

    def _add_output_to_lists(self, output, name, top):
        self._top_outputs[top].append(output)
        self._name_outputs[name].append(
            (top, len(self._top_outputs[top]) - 1, output))

    def _parse_proto(self, proto):
        """Parse the proto file to create network graph

        Args:
            proto: Path of prototxt plaintext file on disk.
                Type `str`.
        """
        with open(proto, "r") as f:
            text_format.Merge(f.read(), self._net_param)
            for layer_param in self._net_param.layer:
                self._layer_params.append(layer_param)

    def _top(self, top):
        return self._top_outputs[top][-1]

    def _make_network(self):
        """Construct the network using the parsed proto file."""
        for layer_param in self._layer_params:
            # Does layer need to be skipped?
            if not self._include_layer(layer_param):
                logging.info("Skipping: %s", layer_param.name)
                continue
            if layer_param.type == "Input":
                self._add_Input(layer_param)
            elif layer_param.type == "Convolution":
                self._add_Convolution(layer_param)
            elif layer_param.type == "ReLU":
                self._add_ReLU(layer_param)
            elif layer_param.type == "Pooling":
                self._add_Pooling(layer_param)
            elif layer_param.type == "InnerProduct":
                self._add_InnerProduct(layer_param)
            elif layer_param.type == "Dropout":
                self._add_Dropout(layer_param)
            elif layer_param.type == "Softmax":
                self._add_Softmax(layer_param)
            elif layer_param.type == "Reshape":
                self._add_Reshape(layer_param)
            elif layer_param.type == "Sigmoid":
                self._add_Sigmoid(layer_param)
            elif layer_param.type == "Concat":
                self._add_Concat(layer_param)
            elif layer_param.type == "Slice":
                self._add_Slice(layer_param)
            else:
                raise NotImplementedError(
                    "Contact the bugger who wrote this.")

    def _is_training(self, phase):
        """Global phase overrides local phase in case the network is
           in test phase; local phase is used otherwise.
        """
        return self._net_param.state.phase == cpb.TRAIN and phase == cpb.TRAIN

    def _parse_ConvolutionParameter(self, cp, dims=2):
        out_channels = cp.num_output

        # Infer input dimensions.
        if len(cp.kernel_size) == 1:
            filter_size = [cp.kernel_size[0]] * dims
        else:
            filter_size = cp.kernel_size

        # Infer padding width.
        if not cp.pad:
            padding = [0] * dims
        elif len(cp.pad) == 1:
            padding = [cp.pad[0]] * dims
        else:
            padding = cp.pad
        # TensorFlow allows specification of different paddings
        # at both ends of an input axis, but Caffe assumes it would
        # be the same. Create a TF compatible output.
        padding = [[0, 0]] + [[p, p] for p in padding] + [[0, 0]]

        # Infer stride.
        if len(cp.stride) == 0:
            strides = [1] * dims
        elif len(cp.stride) == 1:
            strides = [cp.stride[0]] * dims
        else:
            strides = cp.stride
        strides = [1] + strides + [1]

        return filter_size, padding, strides, out_channels

    def _parse_PoolingParameter(self, pp, bottom):
        # Input Tensor dimensions, ignoring batch size and number of channels.
        ndims = len(bottom.get_shape()) - 2
        # If pooling is global.
        if pp.global_pooling:
            # Kernel size equals to full extent of feature map.
            ksize = bottom.get_shape().as_list()[1:-1]
            # No padding.
            padding = None
            # Fixed stride.
            strides = [1] * (ndims + 2)
            return ksize, padding, strides

        # Figure out kernel size.
        # a) kernel_h and kernel_w are set. Feature dimensions
        #    must be equal to 2.
        if pp.kernel_size is None:
            if ndims != 2:
                raise ValueError(
                    "Either kernel_size or both kernel_x and kernel_y"
                    " must be set.")
            ksize = [pp.kernel_h, pp.kernel_w]
        # b) kernel_size is set
        else:
            ksize = [pp.kernel_size] * ndims
        # Adjust for TensorFlow specs. Kernel should be 1 layer
        # wide for batch and channel Tensor indices.
        ksize = [1] + ksize + [1]

        # Figure out stride.
        # a) stride is set
        if pp.stride > 0:
            strides = [pp.stride] * ndims
        # b) stide_h and stride_w are set (we assume both).
        else:
            strides = [pp.stride_h, pp.stride_w]
        # Adjust for TensorFlow specs. Max should be computed
        # over each feature map for each instance in the batch.
        strides = [1] + strides + [1]

        # Infer padding width. No padding along batch and channel
        # indices
        if pp.pad > 0:
            padding = [0] + ([pp.pad] * ndims) + [0]
        elif pp.pad_h > 0 or pp.pad_w > 0:
            padding = [0, pp.pad_h, pp.pad_w, 0]
        else:
            padding = None
        # TensorFlow allows specification of different paddings
        # at both ends of an input axis, but Caffe assumes it would
        # be the same. Create a TF compatible output.
        if padding is not None:
            padding = [[p, p] for p in padding]

        return ksize, padding, strides

    def _parse_ConcatParameter(self, cp):
        # Return the axis index.
        return cp.axis

    def _parse_SliceParameter(self, sp):
        # Return the axis index.
        axis = sp.axis
        slice_points = [slice_point for slice_point in sp.slice_point]
        return axis, slice_points

    def _load(self, model, sess):
        # Load data from file.
        data_dict = np.load(model).item()
        # For each layer type, load data.
        for layer_proto in self._layer_params:
            layer_name = layer_proto.name
            # Make sure layer has variables.
            if not self._has_vars(layer_proto):
                continue
            # It is possible that the file is missing data
            # for some layers. Warn the user.
            if layer_name not in data_dict:
                logging.warning("No variables found for layer: %s",
                                layer_name)
                continue
            # Load each variable in layer.
            for (name, vars) in self._layer_vars[layer_name]:
                # Still possible for stuff to be missing. This is unusual.
                # Warn the poor soul.
                if name not in data_dict[layer_name]:
                    logging.warning("Variables %s not found for layer %s",
                                    name,
                                    layer_name)
                    continue
                try:
                    # Time to assign values.
                    sess.run(vars.assign(data_dict[layer_name][name]))
                except ValueError as e:
                    # Something is very wrong. User likely has the wrong
                    # model file.
                    logging.warning("Dimension mismatch for variables"
                                    "%s in layer %s. %s.",
                                    name,
                                    layer_name,
                                    str(e))

    def _has_vars(self, layer_proto):
        return layer_proto.name in self._layer_vars

    def _satisfied_NSR(self, nsr):
        """Check if `NetStateRule` is satisfied based on `NetState`.

        Args:
            nsr: `NetStateRule` protobuf message.

        Returns:
            Boolean `True` iff `NetState` agrees with `nsr`.
        """
        np_state = self._net_param.state
        # Phase check.
        if nsr.phase != np_state.phase:
            return False

        # Level check.
        if np_state.level < nsr.min_level or np_state.level > nsr.max_level:
            return False

        # Stage check.
        stages = set(np_state.stage)
        for stage in nsr.stage:
            # Missing stage? Exclude.
            if stage not in stages:
                return False
        for not_stage in nsr.not_stage:
            # Stage should be there, but found? Exclude.
            if not_stage in stages:
                return False

        return True

    def _include_layer(self, lp):
        """Check if layer should be included in the network.

        Based on `NetState` of the `NetParameter`.

        Args:
            lp: `LayerParameter` protobuf message.

        Returns:
            Boolean `True` iff layer should be included.
        """
        # Default behavior.
        default = True
        # If inclusion rules defined, exclude by default.
        if lp.include:
            default = False
        # If exclusion rules defined, include by default.
        if lp.exclude:
            default = True

        # Include?
        for include_NSR in lp.include:
            if self._satisfied_NSR(include_NSR):
                return True
        # Exclude?
        for exclude_NSR in lp.exclude:
            if self._satisfied_NSR(exclude_NSR):
                return False
        return default

    def _FillerParameter_to_initializer(self, fp):
        """FillerParameter protobuf message to TensorFlow initializer.

        It is the caller's responsibility to ensure appropriate and
        consistent values are assigned to the FillerParameter fields.

        Args:
            fp: Protobuf message of type `FillerParameter`

        Returns:
            A TensorFlow initializer op.
        """
        if fp.type == "constant":
            return tf.constant_initializer(fp.value)
        elif fp.type == "uniform":
            return tf.random_uniform_initializer(fp.min, fp.max)
        elif fp.type == "gaussian":
            return tf.random_normal_initializer(fp.mean, fp.std)
        else:
            raise NotImplementedError("Initializer type %s not implemented" %
                                      fp.type)

    def _ReshapeParameter_to_shape(self, rp, bottom_shape):
        """ReshapeParameter protobuf message to shape.

        Args:
            rp: Protobuf message of type `ReshapeParameter`.
            bottom_shape: Shape of bottom layer to reshape. List or
                tuple of `int`.

        Return:
            List of `int` based on `ReshapeParameter` specs.

        Raises:
            `ValueError` in case of `ReshapeParameter` fields are
            incompatible with `bottom_shape`.
        """
        # First, figure out what dimensions of the bottom layer we
        # are working with, leaving behind a prefix and a suffix.
        bottom_len = len(bottom_shape)
        start_idx = (rp.axis + bottom_len) % bottom_len
        end_idx = bottom_len
        if rp.num_axes != -1:
            end_idx = start_idx + rp.num_axes + 1
        working_set = bottom_shape[start_idx:end_idx]

        # Reshape them.
        new_shape = [1] * len(rp.shape.dim)
        unk_idx = -1
        # Forward pass until -1 encountered.
        for idx, dim in enumerate(rp.shape.dim):
            if dim == 0:
                new_shape[idx] = working_set[idx]
            elif dim == -1:
                unk_idx = idx
                break
            else:
                new_shape[idx] = dim
        # If -1 encountered at `unk_idx`, we need a backward pass to
        # determine all remaining dimensions, except the one at the
        # aforementioned spot.
        if unk_idx != -1:
            # Iterate backwards through `new_shape`, until,
            # but excluding the index `unk_idx`.
            for idx, dim in enumerate(new_shape[:unk_idx:-1]):
                if dim == 0:
                    new_shape[-idx - 1] = working_set[-idx - 1]
                # Definitely not supposed to find a second
                # instance of -1.
                elif dim == -1:
                    raise ValueError("More than one instance of -1 encountered")
                else:
                    new_shape[-idx - 1] = dim
            # Now, on to figuring out the shape of the
            # mystery dimension.
            working_set_prod = np.prod(working_set)
            new_shape_prod = np.prod(new_shape)
            # Just in case the user messed up.
            if working_set_prod % new_shape_prod != 0:
                raise ValueError(
                    "Incompatible products while inferring unknown dim.")
            new_shape[unk_idx] = working_set_prod // new_shape_prod

        # Ensure compatibility.
        if np.prod(new_shape) != np.prod(working_set):
            raise ValueError("Incompatible shape specification.")

        # Append prefix and suffix, and return.
        return bottom_shape[:start_idx] + new_shape + bottom_shape[end_idx:]

    def _name_scope_builder(self, name):
        """Builds a re-enterable name scope.

        Args:
            name: Desired name scope. Type `str`.

        Returns:
            A name scope string.
        """
        cur_name = tf.get_variable_scope().name
        if cur_name:
            return "%s/%s/" % (cur_name.rstrip("/"), name.rstrip("/"))
        return "%s/" % name.rstrip("/")


import numpy as np
import copy
from scipy.interpolate import interp1d

def interpol_and_norm(mvc_normed):
    mvc_normed_ = {}

    for key, sensors in mvc_normed.items():
        mvc_normed_[key] = [list(sensor) for sensor in sensors]  # Shallow copy of nested arrays
        for sensor, data in enumerate(mvc_normed_[key]):
            for num, sensor_data in enumerate(data):
                x = np.linspace(0, len(sensor_data)-1, len(sensor_data))
                cubic_fx = interp1d(x, sensor_data, kind='cubic')
                resampled_x = np.linspace(0, len(sensor_data)-1, 101)
                mvc_normed_[key][sensor][num] = cubic_fx(resampled_x)

    return mvc_normed_


import os
import pickle
import numpy as np
import pandas as pd
from tqdm import tqdm

from pymantra.namemapping import NameMapper

def get_metabolite_cluster(name, meta):
    matches = meta['Exact Match to Standard (* = isomer family)'] == name
    return meta.index[matches].tolist()

def generate_map(file: str, ids):
    if not os.path.exists(file):
        name_mapper = NameMapper()
        mapping = {
            str(idx): name_mapper.map_id(idx, "hmdb", "internal")
            if str(idx).startswith("HMDB") else None
            for idx in ids
        }
        pickle.dump(mapping, open(file, "wb"))
        return mapping
    return pickle.load(open(file, "rb"))

def apply_mapping(data: pd.DataFrame, mapping_tups: set):
    index = []
    mapped_data = np.zeros((len(mapping_tups), data.shape[1]))
    for i, (idx, int_idx) in enumerate(tqdm(mapping_tups)):
        rep_data = data.loc[idx, :]
        if rep_data.values.ndim > 1:
            mapped_data[i, :] = rep_data.sum()
        else:
            mapped_data[i, :] = rep_data
        index.append(int_idx)
    mapped_df = pd.DataFrame(
        mapped_data, index=index,
        columns=data.columns
    )
    return mapped_df.groupby(mapped_df.index).sum()


################################################################################
#
# This file provides a datamodule which joins the voxceleb and librispeech
# dataset in a multi-task learning setting.
#
# Author(s): Anonymous
################################################################################

from dataclasses import dataclass
from typing import List, Optional

from pytorch_lightning.utilities.types import EVAL_DATALOADERS, TRAIN_DATALOADERS

from src.data.loading_config import SpeechDataLoaderConfig, SpeakerDataLoaderConfig
from src.data.modules import (
    LibriSpeechDataModuleConfig,
    VoxCelebDataModuleConfig,
    LibriSpeechDataModule,
    VoxCelebDataModule,
)
from src.data.modules.abstract import (
    SpeakerLightningDataModule,
    SpeechLightningDataModule,
)
from src.data.pipeline.base import Preprocessor
from src.evaluation.speaker.speaker_recognition_evaluator import EvaluationPair
from src.evaluation.speech.tokenizer import BaseTokenizer

################################################################################
# implement the data module

@dataclass
class JointLibrispeechVoxcelebDataModuleConfig:
    ls_cfg: LibriSpeechDataModuleConfig
    vc_cfg: VoxCelebDataModuleConfig

    test_voxceleb: bool
    test_librispeech: bool

class JointLibrispeechVoxcelebDataModule(
    SpeakerLightningDataModule, SpeechLightningDataModule
):
    def __init__(
        self,
        cfg: JointLibrispeechVoxcelebDataModuleConfig,
        speaker_dl_cfg: SpeakerDataLoaderConfig,
        speaker_train_pipeline: List[Preprocessor],
        speaker_val_pipeline: List[Preprocessor],
        speaker_test_pipeline: List[Preprocessor],
        speech_dl_cfg: SpeechDataLoaderConfig,
        tokenizer: BaseTokenizer,
        speech_train_pipeline: List[Preprocessor],
        speech_val_pipeline: List[Preprocessor],
        speech_test_pipeline: List[Preprocessor],
    ):
        super().__init__()

        self.cfg = cfg

        self.ls_dm = LibriSpeechDataModule(
            cfg=self.cfg.ls_cfg,
            dl_cfg=speech_dl_cfg,
            tokenizer=tokenizer,
            train_pipeline=speech_train_pipeline,
            val_pipeline=speech_val_pipeline,
            test_pipeline=speech_test_pipeline,
        )
        self.vox_dm = VoxCelebDataModule(
            cfg=self.cfg.vc_cfg,
            dl_cfg=speaker_dl_cfg,
            train_pipeline=speaker_train_pipeline,
            val_pipeline=speaker_val_pipeline,
            test_pipeline=speaker_test_pipeline,
        )

    @property
    def num_speakers(self) -> int:
        return self.vox_dm.num_speakers

    @property
    def val_pairs(self) -> List[EvaluationPair]:
        return self.vox_dm.val_pairs

    @property
    def test_pairs(self) -> List[List[EvaluationPair]]:
        return self.vox_dm.test_pairs

    @property
    def test_names(self) -> List[str]:
        names = []

        if self.cfg.test_librispeech:
            names += ["clean", "other"]

        if self.cfg.test_voxceleb:
            names += self.vox_dm.test_names

        return names

    @property
    def vocabulary(self) -> List[str]:
        return self.ls_dm.vocabulary

    @property
    def tokenizer(self) -> BaseTokenizer:
        return self.ls_dm.tokenizer

    def prepare_data(self) -> None:
        self.ls_dm.prepare_data()
        self.vox_dm.prepare_data()

    def setup(self, stage: Optional[str] = None) -> None:
        self.ls_dm.setup()
        self.vox_dm.setup()

    def summary(self):
        self.ls_dm.summary()
        self.vox_dm.summary()

    def train_dataloader(self) -> TRAIN_DATALOADERS:
        vox_dm_loader = self.vox_dm.train_dataloader().repeat().__iter__()
        ls_dm_loader = self.ls_dm.train_dataloader().repeat().__iter__()

        while True:
            speaker_batch = next(vox_dm_loader)
            speech_batch = next(ls_dm_loader)

            yield {"speech": speech_batch, "speaker": speaker_batch}

    def val_dataloader(self) -> EVAL_DATALOADERS:
        return [] + self.ls_dm.val_dataloader() + [self.vox_dm.val_dataloader()]

    def test_dataloader(self) -> EVAL_DATALOADERS:
        loaders = []

        if self.cfg.test_librispeech:
            loaders += self.ls_dm.test_dataloader()

        if self.cfg.test_voxceleb:
            loaders += (
                [self.vox_dm.test_dataloader()]
                if len(self.vox_dm.additional_test_sets) == 0
                else self.vox_dm.test_dataloader()
            )

        return loaders

    def predict_dataloader(self) -> EVAL_DATALOADERS:
        raise NotImplemented()


from sher.models import Account, Service
import simplejson
import urllib2
import urlparse

#service shoterning url,
#is.gd fallback
def shorten_url(url):

    if 'flickr' in url:
        parsed_url = urlparse.urlsplit(url)
        photo_id = parsed_url.path.split('/')[3]

        if photo_id:
            base58_id = base58(photo_id)
            url =  'http://flic.kr/p/%s' % base58_id

    elif 'youtube' in url:
        parsed_url = urlparse.urlsplit(url)
        video_id = urlparse.parse_qs(parsed_url.query)['v'][0]

        if video_id:
            url = 'http://youtu.be/%s' % video_id

    else:
        api = "http://is.gd/create.php?format=json&url=%s" % url
        try:
            json = simplejson.loads(urllib2.urlopen(api).read())
            url = json['shorturl']
        except:
            pass

    return url

def get_services_list(text):

    if text.find(',') != -1: #comma split
        slist = text.split(',')
    else:
        slist = text.split() #by space

    return slist

def get_account(service_name):
    return Account.objects.get(service_name__iexact=service_name)

def get_services(services):

    mod = __import__('sher.services', fromlist=['*'])

    sdict = {}

    for s in services:
        s = s.lower()
        try:
            sdict[s] = getattr(mod, '%s_service' % s)
        except AttributeError:
            pass

    for name,service in sdict.items():
        sdict[name] = service.authenticated(get_account(name))

    return sdict #return service name: authenticated api instance

def base58(n):
    alphabet='123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'
    result = ''
    while n >= 58:
        div,mod = divmod(int(n),58)
        result = alphabet[mod] + result
        n = div
    result = alphabet[n] + result
    base_count=len(alphabet)
    return result


from collections import deque
from bisect import bisect_left

class Solution:
    def swimInWater(self, grid: list[list[int]]) -> int:
        N = len(grid)
        seen = [[False] * N for _ in range(N)]
        seen[0][0] = True
        t = 0
        front = deque([(0, 0)])
        values = sorted([n for row in grid for n in row])
        i = bisect_left(values, max(grid[0][0], grid[-1][-1]))
        for t in values[i:]:
            next_front = deque()
            while front:
                r, c = front.pop()
                if t >= grid[r][c]:
                    for dr, dc in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
                        nr, nc = r + dr, c + dc
                        if 0 <= nr < N and 0 <= nc < N and not seen[nr][nc] and t >= grid[nr][nc]:
                            if nr == nc == N - 1:
                                return t

                            seen[nr][nc] = True
                            front.append((nr, nc))

                for dr, dc in [(0, 1), (0, -1), (1, 0), (-1, 0)]:
                    nr, nc = r + dr, c + dc
                    if 0 <= nr < N and 0 <= nc < N and not seen[nr][nc]:
                        next_front.append((r, c))
                        break

            front = next_front


#-*- coding:utf-8 -*-
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import matplotlib.patches as patches
import matplotlib.path as path
import json
from disk_schedule import ROUND,ALGORITHMS
from matplotlib.backends.backend_pdf import PdfPages

'''
Function draw_pic is based on the code in this page
http://matplotlib.org/examples/api/histogram_demo.html
'''

COLORS=['b','g', 'r','c','m','y','k' ,'grey']
'''
b: blue
g: green
r: red
c: cyan
m: magenta
y: yellow
k: black

'''

def draw_pic(data):
    pp = PdfPages('distributions.pdf',)
    a_pp=PdfPages('summary.pdf',)
    a_fig = plt.figure()
    for algorithm in ALGORITHMS:
        name=algorithm
        al_data=data[algorithm]

        mu, sigma = np.average(al_data),np.std(al_data)
        x = mu + sigma * np.random.randn(10000)
        fig = plt.figure()

        a_ax=a_fig.add_subplot(111)# for summary

        ax = fig.add_subplot(1,2,1)
        x=np.linspace(-5*sigma+mu,mu+5*sigma,100)
        y = mlab.normpdf(x,mu, sigma)
        l = ax.plot(x, y, 'r', linewidth=1)
        ax.set_xlabel('Normal Distribution')
        ax.set_ylabel('Probability')
        ax.set_title(r'$\ %s: \mu=%s,\ \sigma=%s$'%(name,mu,sigma))
        ax.autoscale_view()
        ax.grid(True)

        a_ax.plot(x, y,lw=8, label= algorithm,color= COLORS[ALGORITHMS.index(algorithm)], linewidth=1.0)
        a_ax.legend(loc='upper right')

        ax = fig.add_subplot(1,2,2)
        ax.set_xlabel('Distribution')
        n, bins = np.histogram(al_data, 40)
        left = np.array(bins[:-1])
        right = np.array(bins[1:])
        bottom = np.zeros(len(left))
        top = bottom + n
        XY = np.array([[left,left,right,right], [bottom,top,top,bottom]]).T
        barpath = path.Path.make_compound_path_from_polys(XY)
        patch = patches.PathPatch(barpath, facecolor='blue', edgecolor='gray', alpha=0.8)
        ax.add_patch(patch)
        ax.set_xlim(left[0], right[-1])
        ax.set_ylim(bottom.min(), top.max())
        ax.grid(True)
        fig.set_size_inches(12,8)
        pp.savefig(fig)
    a_fig.set_size_inches(52,8)
    a_pp.savefig(a_fig)
#    plt.show()# if you want to see the picture directly, uncomment this line
    a_pp.close()
    pp.close()

def reform_data():
    reform={}
    with open('output.txt') as data:
        data=data.read()
    data= json.loads(data)
    for no in range(1,ROUND+1):
        for algorithm in range(len(ALGORITHMS)):
            name=data[str(no)]["results"][algorithm]["name"]
            path_cost=data[str(no)]["results"][algorithm]["path_cost"]
            try:
                reform[name].append(path_cost)
            except:
                reform[name]=[path_cost]
    return reform

if __name__ =='__main__':
    data=reform_data()
    draw_pic(data)


from django.db import models
import datetime
from django.utils import timezone

class List(models.Model):
        AVAILABLE = 'Y'
        USED = 'N'
        STATUS_FOR_NOW = ((AVAILABLE, 'Available'), (USED, 'Used'),)
        ip = models.CharField('IP Address', max_length=20)
        status = models.CharField('Status', max_length=1, choices = STATUS_FOR_NOW, default=AVAILABLE)
        user = models.CharField('User Name', max_length=30)
        machine = models.CharField('Machine Name', max_length=50)
        location = models.CharField('location', max_length=50)
        note = models.CharField('Note', max_length=100)
        date = models.DateTimeField('Date started using')
        def __unicode__(self):
                return self.ip