import datetime

def create_log_group(client, name):
    response = client.describe_log_groups(logGroupNamePrefix=name)
    if response['logGroups']:
        print("Log group exists.")
    else:
        print("Creating log group...")
        client.create_log_group(logGroupName=name)

def get_log_stream(client, group, stream):
    response = client.describe_log_streams(
        logGroupName=group,
        logStreamNamePrefix=stream
    )
    # http://stackoverflow.com/a/365934/358804
    return next(iter(response['logStreams']), None)

def create_log_stream(client, group, stream):
    log_stream = get_log_stream(client, group, stream)
    if log_stream:
        print("Log stream exists.")
    else:
        print("Creating log stream...")
        client.create_log_stream(
            logGroupName=group,
            logStreamName=stream
        )

def get_latest_cw_event(cw_client, group, stream):
    cw_events = cw_client.get_log_events(
        logGroupName=group,
        logStreamName=stream,
        startFromHead=False,
        limit=1
    )

    if cw_events['events']:
        latest_event = cw_events['events'][0]
        timestamp = latest_event['timestamp'] / 1000
    else:
        # no previous event - query since the epoch
        timestamp = 0

    return datetime.datetime.utcfromtimestamp(timestamp)

def upload_logs(cw_client, group, stream, events, seq_token=None):
    print("Uploading {:d} events...".format(len(events)))

    # http://stackoverflow.com/a/8686243/358804
    log_args = {
        'logGroupName': group,
        'logStreamName': stream,
        'logEvents': events
    }
    if seq_token:
        log_args['sequenceToken'] = seq_token

    cw_client.put_log_events(**log_args)

def get_seq_token(cw_client, group, stream):
    log_stream = get_log_stream(cw_client, group, stream)
    return log_stream.get('uploadSequenceToken', None)


# python code to solve the linear advection equation using CTCS

import numpy as np # External library for numerical analysis
import matplotlib.pyplot as plt #plotting library

def initialBell (x):
    return np.where(x%1. < 0.5, np.power(np.sin(2*x*np.pi),2),0)

def main():
    # setting the intial pi profile and courant number
    nx = 40 # number of points plotting in space
    c = 0.2 # the courant number

    # spacial variable going from zero to one inclusive
    x = np.linspace(0.0, 1.0, nx+1)

    # three time levels of the dependent variable, phi
    phi = initialBell(x)
    phiNew = phi.copy()
    phiOld = phi.copy()

    #FTCS for the first timestep becuase we don't have a time -1
    for j in range (1,nx):
        phi[j] = phiOld[j] - 0.5*c*(phi[j+1] - phi[j-1])

    #apply periodic boundary conditions -> phi 0 and nx
    phi[0] = phiOld[0] - 0.5*c*(phiOld[1] - phiOld[nx-1])
    phi[nx] = phi[0]

    nt = 40
    for n in range (1, nt): #loop over time
        #loop over space
        for j in range (1,nx):
            phiNew[j] = phiOld[j] - c*(phi[j+1] - phi[j-1])
        phiNew[0] = phiOld[0] - c*(phi[1] - phi[nx-1])
        phiNew[nx] = phiNew[0]
        #update phi for the next step
        phiOld = phi.copy()
        phi = phiNew.copy()
        #plt.plot(x,phi)

    #derive quantities
    u = 1.
    dx = 1./nx
    dt = c*dx/u
    t = nt*dt

    plt.plot(x, initialBell(x-u*t), 'k', label = 'analytic')
    minimum = ('min' , min(initialBell(x-u*t)-phi))  #a method for numerically analyzing how well its working
    maximum = ('max' , max(initialBell(x-u*t)-phi))
    plt.plot(x, phi, 'b', label ='CTS n = 40')
    plt.legend(loc='best')
    plt.ylabel('$\phi$')
    plt.axhline(0,linestyle=':', color='black')

    plt.show()

main()
print('y')


import sys, traceback
import json
import os
import shutil
from pathlib import Path
import zipfile
import pprint

from os import path as pat
from shutil import copy

extension_mp3 = '.mp3'
extension_mp4 = '.mp4'
extension_jpg = '.jpg'

def only_one_element(a_list):
    element_num = len(a_list)
    if element_num > 1:
        try:
            exception = Exception("Folder have 2 or more same type src")
            print(exception)
            raise exception
        except:
            traceback.print_exc()
            sys.exit(1)

    return element_num != 0

def get_file_extension(path):
    return pat.splitext(path)[-1]
class node:
    nodes = {}
    counter = 0
    def get(path):
        if node.nodes.get(path,None):
            return node.nodes[path]
        return None

    def __init__(self,path,node_name):
        self.id = node.counter
        self.child = [None,None]
        self.src = []
        self.path = path
        self.node_name = node_name
        node.nodes[pat.join(path,node_name)] = self
        node.counter += 1

    def set_childL(self,child):
        self.child[0] = child
    def set_childR(self,child):
        self.child[1] = child

    def add_src(self,src):
        self.src.append(src)

    def get_dict(self):
        default_dict = {
                "nodeid":self.id,
                "video":"0x00",
                "audio":"0x00",
                "pic":"0x00",
                "childNodeL":'',
                "childNodeR":'',
                "name":self.node_name,
                }

        video_candiate = list(filter(lambda x: x.find(extension_mp4) != -1, self.src))
        audio_candiate = list(filter(lambda x: x.find(extension_mp3) != -1, self.src))
        picture_candiate = list(filter(lambda x: x.find(extension_jpg) != -1, self.src))

        if only_one_element(video_candiate): default_dict['video'] = str(node.id).zfill(5) + extension_mp4
        if only_one_element(audio_candiate): default_dict['audio'] = str(node.id).zfill(5) + extension_mp3
        if only_one_element(picture_candiate): default_dict['pic'] = str(node.id).zfill(5) + extension_jpg

        if self.child[0]: default_dict['childNodeL'] = self.child[0].id
        if self.child[1]: default_dict['childNodeR'] = self.child[1].id
        # print(json.dumps(default_dict))
        return default_dict

def unzip_file(file_name,output):
    def prev_dir(path):
        return os.path.relpath(os.path.join(path,'..'))

    with zipfile.ZipFile(file_name, 'r') as f:
        zipInfo = f.infolist()
        for member in zipInfo:
            member.filename = member.filename.encode('cp437').decode('utf-8')
            if member.filename.find('_MACOSX') != -1:
                continue
            f.extract(member,output)

root_path: str = './unzipped'
# file_to_unzip = '/Users/homemac/Desktop/今天我們要去哪探險呢？-2.zip' #sys.argv[1]
# unzip_file(file_to_unzip,root_path)
output_folder = './output'
# if not os.path.exists(output_folder):
#     os.makedirs(output_folder)

for _path, dirs, files in os.walk(root_path,topdown=True):
    path = os.path.abspath(_path)
    if _path == root_path:
        continue
    if not node.get(path,):
        _ = node(*pat.split(path))

    visiting_node = node.nodes[path]
    for dir in dirs:
        if not visiting_node.child[0]:
            visiting_node.set_childL( node(path,dir) )
        elif not visiting_node.child[1]:
            visiting_node.set_childR( node(path,dir) )
        else:
            try:
                exception = Exception("Cannot set child node, for parent node has full node")
                print(exception)
                raise exception

            except:
                traceback.print_exc()
                sys.exit(2)
    for file in files:
        if file.find('.') == 0 or file.find('_MACOSX') != -1:
            continue
        node.nodes[path].add_src(pat.join(path,file))

if True:
    output_path = pat.realpath(pat.join(root_path,'..',output_folder))
    json_info = {"Product":"StoryBook",
                 "DataStructure":"BinaryTree",
                 "node_nums":len(node.nodes),
                 "root_nodeid":0,
                 "node":[],
                 "action":[
                     {"btn_Name": "btn_left", "action": "do_left"},
                     {"btn_Name": "btn_right", "action": "do_right"},
                     {"btn_Name": "btn_confirm", "action": "do_confirm"},
                     {"btn_Name": "btn_home", "action": "do_home"},
                 ],

                 }
    for path,node in node.nodes.items():
        for src in node.src:
            src_extension = get_file_extension(src)
            src_path_output = pat.join( output_path \
                                        ,str(node.id).zfill(5)+src_extension)
            copy(src, src_path_output)
        json_info['node'].append(node.get_dict())
    #print(json.dumps(json_info))

    tree_info = {"nodes":{},
                 'struct':None}
    for node in json_info['node']:
        tree_info['nodes'][node['nodeid']] = node
    # print(tree_info)
    for node in tree_info['nodes'].values():
        children = []
        if node['childNodeL'] != '':
            lookup_node_id = node['childNodeL']
            children.append(tree_info["nodes"][lookup_node_id])
        if node['childNodeR'] != '':
            lookup_node_id = node['childNodeR']
            children.append(tree_info["nodes"][lookup_node_id])
        if len(children) != 0:
            node['children'] = children

        delete_attr = ['childNodeL','childNodeR','audio','video','pic','nodeid']
        for attr in delete_attr:
            # print(node)
            if node.get(attr,None) or node.get(attr,None) == '':
                del node[attr]
    print(json.dumps(tree_info['nodes'][0]))
    # with open('./tree.json','w') as file:
    #     file.write(json.dumps(tree_info['nodes'][0]))


from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404

# Create your views here.
from django.views.decorators.csrf import csrf_exempt

from authority.models import Authority
from role.models import Role

@csrf_exempt
def list(request):
    roles = Role.objects.all()
    data = []
    for role in roles:
        info = {'id': role.id, 'name': role.name, 'authorities': []}
        for auth in role.authorities.all():
            info['authorities'].append(auth.name)
        data.append(info)
    return JsonResponse({'roles': data})

@csrf_exempt
def add(request):
    if request.method == 'POST':
        name = request.POST['name']
        authorities = request.POST.getlist('authorities')
        print(authorities, '-----------')
        if Role.objects.filter(name=name).first() is None:
            role = Role(name=name)
            role.save()
            for authority in authorities:
                role.authorities.add(int(authority))
            role.save()
            res = {'status': 'ok', 'info': '添加成功'}
            print(res)
            return JsonResponse(res)
        else:
            res = {'status': 'no', 'info': '添加失败！角色已存在！'}
            return JsonResponse(res)

    else:
        authorities = Authority.objects.all()
        data = {'authorities': []}
        for authority in authorities:
            data['authorities'].append({'id': authority.id, 'name': authority.name, 'url': authority.url})
        # res = {'status': 'no', 'info': '添加失败！请用post请求！'}
        return JsonResponse(data)

@csrf_exempt
def edit(request):
    if request.method == 'GET':
        id = request.GET.get('id')
        role = get_object_or_404(Role, id=id)
        authorities = role.authorities.all()
        roleinfo={'id': id, 'name': role.name, 'authorities': []}

        for auth in authorities:
            roleinfo['authorities'].append(auth.id)

        data = {'role': roleinfo, 'authorities': []}

        authorities = Authority.objects.all()
        for authority in authorities:
            data['authorities'].append({'id': authority.id, 'name': authority.name, 'url': authority.url})
        print(data)
        return JsonResponse(data)
    else:
        id = request.POST['id']
        authorities=request.POST.getlist('authorities')
        print(authorities)
        role = get_object_or_404(Role, id=id)

        name = request.POST['name']
        if name!=role.name and Role.objects.filter(name=name).first:
            data = {'status': 'no', 'info': '编辑失败！角色名已存在！'}
            return JsonResponse(data=data)

        role.name = name
        if role.authorities!=authorities:
            role.authorities.clear()
            for auth in authorities:
                role.authorities.add(auth)
        role.save()
        data = {'status': 'ok', 'info': '成功'}
        return JsonResponse(data=data)

@csrf_exempt
def delete(request):
    id = request.POST['id']
    role = get_object_or_404(Role, id=id)
    role.delete()
    data = {'status': 'ok', 'info': '成功'}
    return JsonResponse(data=data)


#
# Copyright 2015-2019, Institute for Systems Biology
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import absolute_import

from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.cohorts_list, name='cohort_list'),
    url(r'^api/v1/$', views.views_api_v1.cohort_list_api, name='cohort_list_api'),
    url(r'^api/v2/$', views.views_api_v2.cohort_list_api, name='cohort_list_api'),
    #    url(r'^api/preview/$', views.cohort_preview_api, name='cohort_preview_api'),
    url(r'^api/v1/preview/manifest/$', views.views_api_v1.cohort_preview_manifest_api, name='cohort_preview_manifest_api'),
    url(r'^api/v2/preview/manifest/$', views.views_api_v2.cohort_preview_manifest_api, name='cohort_preview_manifest_api'),
    url(r'^api/v1/preview/query/$', views.views_api_v1.cohort_preview_query_api, name='cohort_preview_query_api'),
    url(r'^api/v2/preview/query/$', views.views_api_v2.cohort_preview_query_api, name='cohort_preview_query_api'),
    url(r'^(?P<cohort_id>\d+)/$', views.cohort_detail, name='cohort_details'),
#    url(r'^api/(?P<cohort_id>\d+)/$', views.cohort_detail_api, name='cohort_detail_api'),
    url(r'^(?P<cohort_id>\d+)/stats/$', views.get_cohort_stats, name='cohort_stats'),

#    url(r'^api/(?P<cohort_id>\d+)/$', views.cohort_detail_api, name='cohort_detail_api'),
    url(r'^api/v1/(?P<cohort_id>\d+)/manifest/$', views.views_api_v1.cohort_manifest_api, name='cohort_manifest_api'),
    url(r'^api/v2/(?P<cohort_id>\d+)/manifest/$', views.views_api_v2.cohort_manifest_api, name='cohort_manifest_api'),
    url(r'^api/v1/(?P<cohort_id>\d+)/query/$', views.views_api_v1.cohort_query_api, name='cohort_query_api'),
    url(r'^api/v2/(?P<cohort_id>\d+)/query/$', views.views_api_v2.cohort_query_api, name='cohort_query_api'),
    url(r'^save_cohort/', views.save_cohort, name='save_cohort'),
    url(r'^api/v1/save_cohort/', views.views_api_v1.save_cohort_api, name='save_cohort_api'),
    url(r'^api/v2/save_cohort/', views.views_api_v2.save_cohort_api, name='save_cohort_api'),
    url(r'^delete_cohort/', views.delete_cohort, name='delete_cohort'),
    url(r'^api/v1/delete_cohort/', views.views_api_v1.delete_cohort_api, name='delete_cohort_api'),
    url(r'^api/v2/delete_cohort/', views.views_api_v2.delete_cohort_api, name='delete_cohort_api'),
    #url(r'^clone_cohort/(?P<cohort_id>\d+)/', views.clone_cohort, name='clone_cohort'),
    #url(r'^share_cohort/$', views.share_cohort, name='share_cohorts'),
    #url(r'^share_cohort/(?P<cohort_id>\d+)/', views.share_cohort, name='share_cohort'),
    #url(r'^unshare_cohort/$', views.unshare_cohort, name='unshare_cohorts'),
    #url(r'^unshare_cohort/(?P<cohort_id>\d+)/', views.unshare_cohort, name='unshare_cohort'),
    #url(r'^set_operation/', views.set_operation, name='set_operation'),
    #url(r'^save_cohort_comment/', views.save_comment, name='save_cohort_comment'),
    url(r'^download_manifest/(?P<cohort_id>\d+)/', views.download_cohort_manifest, name='cohort_manifest'),
    url(r'bq_string/(?P<cohort_id>\d+)/', views.get_query_str_response, name='bq_string'),
    url(r'^download_manifest/', views.download_cohort_manifest, name='cohort_manifest_base'),
    url(r'^download_ids/(?P<cohort_id>\d+)/', views.cohort_uuids, name='download_ids'),
    url(r'^get_metadata_ajax/$', views.get_metadata, name='metadata_count_ajax')
]


""" Playbook Module """
import yaml
import re
import os
import codecs

class PlaybookParser(object):
    """ Parse An Individual Playbook """

    def __init__(self, playbooks, is_role=False):
        """ @playbooks is a list of paths to playbooks
            @is_role is used to determine if the playbook is part of a role
        """
        self.playbooks = playbooks
        self.parserdata = []
        self.is_role = is_role
        # basename of playbooks already parsed, to prevent infinate recrusion
        self.already_parsed_playbooks = []

    def parse_playbooks(self):
        """ Parse Each Playbook """
        for playbook in self.playbooks:
            self.parse_playbook(playbook)

    def __get_task_info__(self, task):
        '''
        @author: Y_mil
        @contact: lylinquiman@gmail.com
        @param task: variable task type
        @return: {'task_name': 'xxx', 'task_tags': ['xxx' | None ]} \
            or false in case the no have the name task
        @rtype: dict or boolean
        This Function go through all task and create the dict with task name \
        and tags. In this function is possibly adding more variables.
        '''
        if "name" in task:
            task_info = {'task_name': None, 'task_tags': None, 'task_with_items': None}
            task_name = task["name"]
            task_info["task_name"] = task_name
            if "tags" in task:
                if not task["tags"] == None:
                    task_info["task_tags"] = task["tags"]
            if "with_items" in task:
                if not task["with_items"] == None:
                    task_info["task_with_items"] = task["with_items"]
            return task_info
        return False

    def parse_playbook(self, playbook):
        """ Parse an Individual Playbook """
        with codecs.open(playbook, "r", encoding="utf-8") as f:
            # Get Rolename from filepath
            rolename = None
            if self.is_role:
                m = re.match(r".*/(.*?)/tasks/main.yml", playbook)
                if m:
                    rolename = m.group(1)

            # Do Not Parse If Its Already been Parsed
            playbook_base = os.path.basename(playbook)
            if self.is_role:
                # If Role, prepend rolename to make file unique to role
                playbook_base = "%s/%s" % (rolename, playbook_base)
            # Check if this file has alread been parsed
            if playbook_base in self.already_parsed_playbooks:
                return
            self.already_parsed_playbooks.append(playbook_base)

            # Setup Playbook Metadata
            playbookentry = {}
            playbookentry["relative_path"] = playbook
            playbookentry["rolename"] = rolename
            playbookentry["task_info"] = []

            # Read file content into data
            data = f.read()

            # Parse Comment Data
            for line in data.splitlines():
                m = re.match(r"^[ ]*#[ ]*(.*?)[ ]*:[ ]*(.*?)$", line)
                if m:
                    attribute = m.group(1)
                    value = m.group(2)

                    # Set An Attribute
                    if attribute.lower() == "author" or attribute.lower() == "description":
                        playbookentry[attribute.lower()] = value

            # Parse Task Names from playbook
            yamldata = yaml.load(data)
            # Skip Empty YAML Files
            if yamldata is None:
                return

            # Parase the Yaml
            for task in yamldata:
                # Playbooks have a tasks dict key
                if "tasks" in task:
                    task = task["tasks"]
                # Loop through Role tasks
                if isinstance(task, dict) and self.is_role:
                    task_info = self.__get_task_info__(task)
                    if not task_info == False:
                        playbookentry["task_info"].append(task_info)
                # Loop through Playbook tasks
                elif isinstance(task, list) and not self.is_role:
                    tasks = task
                    for task in tasks:
                        task_info = self.__get_task_info__(task)
                        if not task_info == False:
                            playbookentry["task_info"].append(task_info)
                # Loop through Playbook tasks
            self.parserdata.append(playbookentry)


import numpy as np
import pickle
from lime import label_decomposition
from pathlib import Path

from .inout import parseConfDict, fits_db
from .models import EmissionTensors, PhotoIonizationModels, EmissionFluxModel
from .grids import gridInterpolatorFunction, GridWrapper
from .components.atomic_model import IonEmissivity

def emissivity_grid_calc(lines_array, comp_dict, temp_grid_points=(9000, 20000, 251), den_grid_points=(1, 600, 101)):

    print(f'- Computing emissivity grids for {len(lines_array)} lines\n')

    # Compute the atomic data grids
    objIons = IonEmissivity(tempGrid=temp_grid_points, denGrid=den_grid_points)

    ion_array, wave_array, latex_array = label_decomposition(lines_array, comp_dict=comp_dict)

    # Define the dictionary with the pyneb ion objects
    ionDict = objIons.get_ions_dict(ion_array)

    # Compute the emissivity surfaces for the observed emission lines
    objIons.computeEmissivityGrids(lines_array, ionDict, combined_dict=comp_dict)

    # Compile exoplanet interpolator functions so they can be used wit numpy
    emisGridInterpFun = gridInterpolatorFunction(objIons.emisGridDict, objIons.tempRange, objIons.denRange)

    print(f'-- completed')

    return emisGridInterpFun

class SpectraSynthesizer(GridWrapper, PhotoIonizationModels):

    def __init__(self, emisGridInterFun=None, emis_tensors=None, grid_interp=None, ftau_coeff=None, grid_sampling=False):

        GridWrapper.__init__(self)
        PhotoIonizationModels.__init__(self)

        # Security checks
        self.lowTemp_check = None
        self.highTemp_check = None
        self.idcs_highTemp_ions = None
        self.grid_check = grid_sampling

        # Indexes
        self.indcsLabelLines = {}
        self.indcsIonLines = {}
        self.idcs_highTemp_ions = None
        self.ionicAbundCheck = {}

        # Simulation preload data
        self.ftauCoef = ftau_coeff
        self.emisGridInterpFun = emisGridInterFun
        self.emtt = emis_tensors
        self.gridInterp = grid_interp

        # Number of regions
        self.total_regions = 1

        # Output container
        self.fit_results = None

    def define_region(self, line_labels, line_fluxes, line_errs, lineFlambda=None, comp_dict=None, minErr=0.02):

        # Lines data
        ion_array, wave_array, latexLabel_array = label_decomposition(line_labels)
        self.lineLabels = line_labels
        self.lineIons = ion_array
        self.emissionFluxes = line_fluxes
        self.emissionErr = line_errs if line_errs is not None else line_fluxes * minErr
        self.lineFlambda = lineFlambda

        # Establish minimum error on lines
        if minErr is not None:
            err_fraction = self.emissionErr / self.emissionFluxes
            idcs_smallErr = err_fraction < minErr
            self.emissionErr[idcs_smallErr] = minErr * self.emissionFluxes[idcs_smallErr]

        # Parameters separation for direct and grid sampling models
        if not self.grid_check:

            # If not provided compute emissivity interpolation functions
            if self.emisGridInterpFun is None:
                self.emisGridInterpFun = emissivity_grid_calc(line_labels, comp_dict)

            # If not provided compute the emission flux tensors
            if self.emtt is None:
                self.emtt = EmissionFluxModel(self.lineLabels, self.lineIons)

        # else:
        #     self.HII_Teff_models(self.lineLabels, self.emissionFluxes, self.emissionErr)

        return

    def simulation_configuration(self, prior_conf_dict, highTempIons=None, T_low_diag='S3_6312A', T_high_diag='O3_4363A',
                                 verbose=True,):

        # Priors configuration
        for key, value in prior_conf_dict.items():
            if '_prior' in key:
                param = key.split('_prior')[0]
                priorConf = prior_conf_dict[param + '_prior']
                self.priorDict[param] = priorConf

        if 'logParams_list' in prior_conf_dict:
            self.priorDict['logParams_list'] = prior_conf_dict['logParams_list']

        # Load photoIonization models
        if not self.grid_check:

            self.idx_analysis_lines = np.zeros(self.lineLabels.size)

            # High and low temperature distinction
            self.lowTemp_check = any(T_low_diag in lineLabel for lineLabel in self.lineLabels) # TODO Change this for the intercept method
            self.highTemp_check = any(T_high_diag in lineLabel for lineLabel in self.lineLabels)

            # Index the lines
            self.label_ion_features(self.lineLabels, highTempIons)

        # # Interpolator object
        # if grid_interpolator is not None:
        #     self.gridInterp = grid_interpolator

        # self.obsIons = chemistry_model.obsIons
        # self.idcs_highTemp_ions = chemistry_model.indcsHighTemp # TODO this is dangerous repeat out

        if verbose:
            for i in range(self.lineLabels.size):
                print(f'-- {self.lineLabels[i]} '
                      f'({self.lineIons[i]})'
                      f'flux = {self.emissionFluxes[i]:.4f} +/- {self.emissionErr[i]:.4f} '
                      f'|| err/flux = {100 * self.emissionErr[i] / self.emissionFluxes[i]:.2f} %')

        return

    def label_ion_features(self, line_labels, highTempIons=None):

        ion_array, wavelength_array, latexLabel_array = label_decomposition(line_labels)

        # Establish the ions from the available lines
        self.obsIons = np.unique(ion_array)

        # Determine the line indeces
        for line in line_labels:
            self.indcsLabelLines[line] = (line_labels == line)

        # Determine the lines belonging to observed ions
        for ion in self.obsIons:
            self.indcsIonLines[ion] = (ion_array == ion)

        # Establish index of lines which below to high and low ionization zones # TODO increase flexibility for more Te
        if highTempIons is not None:
            self.idcs_highTemp_ions = np.in1d(ion_array, highTempIons)
        else:
            self.idcs_highTemp_ions = np.zeros(line_labels.size, dtype=bool)

        # Establish the ionic abundance logic from the available lines
        for ion in self.obsIons:
            self.ionicAbundCheck[ion] = True if ion in self.obsIons else False

        return

    def save_fit(self, output_address, ext_name='', output_format='pickle', user_header={}):

        output_path = Path(output_address)
        file_stem, file_root = output_path.stem, output_path.suffix

        if output_format == 'cfg':

            # Input data
            input_data = self.fit_results['inputs']
            sec_label = 'inputs' if ext_name == '' else f'{ext_name}_inputs'
            sec_dict = {}
            for i, lineLabel in enumerate(input_data['lines_list']):
                lineFlux, lineErr = input_data['line_fluxes'][i], input_data['line_err'][i]
                sec_dict[lineLabel] = np.array([lineFlux, lineErr])
            sec_dict['parameter_list'] = input_data['parameter_list']
            parseConfDict(str(output_address), sec_dict, section_name=sec_label, clear_section=True)

            # Output data
            sec_label = 'outputs' if ext_name == '' else f'{ext_name}_outputs'
            sec_dict = {}
            for param in self.fit_results['inputs']['parameter_list']:
                param_trace = self.fit_results['outputs'][param]
                sec_dict[param] = np.array([np.mean(param_trace), np.std(param_trace)])
            parseConfDict(str(output_address), sec_dict, section_name=sec_label, clear_section=True)

            # Synthetic fluxes
            sec_label = 'synthetic_fluxes' if ext_name == '' else f'{ext_name}_synthetic_fluxes'
            sec_dict = {}
            for lineLabel in self.fit_results['inputs']['lines_list']:
                line_trace = self.fit_results['outputs'][lineLabel]
                sec_dict[lineLabel] = np.array([np.mean(line_trace), np.std(line_trace)])
            parseConfDict(str(output_address), sec_dict, section_name=sec_label, clear_section=True)

        if output_format == 'pickle':
            with open(output_address, 'wb') as db_pickle:
                pickle.dump(self.fit_results, db_pickle)

        if output_format == 'fits':
            user_header['logP_values'] = dict(self.inferenModel.check_test_point())
            # user_header['r_hat'] = dict(pymc3.summary(self.fit_results['trace'])['r_hat']) # TODO check r_hat values
            fits_db(output_path, model_db=self.fit_results, ext_name=ext_name, header=user_header)


#! / usr / bin / env python
# -*-coding:utf-8-*-
import pygame, sys
import win32api, win32console, win32gui, codecs
import time, random
from pygame.sprite import Sprite

pygame.init()

# Colours
white = (255, 255, 255)
black = (0, 0, 0)
red = (255, 0, 0)
green = (0, 155, 0)
blue = (0, 0, 255)

# Display settings
display_width = 1440
display_height = 900
gameDisplay = pygame.display.set_mode((display_width, display_height))
pygame.display.set_caption("Ice Spy")

# Imagery
icon = pygame.image.load("ice.png")
pygame.display.set_icon(icon)
img = pygame.image.load("snakehead.png")
img2 = pygame.image.load("snaketail.png")
ice = pygame.image.load("ice.png")
clock = pygame.time.Clock()
iceThickness = 100
block_size = 40
FPS = 30000

smallfont = pygame.font.SysFont("Oswald", 25)
medfont = pygame.font.SysFont("Oswald", 50)
largefont = pygame.font.SysFont("Oswald", 80)

def pause():
    paused = True

    message_to_screen("Paused", black, -100, size="large")
    message_to_screen("Press C to continue or Q to quit", black, 25)

    pygame.display.update()

    while paused:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_c:
                    paused = False
                elif event.key == pygame.K_q:
                    pygame.quit()
                    quit()

        clock.tick(5)

def score(score):
    text = smallfont.render("Score: " + str(score), True, red)
    gameDisplay.blit(text, [0, 0])

def randice():
    randicex = round(random.randrange(0, display_width - iceThickness))
    randicey = round(random.randrange(0, display_height - iceThickness))
    return randicex, randicey

def snake(block_size, snakeList):
    if direction == "right":
        head = pygame.transform.rotate(img, 270)
        tail = pygame.transform.rotate(img2, 270)
    if direction == "left":
        head = pygame.transform.rotate(img, 90)
        tail = pygame.transform.rotate(img2, 90)
    if direction == "up":
        head = img
        tail = img2
    if direction == "down":
        head = pygame.transform.rotate(img, 180)
        tail = pygame.transform.rotate(img2, 180)

    gameDisplay.blit(head, (snakeList[-1][0], snakeList[-1][1]))
    #  gameDisplay.blit(tail, (snakeList[20][20], snakeList[20],[20]))

    for XnY in snakeList[:-1]:
        pygame.draw.rect(gameDisplay, green, (XnY[0], XnY[1], block_size, block_size))

def text_objects(text, color, size):
    if size == "small":
        textSurface = smallfont.render(text, True, color)
    elif size == "medium":
        textSurface = medfont.render(text, True, color)
    elif size == "large":
        textSurface = largefont.render(text, True, color)
    return textSurface, textSurface.get_rect()

def message_to_screen(msg, color, y_change=0, size="small"):
    textSurf, textRect = text_objects(msg, color, size)
    textRect.center = (display_width / 2), (display_height / 2) + y_change
    gameDisplay.blit(textSurf, textRect)

def gameLoop():
    global direction

    direction = "right"
    running = True
    gameOver = False

    len_x = display_width / 2
    len_y = display_height / 2

    len_x_change = 0
    len_y_change = 0

    snakeList = []
    snakeLength = 3
    Snaketail_x = len_x - len(snakeList)
    Snaketail_y = len_y - len(snakeList)
    randicex, randicey = randice()

    while running:
        if gameOver == True:
            message_to_screen("Game over", red, -50, size="large")
            message_to_screen("Press C to play again or Q to quit", white, 50, size="medium")
            pygame.display.update()

        if len_x != randicex and len_x > randicex and len_x >= (display_width - display_width + 1):
            len_x -= 1
            head = pygame.transform.rotate(img, 180)

        elif len_x != randicex and len_x < randicex and len_x <= (display_width - 1):
            len_x += 1
            head = img
        elif len_y != randicey and len_y > randicey and len_y >= (display_height - display_height + 1):
            len_y -= 1
            head = pygame.transform.rotate(img, 270)
        elif len_y != randicey and len_y < randicey and len_y <= (display_height - 1):
            len_y += 1
            head = pygame.transform.rotate(img, 90)

        while gameOver == True:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    gameOver = False
                    running = False
                if event.type == pygame.KEYDOWN:
                    if event.key == pygame.K_q:
                        running = False
                        gameOver = False
                    if event.key == pygame.K_c:
                        gameLoop()

        if len_x >= display_width or len_x < 0 or len_y < 0 or len_y >= display_height:
            gameOver = True

        len_x += len_x_change
        len_y += len_y_change
        gameDisplay.fill(black)

        gameDisplay.blit(ice, (randicex, randicey))
        snakeTail = []
        snakeTail.append(Snaketail_x)
        snakeTail.append(Snaketail_y)
        snakeHead = []
        snakeHead.append(len_x)
        snakeHead.append(len_y)
        snakeList.append(snakeHead)

        if len(snakeList) > snakeLength:
            del snakeList[0]
        for eachSegment in snakeList[:-1]:
            if eachSegment == snakeHead:
                gameOver = True

        # if moving right
        #   next_x = len_x + 60
        #   next_y = len_y
        # elif moving up
        # next_x = len_x
        # next_y = len_y + 60
        # elif ....

        # next_position = [next_x, next_y]

        # for eachSegment in snakeList[:-1]:
        #    if eachSegment == next_position:
        # Blocked -- need to choose a different direction

        snake(block_size, snakeList)

        score(snakeLength - 1)

        if len_x > randicex and len_x < randicex + iceThickness or len_x + block_size > randicex and len_x + block_size < randicex + iceThickness:
            if len_y > randicey and len_y < randicey + iceThickness:
                randicex, randicey = randice()
                snakeLength += 10
            elif len_y + block_size > randicey and len_y + block_size < randicey + iceThickness:
                randicex, randicey = randice()
                snakeLength += 10
        pygame.display.update()
        clock.tick(FPS)

    pygame.quit()
    quit()

gameLoop()


from django import forms
from django.contrib.auth import get_user_model
from ilan.models import Ilan, Comment
from django.conf import settings
from django.utils.translation import gettext_lazy as _

from ckeditor_uploader.widgets import CKEditorUploadingWidget

class IlanForm(forms.ModelForm):
    metin = forms.CharField(widget=CKEditorUploadingWidget())
    class Meta:
        model = Ilan
        fields = ['baslik', 'metin', 'ilan_foto', 'ilan_fon_muzik', 'arkaplan_rengi', 'arkaplan_resmi']
        labels = {
            'baslik': _('Başlık'),
            'metin': _('İlan Metni'),
            'ilan_foto': _('İlan Görseli'),
            'ilan_fon_muzik': _('Fon Müziği'),
            'arkaplan_rengi': _('Arkaplan Rengi'),
            'arkaplan_resmi': _('Arkaplan Resmi'),
        }

class CommentForm(forms.ModelForm):
    class Meta:
        model = Comment
        fields = ['yorum_metni']


from bnnbench.models import BaseModel, MCDropout, MCBatchNorm, DeepEnsemble, DNGO
from bnnbench.utils.data_utils import HPOBenchData
import logging
from enum import IntEnum

from bnnbench.emukit_interfaces.models import PyBNNModel
import numpy as np
from emukit.core.loop.model_updaters import NoopModelUpdater
from emukit.core.loop.candidate_point_calculators import RandomSampling
from emukit.core.loop import OuterLoop
from emukit.bayesian_optimization.loops.bayesian_optimization_loop import BayesianOptimizationLoop
from emukit.core import ParameterSpace
from emukit.core.loop.loop_state import LoopState, create_loop_state, UserFunctionResult
from emukit.examples.gp_bayesian_optimization.single_objective_bayesian_optimization import GPBayesianOptimization
from emukit.examples.gp_bayesian_optimization.enums import AcquisitionType
from typing import Sequence, Union, Tuple, Callable, Dict

_log = logging.getLogger(__name__)

class ModelType(IntEnum):
    MCDROPOUT = 0
    MCBATCHNORM = 1
    DNGO = 2
    ENSEMBLE = 3

model_classes = [MCDropout, MCBatchNorm, DNGO, DeepEnsemble]

def create_random_search_loop(space: ParameterSpace, initial_state: LoopState, **kwargs) -> OuterLoop:
    _log.debug("Generating new random search loop.")

    return OuterLoop(
        candidate_point_calculator=RandomSampling(parameter_space=space),
        model_updaters=NoopModelUpdater(),
        loop_state=initial_state
    )

def create_pybnn_bo_loop(model_type: ModelType, model_params: dict, space: ParameterSpace, initial_state: LoopState,
                         rng: Union[None, int, np.random.RandomState] = None, optimize_hypers_only_once: bool = False,
                         **kwargs) -> BayesianOptimizationLoop:

    _log.debug("Creating Bayesian Optimization Loop for PyBNN model of type %s, parameter space %s, and an initial "
               "loop state containing %d points." % (model_type, space, initial_state.X.shape[0]))
    model_type = model_classes[model_type]
    model_params = model_type.modelParamsContainer()._replace(**model_params, rng=rng)
    pybnn_model = PyBNNModel(model=model_type, model_params=model_params)
    pybnn_model.set_data(initial_state.X, initial_state.Y)
    if optimize_hypers_only_once:
        # Disable internal hyper-parameter optimization for all future calls to "fit()" for this model and store the
        # optimized model parameters for re-use in all upcoming calls to "fit()" on this object's model.
        pybnn_model.model_params = pybnn_model.model.model_params._replace(optimize_hypers=False)

    boloop = BayesianOptimizationLoop(space=space, model=pybnn_model)
    boloop.loop_state = LoopState(initial_results=initial_state.results[:])
    _log.info("BOLoop for PyBNN model of type %s initialized." % str(model_type).split('.')[-1])
    return boloop

def create_gp_bo_loop(space: ParameterSpace, initial_state: LoopState, acquisition_type: AcquisitionType,
                      **kwargs) -> BayesianOptimizationLoop:
    """ Creates a Bayesian Optimization Loop using a GP model. The keyword arguments are passed as is to
    emukit.examples.gp_bayesian_optimization.single_objective_bayesian_optimization.GPBayesianOptimization. """
    _log.debug("Generating new GP BO Loop with %s acquisition function." % str(acquisition_type))
    kwargs.pop("rng") # This isn't compatible
    loop = GPBayesianOptimization(variables_list=space.parameters, X=initial_state.X, Y=initial_state.Y,
                                  acquisition_type=acquisition_type, **kwargs)
    loop.loop_state = LoopState(initial_results=initial_state.results[:])
    return loop

class LoopGenerator:
    """ Generates a new Loop instance for use with Benchmarker by iterating over an internal data generator. Useful to
    substitute for a missing initialization hook in Benchmarker. """

    def __init__(self, loops: Sequence[Tuple[str, Callable, Dict]], data: HPOBenchData, seed: int):
        """
        :param loops: Array of 3-tuples (name, func, kwargs)
            Each tuple corresponds to the requirements for initializing a different Loop object. 'name' is a string,
            'func' is a function that returns the relevant loop object and should accept a keyword argument
            'initial_state' for a LoopState object used to initialize the Loop, and kwargs is a dictionary of fixed
            keyword arguments passed to func.
        :param data: HPOBenchData
            A data holder object that will be used to coordinate the current training/test splits.
        :param seed: int
            An integer value used to re-seed the global numpy RNG before generating every new Loop.
        """

        self._loops = loops
        self.data = data
        self.seed = seed

        def loop_cycle():
            nonlocal self
            while True:
                _log.debug("Starting new iteration of loop initializers.")
                self.data.update()
                for loop in self._loops:
                    yield loop

        self._loop_cycle = loop_cycle() # Actually responsible for properly iterating over the loops
        self.loop_list = [(l[0], self.generate_next_loop) for l in self._loops] # Sole Purpose: Expose to Benchmarker
        _log.info("Initialized LoopGenerator object for %d loops: %s" %
                  (len(self._loops), ", ".join(l[0] for l in self._loops)))

    def generate_next_loop(self, benchmarker_state: LoopState):
        """ Expected to be passed to the initializer of Benchmarker, will iteratively generate up-to-date Loop
        objects. """

        np.random.seed(self.seed)
        loop_name, loop_init_func, loop_kwargs = next(self._loop_cycle)
        _log.debug("Generating loop %s" % loop_name)
        init_state = self._create_initial_loop_state(benchmarker_loop_state=benchmarker_state)
        loop = loop_init_func(initial_state=init_state, **loop_kwargs, rng=np.random.randint(0, 1_000_000_000))
        return loop

    def _create_initial_loop_state(self, benchmarker_loop_state: LoopState):
        # Assume that the data object will handle generation of appropriate splits
        train_X, train_Y, train_meta = self.data.train_X, self.data.train_Y, self.data.train_meta
        meta_headers = self.data.meta_headers
        benchmarker_metadata = benchmarker_loop_state.results[-1].extra_outputs
        return create_loop_state(
            x_init=train_X,
            y_init=train_Y,
            # We only support 1d metadata
            **{key: train_meta[:, idx].reshape(-1, 1) for idx, key in enumerate(meta_headers)},
            # Use the most recent UserFunctionResult entry in the LoopState generated by Benchmarker to populate any
            # missing meta data headers. This ensures that all metadata headers are enabled in the new LoopState.
            **{key: np.full((train_meta.shape[0], 1), benchmarker_metadata[key])
               for key in benchmarker_metadata.keys() if key not in meta_headers}
        )


#!/usr/bin/env python2.6
# $Id$

import copy
import re
import unittest
import mox
import difflib
import checkpkg
import checkpkg_lib
import database
import models as m
import tag
import package_stats
import sqlite3
import sqlobject
import test_base
import common_constants

from testdata.tree_stats import pkgstats as tree_stats
from testdata.neon_stats import pkgstats as neon_stats

"""A set of unit tests for the library checking code.

A bunch of lines to test in the interactive Python shell.

import sys
sys.path.append("lib/python")
import checkpkg

sqlite3 ~/.checkpkg/var-sadm-install-contents-cache-build8x
SELECT * FROM systempkgmap WHERE basename = 'libncursesw.so.5';
"""

class ExtractorsUnitTest(unittest.TestCase):

  def testExtractDescriptionFromGoodData(self):
    data = {"NAME": "nspr_devel - Netscape Portable Runtime header files"}
    result = "Netscape Portable Runtime header files"
    self.assertEqual(result, checkpkg.ExtractDescription(data))

  def testExtractDescriptionWithBadCatalogname(self):
    data = {"NAME": "foo-bar - Bad catalogname shouldn't break this function"}
    result = "Bad catalogname shouldn't break this function"
    self.assertEqual(result, checkpkg.ExtractDescription(data))

  def testExtractMaintainerName(self):
    data = {"VENDOR": "https://ftp.mozilla.org/pub/mozilla.org/"
                      "nspr/releases/v4.8/src/ packaged for CSW by "
                      "Maciej Blizinski"}
    result = "Maciej Blizinski"
    self.assertEqual(result, checkpkg.ExtractMaintainerName(data))

  def testPstampRegex(self):
    pstamp = "hson@solaris9s-csw-20100313144445"
    expected = {
        'username': 'hson',
        'timestamp': '20100313144445',
        'hostname': 'solaris9s-csw'
    }
    self.assertEqual(expected, re.match(common_constants.PSTAMP_RE, pstamp).groupdict())

class SliceListUnitTest(unittest.TestCase):

  def testOne(self):
    l = [1, 2, 3, 4, 5]
    s = 1
    expected = [[1], [2], [3], [4], [5]]
    self.assertTrue(expected, checkpkg_lib.SliceList(l, s))

  def testTwo(self):
    l = [1, 2, 3, 4, 5]
    s = 2
    expected = [[1, 2], [3, 4], [5]]
    self.assertTrue(expected, checkpkg_lib.SliceList(l, s))

class SqliteUnitTest(unittest.TestCase):

  "Makes sure that we can lose state between tests."

  def setUp(self):
    self.conn = sqlite3.connect(":memory:")
    self.c = self.conn.cursor()

  def tearDown(self):
    self.conn = None

  def testCannotCreateTwoTables(self):
    self.c.execute("CREATE TABLE foo (INT bar);")
    self.assertRaises(
        sqlite3.OperationalError,
        self.c.execute, "CREATE TABLE foo (INT bar);")

  def testOne(self):
    self.c.execute("CREATE TABLE foo (INT bar);")

  def testTwo(self):
    self.c.execute("CREATE TABLE foo (INT bar);")

class SqlobjectUnitTest(test_base.SqlObjectTestMixin, unittest.TestCase):

  "Makes sure that we can lose state between methods."

  class TestModel(sqlobject.SQLObject):
    name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)

  # This does not work. Why?
  # def testCannotCreateTwoTables(self):
  #   self.TestModel.createTable()
  #   self.assertRaises(
  #       sqlite3.OperationalError,
  #       self.TestModel.createTable)

  def testOne(self):
    self.TestModel.createTable()

  def testTwo(self):
    self.TestModel.createTable()

if __name__ == '__main__':
  # import logging
  # logging.basicConfig(level=logging.DEBUG)
  unittest.main()


"""Bridges Midi from Voicelive to OSC RASPI, and to Local midi PC sampler. bridges notes from Raspi/LH to sampler"""

import logging
import sys
import time

sys.path.append("..")

import rtmidi
from rtmidi.midiutil import open_midiinput

from pythonosc import udp_client
from pythonosc import osc_server
from pythonosc import dispatcher

import os
import socket

from threading import Timer

from SmoothDefines import *

#==========================================================
# Constant definitions
hostname = socket.gethostname()
LOCALHOST_IP   = socket.gethostbyname(hostname)

LIVESERVER_PORT     = 8000
LH_PORT = 8010

INPUT_PORT  = 'AudioBox USB 96 MIDI In 1'
OUTPUT_PORT = 'Springbeats vMIDI4'

#==========================================================

class MidiInputHandler(object):
    def __init__(self, port):
        self.port = port
        self._wallclock = time.time()
        self.scheduled_timer = None

    def debounced_func(self, wait, func, *args, **kwargs):

        if self.scheduled_timer and not self.scheduled_timer.finished.is_set():
            print("cancelling Timer")
            self.scheduled_timer.cancel()

        self.scheduled_timer = Timer(wait, func, args=args, kwargs=kwargs)
        self.scheduled_timer.start()

    def debounce(self, wait, func, *args, **kwargs):
        """Returns a debounced version of a function.

        The debounced function delays invoking `func` until after `wait`
        seconds have elapsed since the last time the debounced function
        was invoked.
        """
        print("Debouncing")
        return self.debounced_func(wait, func, *args)

    def forwardProgramChange(self, value):
        print("Forwarding Program change : " + str(value))
        client.send_message("/midi/voicelive", [0xC0, value, 0])

        if(value == 1): #We don't receive the preset 1(index 0...), so let's use the number 2(index 1).
            #shutdown
            print("Shuting down immediately")
            client.send_message("/midi/shutdown", 1)
            cmd = "Shutdown -s -f -t 0"
            os.system(cmd)

    def __call__(self, event, data=None):
        message, deltatime = event
        self._wallclock += deltatime
        print("[%s] @%0.6f %r" % (self.port, self._wallclock, message))
        #Forward program changes received from Voicelive
        if(message[0] == 0xC0):
           #send program change thru OSC
           programVal = message[1]
           self.debounce(DEBOUNCE_PRESET_DELAY, self.forwardProgramChange, programVal)

        else:
           #Forward Control change received from Voicelive
           #print("Forwarding Control change" + str(message[1]))
           client.send_message("/midi/voicelive", [message[0], message[1], message[2]])

#Receives note on and off coming from LaserHarp
def print_laserharp_handler(osc_address, args, velocity):
  print("received OSC message from laser harp")
  print("address" + osc_address)
  print(velocity)

  temp = osc_address.split("/")
  note = int(temp[4])
  print (note);
  if(velocity != 0):
    note = [0x90, note, velocity]    # Note ON
  else:
    note = [0x80, note, 0]           #note OFF
  midiout.send_message(note)

#==========================================================
#Main code

#Open output port
midiout = rtmidi.MidiOut()
ports = midiout.get_ports()
for port, name in enumerate(ports):
    print("[%i] #%s#" % (port, name))
    if(OUTPUT_PORT in name):
       print("Opening output port [%i] #%s#" % (port, name))
       midiout.open_port(port)

log = logging.getLogger('midiin_callback')
logging.basicConfig(level=logging.DEBUG)

#Open input port
midiin = rtmidi.MidiIn()
ports = midiin.get_ports()
for port, name in enumerate(ports):
    print("[%i] #%s#" % (port, name))
    if(INPUT_PORT in name):
       print("Opening input port [%i] #%s#" % (port, name))
       midiin.open_port(port)
       port_name = name

client = udp_client.SimpleUDPClient(LOCALHOST_IP, LIVESERVER_PORT)

dispatcher = dispatcher.Dispatcher()
dispatcher.map("/vkb_midi/0/*", print_laserharp_handler, "midi_laserharp")

server = osc_server.ThreadingOSCUDPServer(
    (LOCALHOST_IP, LH_PORT), dispatcher)

print("Attaching MIDI input callback handler.")

midiin.set_callback(MidiInputHandler(port_name))

try:
    print("Entering main loop. Press Control-C to exit.")
    server.serve_forever()
except KeyboardInterrupt:

    exit()


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output

import pandas as pd

df = pd.read_csv('https://raw.githubusercontent.com/a-camarillo/NFL-running-backs-2019/master/Data%20Cleaning/rushing_stats_2019')

app = dash.Dash(__name__)

app.layout = html.Div([
    html.Div([
        dcc.Graph(
            id='graph-with-slider',
            clickData={'points':[{'text': 'Dalvin Cook'}]}),
        dcc.Slider(
         id='week-slider',
         min=df['week'].min(),
         max=df['week'].max(),
         value=df['week'].min(),
         marks={str(week): 'Week {}'.format(week) for week in df['week'].unique()},
            step=None
        )
    ]),
    html.Div([
        dcc.Graph(id='line-graph')
    ])
])

@app.callback(
    Output('graph-with-slider', 'figure'),
    [Input('week-slider', 'value')])
def update_figure(selected_week):
    filtered_df = df[df.week == selected_week]
    traces = []
    for i in filtered_df.player.unique():
        df_by_player = filtered_df[filtered_df['player'] == i]
        traces.append(dict(
            x=df_by_player['tot_yds'],
            y=df_by_player['tot_att'],
            text=df_by_player['player'],
            mode='markers',
            opacity=0.7,
            marker={
                'size': 15,
                'line' : {'width': 0.5, 'color': 'white'}
            },
            name=i
        ))
    return {
        'data': traces,
        'layout':dict(
            xaxis={'type':'scatter','title':'Yards'},
            yaxis={'type':'scatter','title':'Attempts'},
            hovermode='closest',
            transition = {'duration': 500}
        )
    }

@app.callback(
    Output('line-graph', 'figure'),
    [Input('graph-with-slider', 'clickData')])
def update_linegraph(clickData):
    player_name = clickData['points'][0]['text']
    player_df = df[df.player == player_name]
    return {
        'data':[dict(
            x=player_df['week'],
            y=player_df['yds']
        )],
        'layout': dict(
            xaxis={'title':'Week'},
            yaxis={'title':'Yards'},
            title=player_name
    )
}

if __name__ == '__main__':
    app.run_server(debug=True)


from django.http import QueryDict, HttpResponseRedirect
from django.views.generic import TemplateView
from django.contrib.sites.models import Site

from search.models import SearchPage
from aids.views import SearchView, AdvancedSearchView, AidDetailView
from alerts.views import AlertCreate

class MinisiteMixin:
    """Common code for all minisite urls.

    For every minisite view, we need to fetch the actual SearchPage object and
    inject it into the template.

    Note: there are several ways access the search page, hence there are
    several ways to access the SearchPage slug.

     1/ https://aides-territoires.beta.gouv.fr/recherche/<partner>/
     2/ https://<partner>.aides-territoires.beta.gouv.fr/
     3/ https://aides.<partner>.com (completely custom domain)

    In the fist case, we get the SearchPage's slug from the url parameter;
    Second case, from the subdomain;
    Third case, it is passed by a custom http header set in Nginx's settings.
    """

    def get(self, request, *args, **kwargs):
        self.search_page = self.get_search_page()

        if self.search_page is None:
            # Here, we have a bit of a problem.
            # If the SearchPage object is not found, we can't display a 404
            # error, because the 404 template needs a SearchPage to be
            # displayed.
            # Hence, the best way is to redirect to a sensible url.
            # Those errors are mainly caused by very old links with invalid
            # urls existing in the wild,
            # e.g https://www.aides-territoires.beta.gouv.fr
            return HttpResponseRedirect(self.get_redirection_url())

        return super().get(request, *args, **kwargs)

    def get_redirection_url(self):
        """What url to redirect to in case of missing SearchPage object.

        For now, just redirect to the main site's homepage.
        """
        site = Site.objects.get_current()
        url = 'https://{domain}'.format(domain=site.domain)
        return url

    def get_search_page(self):
        """Get the custom page from url."""

        HEADER = 'X-Minisite-Name'

        if 'search_slug' in self.kwargs:
            page_slug = self.kwargs.get('search_slug')
        elif HEADER in self.request.headers:
            page_slug = self.request.headers[HEADER]
        else:
            host = self.request.get_host()
            page_slug = host.split('.')[0]

        qs = SearchPage.objects.filter(slug=page_slug)
        try:
            obj = qs.get()
        except qs.model.DoesNotExist:
            obj = None
        return obj

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['search_page'] = self.search_page
        context['site_url'] = self.request.build_absolute_uri('').rstrip('/')
        return context

class SiteHome(MinisiteMixin, SearchView):
    """A static search page with admin-customizable content."""

    template_name = 'minisites/search_page.html'

    def get_form_kwargs(self):
        """Set the data passed to the form.

        If no data was provided by the user, then we use the initial
        querystring provided by admins.

        If the form was submitted, the GET values are set, we use those
        instead.
        """
        initial_data = QueryDict(
            self.search_page.search_querystring, mutable=True)
        user_data = self.request.GET.copy()
        user_data.pop('page', None)
        user_data.pop('integration', None)
        data = user_data or initial_data
        kwargs = super().get_form_kwargs()
        kwargs['data'] = data
        return kwargs

    def get_form(self, form_class=None):
        form = super().get_form(form_class)

        if self.search_page.available_categories:
            categories_qs = self.search_page \
                .available_categories \
                .select_related('theme')
            form.fields['categories'].queryset = categories_qs

        return form

class SiteSearch(MinisiteMixin, AdvancedSearchView):
    """The full search form."""

    template_name = 'minisites/advanced_search.html'

class SiteAid(MinisiteMixin, AidDetailView):
    """The detail page of a single aid."""

    template_name = 'minisites/aid_detail.html'

class SiteAlert(MinisiteMixin, AlertCreate):
    pass

class SiteLegalMentions(MinisiteMixin, TemplateView):
    template_name = 'minisites/legal_mentions.html'

class Error(MinisiteMixin, TemplateView):
    template_name = 'minisites/404.html'
    status_code = 404

    def render_to_response(self, context, **response_kwargs):
        response_kwargs['status'] = self.status_code
        return super().render_to_response(context, **response_kwargs)


from flask import current_app as app
from flask import jsonify, request, url_for
from . import api
from ..models import Shop, Offer

@api.route('/shops/<int:id>')
def get_shop(id):
	shop = Shop.query.get_or_404(id)
	return jsonify(shop.to_json())

@api.route('/shops/<int:id>/offers/')
def get_shop_offers(id):
	shop = Shop.query.get_or_404(id)
	page = request.args.get('page', 1, type=int)
	pagination = shop.offers.order_by(Offer.timestamp.desc()).paginate(page, per_page=app.config['PROSFORES4U_POSTS_PER_PAGE'], error_out=False)
	offers = pagination.items
	prev = None
	if pagination.has_prev:
		prev = url_for('api.get_shop_offers', id=id, page=page-1)
	next = None
	if pagination.has_next:
		next = url_for('api.get_shop_offers', id=id, page=page+1)
	return jsonify({
		'offers': [offer.to_json() for offer in offers],
		'prev': prev,
		'next': next,
		'count': pagination.total
	})


'''
Eric Pfleiderer
PHY3075-Numerical modelling: Modelisation numerique en phyique
Projet 5: Mutation adaptive
'''

import pandas as pd
import numpy as np
from scipy.optimize import bisect

'''
BUG:
    -All data becomes NaN if e>1 or e<-1, must impose bounderies?
    -No error caused by omega, but same idea? impose bounderies?
'''

'''
TO DO:
    -Constrain e (DONE)
    -Constrain w (DONE
    -Modify mutation_vector on the fly
    -Implement try/except on bisect optimizer in event f(a)*f(b)>0 (DONE)
    -Implement partial randomized initialization of parameters using input vector (continue from last opti)
'''

#Genetic optimizer
class GenOpti:

    #Constructor
    #N must be even!
    def __init__(self, N=10, vector_size=6, mutation_size=0.25, path='data/eta_Bootis.csv'):
        self.N = N
        self.vector_size = vector_size
        self.data = pd.read_csv(path)
        self.population = self.initialize_random() #P, tau, omega, e, K, V0
        self.mutation_vector = self.initialize_mutation(mutation_size=mutation_size)

    #Initialize population of N solution vectors according to specific intervals obtained from data analysis
    def initialize_random(self):

        population = np.zeros((self.N, self.vector_size))

        for n in range(self.N):
            population[n][0] = np.random.uniform(200, 800) #P
            population[n][1] = np.random.uniform(self.data.iloc[0, 0], self.data.iloc[0, 0] + population[n][0]) #tau
            population[n][2] = np.random.uniform(0, 2*np.pi) #omega
            population[n][3] = np.random.uniform(0, 1) #e
            population[n][4] = np.random.uniform(0, self.data['radial_velocity'].max() - self.data['radial_velocity'].min()) #K
            population[n][5] = np.random.uniform(self.data['radial_velocity'].min(), self.data['radial_velocity'].max()) #V0

        return population

    #Simulate the life of N generations with specified genetic pressures
    def simulate_generations(self, generations=20, selective_pressure=1/5, p_c=0.8, p_m=0.1):

        for generation in range(generations):

            if generation % 50 is 0:
                print('Generation:', generation)

            #Evaluate current solutions
            current_solution = self.evaluate_objective(self.population)

            #Rank solutions from worse to best by index
            rank = np.argsort(current_solution)

            #Select N new solutions, repetitions allowed (parents)
            parents_idx = np.empty(0)

            for n in range(self.N//2):
                best_parent, second_parent = self.generate_parents(rank, selective_pressure)
                parents_idx = np.append(parents_idx, best_parent)
                parents_idx = np.append(parents_idx, second_parent)
                parents_idx = np.int32(parents_idx)

            children = self.generate_kids(parents_idx, p_c)

            #Mutate children if need be
            for n in range(self.N):
                for component in range(self.vector_size):
                    if np.random.uniform(0, 1) <= p_m:
                        children[n][component] += np.random.normal(0, abs(self.mutation_vector[component]))

            #Constrain e to [0, 1] by definition
            mask_neg = np.where(children[:, 3] < 0)
            mask_pos = np.where(children[:, 3] > 1)
            children[mask_neg, 3] = 0.05
            children[mask_pos, 3] = 0.95

            #Constrain omega to positive values by definition
            mask_neg = np.where(children[:, 2] < 0)
            children[mask_neg, 2] = 0.05

            #Preserve best solution (elitism) and update current population
            top_sol = self.population[rank[self.N-1], :]
            self.population = children
            self.population[0, :] = top_sol

    #Generate a new population from the parents and genetic pressures
    def generate_kids(self, parents_idx, p_c=0.8):

        r = np.empty(self.vector_size)

        #Generate a ratio for every vector component
        for n in range(r.size):
            r[n] = np.random.uniform(0, 1)

        children = np.empty((self.N, self.vector_size))

        #For every couple of parents
        for n in range(self.N//2):

            #Find solution vectors for current pair of reproducing parents
            parent1_sol = self.population[parents_idx[2*n], :]
            parent2_sol = self.population[parents_idx[2*n+1], :]

            #Generate 2 new solutions vectors from parent vectors
            for component in range(parent1_sol.size):
                if np.random.uniform(0, 1) <= p_c:
                    children[2*n][component] = r[component] * parent1_sol[component] + (1-r[component]) * parent2_sol[component]
                    children[2*n+1][component] = (1-r[component]) * parent1_sol[component] + r[component] * parent2_sol[component]
                else:
                    children[2*n][component] = parent1_sol[component]
                    children[2*n+1][component] = parent2_sol[component]

        return children

    #Returns index of chosen parents from rank
    def generate_parents(self, rank, selective_pressure=1/5):
        # Choose best parent randomly according to selective pressure
        best_index = rank[np.random.randint(int((1 - selective_pressure) * self.N), self.N)]

        # Choose second parent
        second_index = best_index
        while second_index == best_index:
            second_index = np.random.randint(0, self.N - 1)

        return best_index, second_index

    #Evaluate objective function for each individual (solution matrix must be n by 6 2darray)
    def evaluate_objective(self, solution_matrix):
        chi = 0
        for idx, row in self.data.iterrows():
            chi += ((row[1] - self.evaluate_solution(solution_matrix, row[0]))/row[2])**2
        chi *= 1/(self.data.shape[0])

        return 1/chi

    #Evaluate solution at specific time t given a certain solution matrix
    def evaluate_solution(self, solution_matrix, t):

        E = np.empty(0)

        #Parameter syntaxic shortcuts
        P = solution_matrix[:, 0]
        tau = solution_matrix[:, 1]
        omega = solution_matrix[:, 2]
        e = solution_matrix[:, 3]
        K = solution_matrix[:, 4]
        V0 = solution_matrix[:, 5]

        #Solve keplers equation for the root
        for n in range(P.shape[0]):

            #Try/catch in case the interval does not bound the zero
            bounding_interval = False
            a, b = -1000, 1000
            while not bounding_interval:
                try:
                    E = np.append(E, bisect(self.evaluate_kepler, a, b, args=(t, P[n], tau[n], e[n])))
                    bounding_interval = True
                except:
                    a, b = 2*a, 2*b

        #Solve for projected velocity v
        orbital_velocity = 2*np.arctan(np.sqrt((1+e)/(1-e)) * np.tan(E/2)) #CANT IMPLEMENT ATAN2, ONLY 1 PARAM?

        #Solve for radial velocity
        radial_velocity = V0 + K * (np.cos(omega + orbital_velocity) + e*np.cos(omega))

        return radial_velocity

    #Keplers equation
    def evaluate_kepler(self, E, t, P, tau, e):
        return E - e * np.sin(E) - (2*np.pi/P) * (t-tau)

    #To check if needed for different order of parameters
    def initialize_mutation(self, mutation_size=0.25):

        mutation_vector = np.zeros(self.vector_size)

        for vector_idx in range(self.vector_size):
            mutation_vector[vector_idx] = mutation_size * (1/self.vector_size)*np.sum(self.population[:, vector_idx])

        return mutation_vector


import json
import requests

class GCMSender:
    URL_SEND = 'https://gcm-http.googleapis.com/gbm/send'

    def __init__(self, api_key):
        global headers
        headers = {
            'Content-Type': 'application/json',
            'Authorization': 'key=' + api_key
        }

    def send(self, message, timestamp, reg_token):
        global headers, URL_SEND
        payloads = {
            "data": {
                "message": message,
                "timestamp": timestamp
            },
            "to": reg_token
        }
        try:
            r = requests.post(url=URL_SEND, data=json.dumps(payloads), header=headers)
        except Exception as e:
            # Cannot complete POST requests
            pass


"""
Add coreference information to the GENIA data set.

Also remove blacklisted documents, for which the merge didn't go correctly and off-by-one-errors
were introduced.
"""

from collections import defaultdict

from os import path
import os
import glob
import re
import json
from bs4 import BeautifulSoup as BS
import pandas as pd
import argparse

import shared
import copy

from stanfordcorenlp import StanfordCoreNLP
from tqdm import tqdm

genia_base = "./data/genia"
genia_raw = f"{genia_base}/raw-data"
genia_processed = f"{genia_base}/processed-data"

json_dir = f"{genia_processed}/json-ner"
tree_dir = f"{genia_raw}/GENIA_treebank_v1"
coref_dir = f"{genia_raw}/GENIA_MedCo_coreference_corpus_1.0"
alignment_file = f"{genia_raw}/align/alignment.csv"

# output_directory = "json-treebank"
# output_directory = "json-matched"
# output_directory = "json-tf"
output_directory = "json-dep"
only_match = False
use_tree_feature = False
MAX_DEPTH = 3
MAX_PATH = 5

use_dep = True
use_tree = False

alignment = pd.read_csv(alignment_file).set_index("ner")

class Coref(object):
    """Represents a single coreference."""

    def __init__(self, xml, soup_text, sents):
        "Get the text, id, referrent, and text span."
        self.xml = xml
        self.text = xml.text
        self.tokens = [tok for tok in re.split('([ -/,.+])', self.text)
                       if tok not in ["", " "]]
        self.id = xml.attrs["id"]
        # A very small number of corefs have two parents. I'm going to just take the first parent.
        # TODO(dwadden) If time, go back and fix this.
        self.ref = xml.attrs["ref"].split(" ")[0] if "ref" in xml.attrs else None
        self.span = self._get_span(sents, soup_text)
        self.type = xml.attrs["type"] if "type" in xml.attrs else None

    def _get_span(self, sents, soup_text):
        """Get text span of coref. We have inclusive endpoints."""

        spans = shared.find_sub_lists(self.tokens, sents)
        n_matches = len(spans)
        # Case 1: If can't match the span, record and return. This doesn't happen
        # much.
        if n_matches == 0:
            stats["no_matches"] += 1
            return None
        # Case 2: IF there are multiple span matches, go back and look the original
        # XML tag to determine which match we want.
        elif n_matches > 1:
            xml_tag = self.xml.__repr__()
            tmp_ixs = shared.find_sub_lists(list(self.text), list(soup_text))
            text_ixs = []
            # Last character of the match must be a dash or char after must be an
            # escape, else we're not at end of token.
            text_ixs = [ixs for ixs in tmp_ixs if
                        soup_text[ixs[1] + 1] in '([ -/,.+])<' or soup_text[ixs[1]] == "-"]
            if len(text_ixs) != n_matches:
                # If the number of xml tag matches doesn't equal the number of span
                # matches, record and return.
                stats["different_num_matches"] += 1
                return None
            tag_ix = shared.find_sub_lists(list(xml_tag), list(soup_text))
            assert len(tag_ix) == 1
            tag_ix = tag_ix[0]
            text_inside = [x[0] >= tag_ix[0] and x[1] <= tag_ix[1] for x in text_ixs]
            assert sum(text_inside) == 1
            match_ix = text_inside.index(True)
        else:
            match_ix = 0
        stats["successful_matches"] += 1
        span = spans[match_ix]
        return span

class Corefs(object):
    """Holds all corefs and represents relations between them."""

    def __init__(self, soup, sents_flat, coref_types):
        self.coref_types = coref_types
        coref_items = soup.find_all("coref")
        corefs = [Coref(item, soup.__repr__(), sents_flat) for item in coref_items]
        # Put the cluster exemplars first.
        corefs = sorted(corefs, key=lambda coref: coref.ref is None, reverse=True)
        coref_ids = [coref.id for coref in corefs]
        corefs = self._assign_parent_indices(corefs, coref_ids)
        clusters = self._get_coref_clusters(corefs)
        clusters = self._cleanup_coref_clusters(corefs, clusters)
        cluster_spans = self._make_cluster_spans(clusters)
        self.corefs = corefs
        self.clusters = clusters
        self.cluster_spans = cluster_spans

    @staticmethod
    def _assign_parent_indices(corefs, coref_ids):
        """Give each coref the index of it parent in the list of corefs."""
        for coref in corefs:
            if coref.ref is None:
                coref.parent_ix = None
            else:
                coref.parent_ix = coref_ids.index(coref.ref)
        return corefs

    @staticmethod
    def _get_coref_clusters(corefs):
        def get_cluster_assignment(coref):
            ids_so_far = set()
            this_coref = coref
            while this_coref.ref is not None:
                # Condition to prevent self-loops.
                if this_coref.id in ids_so_far or this_coref.id == this_coref.ref:
                    return None
                ids_so_far.add(this_coref.id)
                parent = corefs[this_coref.parent_ix]
                this_coref = parent
            return this_coref.id

        clusters = {None: set()}
        for coref in corefs:
            if coref.ref is None:
                # It's a cluster exemplar
                coref.cluster_assignment = coref.id
                clusters[coref.id] = set([coref])
            else:
                cluster_assignment = get_cluster_assignment(coref)
                coref.cluster_assignment = cluster_assignment
                clusters[cluster_assignment].add(coref)
        return clusters

    def _cleanup_coref_clusters(self, corefs, clusters):
        """
        Remove items that didn't get spans, don't have an allowed coref type, or
        weren't assigned a cluster
        """
        # Remove unassigned corefs.
        _ = clusters.pop(None)
        for coref in corefs:
            # If the referent entity didn't get a span match, remove the cluster.
            if coref.ref is None:
                if coref.span is None:
                    _ = clusters.pop(coref.id)
            # If a referring coref didn't have a span or isn't the right coref type, remove it.
            else:
                if coref.type not in self.coref_types or coref.span is None:
                    # Check to make sure the cluster wasn't already removed.
                    if coref.cluster_assignment in clusters:
                        clusters[coref.cluster_assignment].remove(coref)
        # Now remove singleton clusters.
        # Need to make it a list to avoid `dictionary size changed iteration` error."
        for key in list(clusters.keys()):
            if len(clusters[key]) == 1:
                _ = clusters.pop(key)
        return clusters

    @staticmethod
    def _make_cluster_spans(clusters):
        """Convert to nested list of cluster spans, as in scierc data."""
        res = []
        for key, cluster in clusters.items():
            cluster_spans = []
            for coref in cluster:
                cluster_spans.append(list(coref.span))
            res.append(sorted(cluster_spans))
        return res

class Tree(object):
    def __init__(self, nodes):
        self.nodes = nodes
        self.match = True

        self.leaf_nodes = []
        self.leaf_nodes_idx = []
        for idx, node in enumerate(self.nodes):
            # if node.data != "":
            if len(node.children) == 0 and node.data != "":  # some leaf nodes don't correspond to tokens
                self.leaf_nodes.append(node)
                self.leaf_nodes_idx.append(idx)

    def get_span_for_leaf_node(self, sentence):
        if len(self.leaf_nodes) != len(sentence):
            self.match = False
            for idx, node in enumerate(self.leaf_nodes):
                node.span = [idx, idx]  # inclusive endpoints
        else:
            for idx, (node, text) in enumerate(zip(self.leaf_nodes, sentence)):
                if node.data != text:
                    self.match = False
                node.span = [idx, idx]  # inclusive endpoints

    def get_span_for_node(self, sentence):

        # assert (len(sentence) == len(self.leaf_nodes))
        # for idx, (node, text) in enumerate(zip(self.leaf_nodes, sentence)):
        #     assert (node.data == text)
        #     node.span = [idx, idx]  # inclusive endpoints

        sentence_start, sentence_end = Tree.get_span_for_node_(self.nodes, 0)
        # assert(sentence_start==0)
        # assert(sentence_end==len(sentence)-1)

        for node in self.nodes:
            if node.span[1] == -1: # update the span for Null constituent
                node.span[0] = -1

    def show_leaf_node(self):
        ret = [n.data for n in self.leaf_nodes]
        return ret

    @staticmethod
    def get_span_for_node_(nodes, node_idx):
        node = nodes[node_idx]
        if len(node.children) == 0:
            if node.data != "":
                return node.span[0], node.span[1]
            else:
                node.span = [999, -1]
                return 999, -1
        else:
            span_start = 999
            span_end = -1
            for child_idx in node.children:
                child_span_start, child_span_end = Tree.get_span_for_node_(nodes, child_idx)
                span_start = child_span_start if child_span_start < span_start else span_start
                span_end = child_span_end if child_span_end > span_end else span_end

            # assert(span_start<=span_end)
            node.span = [span_start, span_end]
            return span_start, span_end

    def print_tree(self):
        return Tree._print_node(self.nodes, 0, "", 0)

    @staticmethod
    def _print_node(nodes, node_idx, ret, level):
        node = nodes[node_idx]
        indent = '  '*level
        ret += '\n'+indent+'('+node.cat if node.data == '' else ' ('+node.cat+' '+node.data
        for child_idx in node.children:
            ret = Tree._print_node(nodes, child_idx, ret, level+1)
        ret += ')'
        return ret

    def to_json(self):
        nodes = []
        ret = {'match':self.match, 'nodes': nodes}
        for node in self.nodes:
            nodes.append(node.to_json())
        return ret

class Node(object):
    def __init__(self, parent, cat, data):
        self.parent = parent
        self.cat = cat
        self.data = data
        self.children = []

    def __str__(self):
        return self.cat+"|"+self.data+"|"+str(self.parent)

    def to_json(self):
        return [self.parent, self.children, self.cat, self.data, self.span]

def get_node(input, nodes, parent_idx, tokenized):
    for child in input.children:
        if child.name == 'cons':
            # if "null" in child.attrs:
            #     continue # ignore Null constituent
            node = Node(parent_idx, child.attrs["cat"], "")
            nodes.append(node)
            if parent_idx >= 0:
                nodes[parent_idx].children.append(len(nodes)-1)
            get_node(child, nodes, len(nodes)-1, tokenized)
        elif child.name == 'tok':
            if tokenized:
                tokens = [tok for tok in re.split('([ -/,.+])', child.text)
                           if tok not in ["", " "]]
                for token in tokens:
                    node = Node(parent_idx, child.attrs["cat"], token)
                    nodes.append(node)
                    nodes[parent_idx].children.append(len(nodes) - 1)
            else:
                node = Node(parent_idx, child.attrs["cat"], child.text)
                nodes.append(node)
                nodes[parent_idx].children.append(len(nodes)-1)

def matched(tree, sentence):
    if len(sentence) >= 3 and len(tree.leaf_nodes) >= 3:
        if tree.leaf_nodes[0].data == sentence[0] and tree.leaf_nodes[1].data == sentence[1] and tree.leaf_nodes[2].data == sentence[2] and \
            tree.leaf_nodes[-1].data == sentence[-1] and tree.leaf_nodes[-2].data == sentence[-2] and tree.leaf_nodes[-3].data == sentence[-3]:
            return True
        else:
            return False
    else:
        if tree.leaf_nodes[0].data == sentence[0] and tree.leaf_nodes[1].data == sentence[1] and \
            tree.leaf_nodes[-1].data == sentence[-1] and tree.leaf_nodes[-2].data == sentence[-2]:
            return True
        else:
            return False

def getPathToRoot(token, token_idx, tree):
    tmp = token
    # path_to_root = [token_idx]
    path_to_root = []
    while tmp.parent != -1:
        path_to_root.append(tmp.parent)
        tmp = tree.nodes[tmp.parent]
    return path_to_root

def getLowestCommonAncestor(token_i_path, token_j_path):
    lca = -1
    for i, node_idx_i in enumerate(token_i_path):
        for j, node_idx_j in enumerate(token_j_path):
            if node_idx_i == node_idx_j:
                lca = node_idx_i
                break
        if lca != -1:
            break
    assert lca != -1
    return i, j, lca

def getTreeFeature_Path(token_i, token_i_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    # ret = []
    # ii = 0
    # while ii < i:
    #     ret.append(tree.nodes[token_i_path[ii]].cat)
    #     ii += 1
    # ret.append(tree.nodes[lca].cat)
    # jj = j - 1
    # while jj >= 0:
    #     ret.append(tree.nodes[token_j_path[jj]].cat)
    #     jj -= 1
    # ret = '-'.join(ret)
    # return ret

    ret = []
    ii = 0
    jj = 0
    i_exhaust = False
    j_exhaust = False
    while (not i_exhaust) or (not j_exhaust):

        if ii < i:
            ret.append(tree.nodes[token_i_path[ii]].cat)
            ii += 1
        else:
            i_exhaust = True

        if len(ret) >= MAX_PATH:
            break

        if jj < j:
            ret.append(tree.nodes[token_j_path[jj]].cat)
            jj += 1
        else:
            j_exhaust = True

        if len(ret) >= MAX_PATH:
            break

    if len(ret) < MAX_PATH:
        ret.append(tree.nodes[lca].cat)

    ret = '-'.join(ret)
    return ret

def getTreeFeature_DirectionalPath1(token_i, token_i_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    ret = ''
    steps = []
    ii = 0
    while ii < i:
        if len(steps) >= MAX_PATH:
            break
        ret += tree.nodes[token_i_path[ii]].cat if len(steps) == 0 else ">"+tree.nodes[token_i_path[ii]].cat
        steps.append(tree.nodes[token_i_path[ii]].cat)
        ii += 1
    if len(steps) < MAX_PATH:
        ret += tree.nodes[lca].cat if len(steps) == 0 else ">"+tree.nodes[lca].cat
        steps.append(tree.nodes[lca].cat)
    jj = j - 1
    while jj >= 0:
        if len(steps) >= MAX_PATH:
            break
        ret += tree.nodes[token_j_path[jj]].cat if len(steps) == 0 else "<"+tree.nodes[token_j_path[jj]].cat
        steps.append(tree.nodes[token_j_path[jj]].cat)
        jj -= 1

    return ret

def getTreeFeature_DirectionalPath2(token_i, token_i_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    ret = ''
    ii = 0
    while ii < i:
        ret += tree.nodes[token_i_path[ii]].cat if len(ret) == 0 else ">"+tree.nodes[token_i_path[ii]].cat
        ii += 1
    ret += tree.nodes[lca].cat if len(ret) == 0 else ">"+tree.nodes[lca].cat
    jj = j - 1
    while jj >= 0:
        ret += tree.nodes[token_j_path[jj]].cat if len(ret) == 0 else "<"+tree.nodes[token_j_path[jj]].cat
        jj -= 1

    return ret

def getTreeFeature_LcaRootSyntax(token_i, token_i_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    return tree.nodes[lca].cat

def getTreeFeature_LcaLeftDepth(token_i, token_i_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    if i > MAX_DEPTH:
        i = MAX_DEPTH
    return str(i)

def getTreeFeature_LcaRightDepth(token_i, token_i_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    if j > MAX_DEPTH:
        j = MAX_DEPTH
    return str(j)

# i is the idx in tree.leaf_nodes, while token_i_idx is the idx in tree.nodes
def getTreeFeature_LcaMatch(token_i_leaf_idx, token_i, token_i_idx, token_j_leaf_idx, token_j, token_j_idx, tree):
    token_i_path = getPathToRoot(token_i, token_i_idx, tree)
    token_j_path = getPathToRoot(token_j, token_j_idx, tree)
    i, j, lca = getLowestCommonAncestor(token_i_path, token_j_path)

    span_left_token_idx = tree.nodes[lca].span[0]
    span_right_token_idx = tree.nodes[lca].span[1]

    left_leaf_idx = token_j_leaf_idx if token_i_leaf_idx > token_j_leaf_idx else token_i_leaf_idx
    right_leaf_idx = token_i_leaf_idx if token_i_leaf_idx > token_j_leaf_idx else token_j_leaf_idx

    if span_left_token_idx == left_leaf_idx and span_right_token_idx == right_leaf_idx:
        return 'm' # match
    elif span_left_token_idx == left_leaf_idx and span_right_token_idx != right_leaf_idx:
        return 'lm' # left match
    elif span_left_token_idx != left_leaf_idx and span_right_token_idx == right_leaf_idx:
        return 'rm' # right match
    else:
        return 'nm' # not match

def getTreeFeatures(tree):
    tree_features = {}
    if not tree.match:
        return tree_features
    # list_list, denotes the relation between token_i and token_j
    # e.g., given a sentence t1 t2 t3, return [[t1_t1, t1_t2, t1_t3],[t2_t1, t2_t2, t2_t3],[t3 ...]]
    tree_features['F1'] = []
    # tree_features['F2'] = []
    # tree_features['F3'] = []
    # tree_features['F4'] = []
    # tree_features['F5'] = []
    # tree_features['F6'] = []
    # tree_features['F7'] = []
    for i, (token_i, token_i_idx) in enumerate(zip(tree.leaf_nodes, tree.leaf_nodes_idx)):
        # f1, f2, f3, f4, f5, f6, f7 = [], [], [], [], [], [], []
        f1 = []
        for j, (token_j, token_j_idx) in enumerate(zip(tree.leaf_nodes, tree.leaf_nodes_idx)):
            if i == j:
                f1.append('self')
                # f2.append('self')
                # f3.append('self')
                # f4.append('self')
                # f5.append('self')
                # f6.append('self')
                # f7.append('self')
            else:
                f1.append(getTreeFeature_Path(token_i, token_i_idx, token_j, token_j_idx, tree))
                # f2.append(getTreeFeature_LcaRootSyntax(token_i, token_i_idx, token_j, token_j_idx, tree)) # lowest common ancestor
                # f3.append(getTreeFeature_LcaLeftDepth(token_i, token_i_idx, token_j, token_j_idx, tree))
                # f4.append(getTreeFeature_LcaRightDepth(token_i, token_i_idx, token_j, token_j_idx, tree))
                # f5.append(getTreeFeature_LcaMatch(i, token_i, token_i_idx, j, token_j, token_j_idx, tree))
                # f6.append(getTreeFeature_DirectionalPath1(token_i, token_i_idx, token_j, token_j_idx, tree))
                # f7.append(getTreeFeature_DirectionalPath2(token_i, token_i_idx, token_j, token_j_idx, tree))
        tree_features['F1'].append(f1)
        # tree_features['F2'].append(f2)
        # tree_features['F3'].append(f3)
        # tree_features['F4'].append(f4)
        # tree_features['F5'].append(f5)
        # tree_features['F6'].append(f6)
        # tree_features['F7'].append(f7)

    return tree_features

def getDepTree1(tokens, edges_list):
    nodes = [[] for token in tokens]
    nodes_dict = {"nodes":nodes}

    for edge in edges_list:
        if edge['dep'] == 'ROOT':
            continue
        if edge['dependent']-1 not in nodes[edge['governor']-1]:
            nodes[edge['governor']-1].append(edge['dependent']-1)
        if edge['governor']-1 not in nodes[edge['dependent']-1]:
            nodes[edge['dependent']-1].append(edge['governor']-1)

    return nodes_dict

def build_tree(soup, doc, tokenized, pmid, medline_id, nlp):
    tree_sentences = soup.find_all("sentence")

    # assert(len(tree_sentences)==len(sentences))
    if len(tree_sentences) != len(doc['sentences']):
        # print("doc mismatch, term doc: {}, doc len {}, treebank doc: {}, doc len {}".format(pmid, len(doc['sentences']), medline_id, len(tree_sentences)))
        stats_treebank['doc_mismatches'] += 1
    else:
        stats_treebank['doc_matches'] += 1

    trees = [] if use_tree else None
    if use_tree_feature:
        tf = []
    else:
        tf = None
    if use_dep:
        dep = []
    else:
        dep = None
    start = 0
    for sentence in doc['sentences']:
        b_match = False
        for idx, tree_sentence in enumerate(tree_sentences[start:]):
            nodes = []
            get_node(tree_sentence, nodes, -1, tokenized)
            tree = Tree(nodes)
            if matched(tree, sentence):
                b_match = True
                break
        if b_match:
            tree.get_span_for_leaf_node(sentence)
            if not tree.match:
                # print(
                #     "sent mismatch, term doc: {}, term sentence: {}, treebank doc: {}, treebank sentence {}".format(
                #         pmid, sentence, medline_id, tree.show_leaf_node()))
                stats_treebank['sent_mismatches'] += 1
            else:
                stats_treebank['sent_matches'] += 1
            tree.get_span_for_node(sentence)
            if use_tree:
                trees.append(tree.to_json())
            if use_tree_feature:
                tree_features = getTreeFeatures(tree)
                tf.append(tree_features)
            if use_dep:
                try:
                    nlp_res_raw = nlp.annotate(' '.join(sentence), properties={'annotators': 'tokenize,ssplit,pos,parse'})
                    nlp_res = json.loads(nlp_res_raw)
                    if len(nlp_res['sentences']) >= 2:
                        print("tokenization mismatch: {}".format(sentence))
                        dep.append({})
                    else:
                        dep_nodes = getDepTree1(nlp_res['sentences'][0]['tokens'],
                                                nlp_res['sentences'][0]['basicDependencies'])
                        dep.append(dep_nodes)
                except Exception as e:
                    print("nlp timeout: {}".format(sentence))
                    dep.append({})

            start = start + idx + 1
        else:
            # print(
            #     "sent not find tree, term doc: {}, term sentence: {}, treebank doc: {}".format(
            #         pmid, sentence, medline_id))
            stats_treebank['sent_notree'] += 1
            if use_tree:
                trees.append({})
            if use_tree_feature:
                tf.append({})
            if use_dep:
                dep.append({})

    # for tree_sentence, sentence in zip(tree_sentences, doc['sentences']):
    #     nodes = []
    #     get_node(tree_sentence, nodes, -1, tokenized)
    #     tree = Tree(nodes)
    #     # print(tree.print_tree())
    #     tree.get_span_for_leaf_node(sentence)
    #     if not tree.match:
    #         print("sent mismatch, term doc: {}, term sentence: {}, treebank doc: {}, treebank sentence {}".format(pmid, sentence, medline_id, tree.show_leaf_node()))
    #         stats_treebank['sent_mismatches'] += 1
    #     else:
    #         stats_treebank['sent_matches'] += 1
    #     tree.get_span_for_node(sentence)
    #     trees.append(tree.to_json())

    return trees, tf, dep

def get_excluded():
    "Get list of files that had random off-by-1-errors and will be excluded."
    current_path = os.path.dirname(os.path.realpath(__file__))
    excluded = pd.read_table(f"{current_path}/exclude.txt", header=None, squeeze=True).values
    return excluded

from collections import Counter

def is_span_overlapped(start, end, other_start, other_end):
    if other_start < start and other_end >= start:
        return True
    elif other_start <= end and other_end > end:
        return True
    elif other_start >= start and other_end <= end:
        return True
    else:
        return False

def one_fold(fold, coref_types, out_dir, keep_excluded, nlp):
    """Add coref field to json, one fold."""
    print("Running fold {0}.".format(fold))
    entity_length = Counter()
    entity_stat = dict(entity=0, entity_common=0)
    excluded = get_excluded()
    f_json = open(path.join(json_dir, "{0}.json".format(fold)))
    f_out = open(path.join(out_dir, "{0}.json".format(fold)), "w")

    lines = f_json.readlines()
    for line in tqdm(lines):
        doc = json.loads(line)

        for sentence, ner, relation_for_this_sentence in zip(doc['sentences'], doc['ner'], doc['relations']):
            for entity_idx, entity in enumerate(ner):
                entity_stat['entity'] += 1
                entity_length[entity[1]-entity[0]+1] += 1
                if entity[1] - entity[0] + 1 <= 8:
                    entity_stat['entity_common'] += 1
                for other_idx, other in enumerate(ner):
                    if other_idx == entity_idx:
                        continue
                    if entity[0] == other[0] and entity[1] == other[1]:
                        continue
                    if is_span_overlapped(entity[0], entity[1], other[0], other[1]):
                        relation_for_this_sentence.append(
                            [entity[0], entity[1], other[0],
                             other[1], "Overlap"])
            pass

        pmid = int(doc["doc_key"].split("_")[0])
        medline_id = alignment.loc[pmid][0]
        # xml_file = path.join(coref_dir, str(medline_id) + ".xml")
        # sents_flat = shared.flatten(doc["sentences"])
        xml_tree_file = path.join(tree_dir, str(medline_id) + ".xml")
        # with open(xml_file, "r") as f_xml:
        #     soup = BS(f_xml.read(), "lxml")
        #     corefs = Corefs(soup, sents_flat, coref_types)
        # doc["clusters"] = corefs.cluster_spans
        with open(xml_tree_file, 'r') as f_xml:
            soup = BS(f_xml.read(), "lxml")
            trees, tf, dep = build_tree(soup, doc, True, pmid, medline_id, nlp)
        if use_tree:
            doc["trees"] = trees
        if use_tree_feature:
            doc['tf'] = tf
        if use_dep:
            doc['dep'] = dep
        if only_match:
            good = True
            for tree in doc['trees']:
                if 'match' not in tree:
                    good = False
                    break
                if not tree['match']:
                    good = False
                    break
            if good:
                # Save unless it's bad and we're excluding bad documents.
                if keep_excluded or doc["doc_key"] not in excluded:
                    f_out.write(json.dumps(doc) + "\n")
            # else:
            #     a = 1
        else:
            # Save unless it's bad and we're excluding bad documents.
            if keep_excluded or doc["doc_key"] not in excluded:
                f_out.write(json.dumps(doc) + "\n")

    f_json.close()
    f_out.close()
    print(entity_length)
    print(entity_stat)

def get_clusters(coref_types, out_dir, keep_excluded, nlp):
    """Add coref to json, filtering to only keep coref roots and `coref_types`."""
    global stats
    stats = dict(no_matches=0, successful_matches=0, different_num_matches=0)
    global stats_treebank
    stats_treebank = dict(doc_mismatches=0, doc_matches=0, sent_mismatches=0, sent_matches=0, sent_notree=0)
    for fold in ["train", "dev", "test"]:
        one_fold(fold, coref_types, out_dir, keep_excluded, nlp)
    print(stats)
    print(stats_treebank)

def main():
    # get_coref_types()
    parser = argparse.ArgumentParser()
    parser.add_argument("--ident-only", action="store_true",
                        help="If true, only do `IDENT` coreferences.")
    parser.add_argument("--keep-excluded", action="store_true",
                        help="If true, keep training docs that were excluded due to off-by-1 errors.")
    args = parser.parse_args()
    if args.ident_only:
        coref_types = ["IDENT"]
    else:
        coref_types = [
            "IDENT",
            "NONE",
            "RELAT",
            "PRON",
            "APPOS",
            "OTHER",
            "PART-WHOLE",
            "WHOLE-PART"
        ]

    out_dir = f"{genia_processed}/{output_directory}"

    if not path.exists(out_dir):
        os.mkdir(out_dir)

    with StanfordCoreNLP('./files_for_discontinuous_ner/stanford-corenlp-full-2018-10-05', memory='8g', timeout=100000) as nlp:

        get_clusters(coref_types, out_dir, args.keep_excluded, nlp)

if __name__ == '__main__':
    main()


from django.dispatch import Signal

'''
Note that sometimes you will get duplicate signals emitted, depending on configuration of your systems.
If you do encounter this, you will need to add the "dispatch_uid" to your connect handlers

See http://code.djangoproject.com/wiki/Signals#Helppost_saveseemstobeemittedtwiceforeachsave for details
'''

# Sent when a payment is successfully processed.
payment_was_successful = Signal()

# Sent when a payment is flagged.
payment_was_flagged = Signal()

# Sent when a subscription cancellation is received/ processed
subscription_was_cancelled = Signal()

# Sent when a subscription End of Term is received/ processed
subscription_was_eot = Signal()

# Sent when a subscription End of Term is received/ processed
subscription_was_modified = Signal()

# Sent when a subscription End of Term is received/ processed
subscription_was_signed_up = Signal()