from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

# Dependency imports

import tensorflow.compat.v2 as tf

from tensorflow_probability.python.distributions import independent as independent_lib
from tensorflow_probability.python.distributions import sample as sample_lib
from tensorflow_probability.python.internal import prefer_static as ps
from tensorflow_probability.python.internal import samplers

class ShardedSample(sample_lib.Sample):
  """A version of `tfd.Sample` that shards its output across devices."""

  def __init__(self,
               distribution,
               sample_shape=(),
               shard_axis=0,
               validate_args=False,
               name=None):
    """Construct the `ShardedSample` distribution.

    Args:
      distribution: The base distribution instance to transform. Typically an
        instance of `Distribution`.
      sample_shape: `int` scalar or vector `Tensor` representing the shape of a
        single sample.
      shard_axis: `int` representing which axis of `sample_shape` will be
        sharded across devices.
      validate_args: Python `bool`.  Whether to validate input with asserts. If
        `validate_args` is `False`, and the inputs are invalid, correct behavior
        is not guaranteed.
      name: The name for ops managed by the distribution.
        Default value: `None` (i.e., `'Sample' + distribution.name`).
    """
    with tf.name_scope(name or 'ShardedSample' + distribution.name) as name:
      self._shard_axis = shard_axis

      super(ShardedSample, self).__init__(
          distribution,
          validate_args=validate_args,
          sample_shape=sample_shape,
          name=name)

  @property
  def sample_shape(self):
    sample_shape = ps.reshape(self._sample_shape, shape=[-1])
    shard_axis_size = sample_shape[self.shard_axis]
    num_devices = self.num_devices
    if shard_axis_size % num_devices != 0:
      raise ValueError('Does not shard evenly.')
    shard_size = shard_axis_size // num_devices
    sample_shape = ps.concat([
        sample_shape[:self.shard_axis], [shard_size],
        sample_shape[self.shard_axis + 1:]
    ],
                             axis=0)
    return sample_shape

  @property
  def shard_axis(self):
    return self._shard_axis

  @property
  def replica_id(self):
    ctx = tf.distribute.get_replica_context()
    return ctx.replica_id_in_sync_group

  @property
  def num_devices(self):
    ctx = tf.distribute.get_replica_context()
    return ctx.num_replicas_in_sync

  def _sample_n(self, n, seed, **kwargs):
    seed = samplers.sanitize_seed(seed, salt='sharded_sample_sample')
    return super(ShardedSample, self)._sample_n(n, seed + self.replica_id,
                                                **kwargs)

class ShardedIndependent(independent_lib.Independent):
  """A version of `tfd.Independent` that folds device id into its randomness."""

  @property
  def replica_id(self):
    ctx = tf.distribute.get_replica_context()
    return ctx.replica_id_in_sync_group

  def _sample_n(self, n, seed, **kwargs):
    seed = samplers.sanitize_seed(seed, salt='sharded_independent_sample')
    return super(ShardedIndependent, self)._sample_n(n, seed + self.replica_id,
                                                     **kwargs)


# -*- coding: utf-8 -*-
"""
Created on Fri Jun  4 16:20:37 2021

@author: Nabs
"""

#%% Importing Libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import decimal
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from numpy import mean
from numpy import std
from sklearn import tree
from sklearn.metrics import confusion_matrix

#%% Data Preprocessing
#dataset = pd.read_csv('data_m2.csv')
dataset = pd.read_csv(r"C:\Users\Abdul Basit Aftab\Desktop\MachineLearningThesis\Data + Codes\data_m5.CSV")
dataset['avg'] = dataset.iloc[:,4:7].mean(axis=1)

X_train = dataset.iloc[:1000,[4,5,6,10]].values

X_train = dataset.iloc[:,4:7].values
X_train_new = dataset.iloc[:,13]

#%% For Binning
criteria = [dataset['avg'].between(1, 4),
            dataset['avg'].between(6, 6.3),
            dataset['avg'].between(6.3, 6.50),
            dataset['avg'].between(6.50, 7.0),
            dataset['avg'].between(7.0, 7.3),
            dataset['avg'].between(7.3,7.7),
            dataset['avg'].between(7.7,8),
            dataset['avg'].between(8.0, 8.3),
            dataset['avg'].between(8.3,8.9)]
# criteria = [dataset['avg'].between(1, 4),
#             dataset['avg'].between(6, 6.3),
#             dataset['avg'].between(6.3, 6.5),
#             dataset['avg'].between(6.5, 7),
#             dataset['avg'].between(7, 7.3),
#             dataset['avg'].between(7.3,7.7),
#             dataset['avg'].between(8, 8.3),
#             dataset['avg'].between(8.7,8.9)]

def inrange(a, x, b):
    return min(a, b) < x < max(a, b)

from random import randint
values = [0,10, 20, 30,40,50,60,70,80]
#values1 = [randint(1250,1300),randint(1225,1275), randint(1200,1250), randint(1175,1225),randint(1150,1200),randint(1125,1175),randint(1100,1150),randint(1075,1125),randint(1050,1100)]

#Type_new = pd.Series([])
dataset['qualitytool'] = np.select(criteria, values, 0)
#dataset['Vibration'] = np.select(criteria, values1, 0)
#
#for i in range(dataset.shape[0]):
#    if  inrange(1,dataset["avg"][i],4):
#        Type_new[i]= randint(3,6)
#
#    elif inrange(6,dataset["avg"][i],6.3):
#        Type_new[i]= randint(4,7)
#
#    elif dataset["avg"][i] in range(6.3,6.50):
#        Type_new[i]= randint(4,7)
#
#    elif dataset["avg"][i] in range(6.50,7.0):
#        Type_new[i]= randint(5,8)
#
#    elif dataset["avg"][i] in range(7.0,7.3):
#        Type_new[i]= randint(6,9)
#
#    elif dataset["avg"][i] in range(7.3,7.7):
#        Type_new[i]= randint(7,10)
#
#    elif dataset["avg"][i] in range(7.7,8):
#        Type_new[i]= randint(5,8)
#
#    elif dataset["avg"][i] in range(8.0,8.3):
#        Type_new[i]= randint(5,8)
#
#    elif dataset["avg"][i] in range(8.3,8.9):
#        Type_new[i]= randint(5,8)

# inserting new column with values of list made above
#dataset.insert(14, "VIB", Type_new)
#%% TRaining Testing Dataset
Y_train = dataset.iloc[:,14].values

X_train = dataset.iloc[:,[4,5,6,10]].values
#X_train = X_train.append(dataset['Vibration']).values
#X_train = dataset.iloc[:,12].values
#X_train= X_train.reshape(-3, 3)
X= X_train
#Y_train= Y_train.reshape(-1, 1)
y= Y_train

#%% Test Train Split
X_train_new , X_test_new , Y_train_new , Y_test_new = train_test_split(X,y,test_size= 0.3, random_state= 0)
X= X_train
#Y_train= Y_train.reshape(-1, 1)
y= Y_train

#%% For accuract score
X_train_newest = dataset.iloc[:,[4,5,6,10]].values
Y_train_newest = dataset.iloc[:,14].values

#%% Logistic Regression

from sklearn.linear_model import LogisticRegression
lm = LogisticRegression(random_state=0)
lm.fit(X_train_new,Y_train_new)

y_pred_lm_split = lm.predict(X_train_newest)
thisisaccuracy_lm = accuracy_score(Y_train_newest, y_pred_lm_split)

print(y_pred_lm_split)
print(thisisaccuracy_lm)
lm_conf_mat = confusion_matrix(Y_train_newest,y_pred_lm_split)
print(lm_conf_mat)

#%% Decision Tree Classifier
clf2 = tree.DecisionTreeClassifier()
clf2 = clf2.fit(X_train_new,Y_train_new)
y_pred_clf2_split = clf2.predict(X_train_newest)
thisisaccuracy_clf2 = accuracy_score(Y_train_newest, y_pred_clf2_split)

print(thisisaccuracy_clf2)
clf2_conf_mat = confusion_matrix(Y_train_newest,y_pred_clf2_split)
print(clf2_conf_mat)

#%%Random Forest Regressor
from sklearn.ensemble import RandomForestClassifier
rfc = RandomForestClassifier(n_estimators=5,random_state=0,bootstrap=True)
rfc.fit(X_train_new,Y_train_new)

y_pred_rfc = rfc.predict(X_test_new)
thisisaccuracy_rfc = accuracy_score(Y_test_new, y_pred_rfc)
print(thisisaccuracy_rfc)

rfc_conf_mat = confusion_matrix(Y_test_new,y_pred_rfc)
print(rfc_conf_mat)

#%% Naive Bayes Theorem
from sklearn.naive_bayes import GaussianNB
clf4 = GaussianNB()
clf4.fit(X_train_new,Y_train_new)
a=clf4.predict(X_test_new)
#y_pred_clf4 = clf4.predict(X_train_newest)

thisisaccuracy_clf4 = accuracy_score(Y_test_new, a)
print(thisisaccuracy_clf4)

clf4_conf_mat = confusion_matrix(Y_test_new,a)
print(clf4_conf_mat)


import re
import string
from nltk.corpus import stopwords
from nltk.stem import *

file = open('Texto1.txt','rt')
text = file.read().lower()

words1=text.split()
words2=text.split()

#Primera forma de quitar simbolos
words1 = re.split(r'\W+',text)
print(words1[:100])

#Segunda forma de quitar simbolos
print(string.punctuation)
re_punc = re.compile('[%s]'%re.escape(string.punctuation))
stripped = [re_punc.sub('',w) for w in words2]
print(stripped[:100])

#longitud de las cadenas
print('word:',len(words1),'stripped:',len(stripped))

#Palabras vacias en ingles
stop_words = stopwords.words('english')
#print(stop_words)

cln_words = list()
for word in stripped[:100]:
    if(word not in stop_words):
        cln_words.append(word)

#PorterStemmer
print('\nPorterStemmer')
porter = PorterStemmer()
porterWords = [porter.stem(word) for word in cln_words]
print(' '.join(porterWords))

#Snowball
print('\nSnowball')
print('Lenguajes soportados:')
print(" ".join(SnowballStemmer.languages))
snowball = SnowballStemmer("english")
snowball2 = SnowballStemmer("english", ignore_stopwords=True)
Snowballwords = [snowball.stem(word) for word in cln_words]
print(' '.join(Snowballwords))

#wordnet
print('\nwordnet')
wnl = WordNetLemmatizer()
wnlwords = [wnl.lemmatize(word) for word in cln_words]
print(' '.join(wnlwords))

file.close()


import time
import requests
import re

def get_code_from_nada(email: str):
    if any(['@dropjar.com' in email, '@inboxbear.com' in email, '@tafmail.com' in email]):
        url = f"https://getnada.com/api/v1/inboxes/{email}"
        response = requests.get(url).json()
        try:
            subject = response['msgs'][0]['s']
            if subject == 'Proton Verification Code':
                uid = response['msgs'][0]['uid']
                code = re.search(r'\d{4,10}', requests.get(f"https://getnada.com/api/v1/messages/html/{uid}")
                                 .text.split('\n')[2])
                code = code.group() if code else code
        except:
            return None
    return code

def get_code_by_many_tries(email, tries=5, time_to_sleep=5):
    for i in range(tries):
        print(f"Try {i+1}")
        code = get_code_from_nada(email)
        if code:
            print('Code found!')
            return code
        else:
            print('Code not found.. ')
        time.sleep(time_to_sleep)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.test import override_settings
from django.utils.text import slugify
from test_plus.test import TestCase

from pycones.blog.models import Post
from pycones.blog.tests.factories import PostFactory
from pycones.users.tests.factories import UserFactory

@override_settings(LANDING_GLOBAL_REDIRECT=False)
class PostModelTests(TestCase):

    def test_create_post(self):
        prev_posts = Post.objects.count()
        Post.objects.create(
            author=UserFactory(),
            content="foo",
            title="Title",
            slug="Title"
        )
        self.assertEqual(prev_posts + 1, Post.objects.count())

    def test_create_slug(self):
        post = PostFactory(author=UserFactory())
        post.save()
        self.assertEqual(post.slug, slugify(post.title))

    def test_more(self):
        original_summary = "part1"
        post = Post.objects.create(
            author=UserFactory(),
            content="{}more".format(original_summary),
            title="Title",
            slug="Title"
        )
        summary = post.summary()
        self.assertEqual(summary, '{}
Seguir leyendo...
'.format(original_summary))


from collections import deque
from typing import Optional, List, Deque

class TreeNode:
    def __init__(self, x: int):
        self.val = x
        self.left = None
        self.right = None

class Codec:
    def serialize(self, root: Optional[TreeNode]) -> str:
        if not root:
            return ''

        queue = deque([root])
        res = []
        while queue:
            node = queue.popleft()
            if node:
                res.append(str(node.val))
                queue.extend([node.left, node.right])
            else:
                res.append('None')
        return ','.join(res)

    def deserialize(self, data: str) -> Optional[TreeNode]:
        if not data:
            return None

        ls = data.split(',')
        root = TreeNode(int(ls[0]))
        queue = deque([root])

        i = 1
        while queue and i < len(ls):
            node = queue.popleft()
            left_val, right_val = ls[i], ls[i+1] if i + 1 < len(ls) else 'None'

            if left_val != 'None':
                node.left = TreeNode(int(left_val))
                queue.append(node.left)

            if right_val != 'None':
                node.right = TreeNode(int(right_val))
                queue.append(node.right)

            i += 2

        return root

# Your Codec object will be instantiated and called as such:
root = TreeNode(1)
root.left = TreeNode(2)
root.right = TreeNode(5)
root.left.left = TreeNode(3)
root.left.right = TreeNode(4)

ser = Codec()
deser = Codec()

tree = ser.serialize(root)
print(tree)  # '1,2,5,3,4,None,None,None,None,None,None,'

ans = deser.deserialize(tree)
print(ans.val)  # 1
print(ans.left.val)  # 2
print(ans.right.val)  # 5
print(ans.left.left.val)  # 3
print(ans.left.right.val)  # 4


# -*- coding:utf-8 -*-
import xlrd, xlwt, xlutils
import requests,json
from xlutils.copy import copy
import time
import datetime

# xsfile = r'./MoliAccurancyData/TestCase5Domain.xls'
xsfile = r'E:\Moli\测试用例\python脚本\Moli精度脚本\Moli精度数据\US\USTestCase5Domain.xls' #北美测试集
# xsfile = r'E:\Moli\测试用例\python脚本\Moli精度脚本\Moli精度数据\IN\TestCase5Domain.xls' #印度测试集
# xsfile = r'E:\Moli\测试用例\python脚本\Moli精度脚本\测试集\线上测试全集\TestCase5Domain.xls' #线上数据全集
# xsfile = r'E:\Moli\测试用例\python脚本\Moli精度脚本\测试集\线上测试全集\TestCase5Domain全集(未筛选上线subintent）.xls' #线上数据全集
workbook = xlrd.open_workbook(xsfile, formatting_info= True,encoding_override='utf-8')
worksheet = workbook.sheet_by_name('TestCase')
cbook = copy(workbook)
csheet = cbook.get_sheet('TestCase')

# 设置字体颜色为红色
style_red = xlwt.XFStyle() #初始化excel全局样式
font_red = xlwt.Font()     #初始化font样式
font_red.colour_index = 2  #为font进行赋值
style_red.font = font_red       #将赋值好的font 传递给excel全局样式

# 设置字体颜色为绿色
style_green = xlwt.XFStyle()
font_green = xlwt.Font()
font_green.colour_index = 50
style_green.font = font_green

# 引入时间戳作为userid
ticks = time.time()

URL = r'http://10.110.147.198:9090/moliAgent/caap/centralcontrol'
row = 2

# 发送请求，并接受返回json串
def request(URL,message):
    payload = {
        "chnl": "pc",
        "isDebug": "true",
        "lan": "en",
        "geo": "US",
        "query": {
            "id": "c6c5f342-2ea1-4c",
            "msg": message,
            "type": "text"
        },
        "timestamp": 1542073217352,
        "userInfo": {
            "chnlUsrId":ticks ,
            "firstName": "",
            "ipAddress": "103.244.59.4",
            "lastName": "",
            "location": "",
            "timeZone": ""
        }
    }
    recieption = requests.post(URL, json=payload).json()
    recieption_statuscode = requests.get(URL, json=payload).status_code
    return recieption, recieption_statuscode

# 逐层判断并获取domain、intent、subintent信息
def rjson(recieption):
    # global recieption_subintent
    # 将接受过来的request返回的参数（元组）中的元素分别取出并使用
    recieption_json = recieption[0]
    recieption_statuscode = recieption[1]
    # 请求失效的异常处理
    if  recieption_statuscode != 200:
        csheet.write(row,13,'请求失败',style_red)
        recieption_domain = "empty"
        recieption_intent = "empty"
        recieption_subintent = "empty"
    # 正常流程下的数据判断及获取
    else:
        if recieption_json.__contains__("data"):
            if recieption_json["data"].__contains__("nluResult"):
                if recieption_json["data"]["nluResult"].__contains__("semanticFrames"):
                    recieption_domain = []
                    recieption_intent = []
                    recieption_subintent = []
                    for i in recieption_json["data"]["nluResult"]["semanticFrames"]:
                        if i.__contains__("domain"):
                            recieption_domain.append(i["domain"]["name"])
                            if i.__contains__("intent"):
                                recieption_intent.append(i["intent"]["code"])
                                if i.__contains__("subintent"):
                                    recieption_subintent.append(i["subintent"]["code"])
                                else:
                                    recieption_subintent = "empty"
                                    csheet.write(row,11,"未找到SubIntentCode", style_red)
                                    print('未找到subintent')
                            else:
                                recieption_intent = "empty"
                                recieption_subintent = "empty"
                                csheet.write(row, 8, "未找到IntentCode", style_red)

                        else:
                            recieption_domain = "empty"
                            recieption_intent = "empty"
                            recieption_subintent = "empty"
                            csheet.write(row, 6, "未找到Domain", style_red)
                else:
                    csheet.write(row, 15, "未找到semanticFrames", style_red)
            else:
                csheet.write(row, 14, "未找到nluResult", style_red)
        else:
            csheet.write(row, 13, "未找到data", style_red)
    return recieption_domain,recieption_intent,recieption_subintent

# 对比domain并写入
def comparison_domain(recieption):
    # 入参为rjson返回的包含三个元素的一个元组对象，需要拆开元组获取需要的对象，下方intent及subintent同理
    pointer = []
    recieption_domain = recieption[0]
    right_domain = worksheet.cell_value(row,5).split('|')
    if recieption_domain != 'empty':
        for j in right_domain:
        # if reflection[worksheet.cell_value(row, 5)] in recieption_domain:
            if reflection[j] in recieption_domain:
                p = True
                pointer.append(p)
                break
            else:
                p = False
                pointer.append(p)
        if True in pointer:
            # "|".join()函数可以将""中的符号作为列表、字符串、元组、字典中的分隔符插入
            csheet.write(row, 6, "|".join(recieption_domain), style_green)
        else:
            csheet.write(row, 6, "|".join(recieption_domain), style_red)
    else:
        print('第',row+1,'行domain为空')

# 对比intent并写入
def comparison_intent(recieption):
    pointer = []
    recieption_intent = recieption[1]
    right_intent = worksheet.cell_value(row,7).split('|')
    if recieption_intent != 'empty':
        for j in right_intent:
            # if reflection[worksheet.cell_value(row, 5)] in recieption_domain:
            if j in recieption_intent:
                p = True
                pointer.append(p)
                break
            else:
                p = False
                pointer.append(p)
        if True in pointer:
            csheet.write(row, 8, "|".join(recieption_intent), style_green)
        else:
            csheet.write(row, 8, "|".join(recieption_intent), style_red)
    else:
        print('第', row + 1, '行intent为空')

# 对比subintent并写入
def comparison_subintent(recieption):
    # 用于返回列表与内容列表是否存在相同项的判断计数
    pointer = []
    recieption_subintent = recieption[2]
    right_subintent = worksheet.cell_value(row,10).split('|')
    if recieption_subintent != 'empty':
        for j in right_subintent:
            # if reflection[worksheet.cell_value(row, 5)] in recieption_domain:
            if j in recieption_subintent:
                p = True
                pointer.append(p)
                break
            else:
                p = False
                pointer.append(p)
        if True in pointer:
            csheet.write(row, 11, "|".join(recieption_subintent), style_green)
            global right_count
            right_count += 1
        else:
            csheet.write(row, 11, "|".join(recieption_subintent), style_red)
            global wrong_count
            wrong_count += 1
    else:
        print('第', row + 1, '行subintent为空')
    return right_count, wrong_count

# 针对domain没有code构建映射键值对
reflection = {'After sale': 'after_sale' , 'fact': 'fact', 'How to': 'how_to', 'Pre sale': 'pre_sale', 'Trouble Shooting': 'trouble_shooting'}
# 循环发送请求，并在控制台打印运行信息

localtime_start = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(localtime_start)
# 用于计算正确及错误subintent数量
right_count = 0
wrong_count = 0

while row < worksheet.nrows:
    csheet.write(row,2,datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    message = worksheet.cell_value(row,4)
    return_message = request(URL, message)
    try:
        code = rjson(return_message)
        comparison_domain(code)
        # print(code[0])
        comparison_intent(code)
        # print(code[1])
        a = comparison_subintent(code)
    except:
        csheet.write(row, 12, str(return_message[0]), style_red)
    right_count = a[0]
    wrong_count_count = a[1]
    # print(code[2])
    json_data = return_message[0]
    csheet.write(row, 12, str(json_data))
    # print('当前第', row + 1, '行', '共', worksheet.nrows, '行')
    csheet.write(row, 3, datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
    print(code[0])
    print(code[1])
    print(code[2])
    print('This is the ', row + 1, 'row', 'Totally', worksheet.nrows, 'rows')
    print('-'*40)
    row += 1

# cbook.save(r'./MoliAccurancyResult/AccuracyTestResult.xls')
result = 'Total',worksheet.nrows-2,'rows',right_count,'Pass',wrong_count,'Fail','Accurate Rate is ', '%.2f%%' %(right_count/(worksheet.nrows-2)*100)
csheet.write(row+1,0,result)
cbook.save(r'./Moli精度结果/USAccuracyTestResult.xls') # 北美结果
# cbook.save(r'./Moli精度结果/INAccuracyTestResult.xls') # 印度结果
# cbook.save(r'./Moli精度结果/ALLAccuracyTestResult.xls')
loacaltime_end = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('Finished',loacaltime_end,'！'*40)
# print('Total',worksheet.nrows-2,'rows',right_count,'Pass',wrong_count,'Fail','Accurate Rate is ', '%.2f%%' %(right_count/(worksheet.nrows-2)*100))
print(result)

Moli精度测试北美脚本.py

Moli精度测试北美脚本.py


import logging

import pytest
from codenamesbot.state import (UNLIMITED, Game, GameMode, GamePhase, InvalidGameState, Player,
                                Players, PlayingTeam, Team)

logging.basicConfig(level=logging.DEBUG)

def test_player_rename():
    p = Player("tris")
    assert p.name == "tris"

    p.rename("nottris")
    assert p.name == "nottris"

def test_toggle_spymaster_preference():
    p = Player("tris")
    assert not p.spymaster_preference

    p.toggle_spymaster_preference()
    assert p.spymaster_preference

    p.toggle_spymaster_preference()
    p.toggle_spymaster_preference()
    assert p.spymaster_preference

def test_players_track_rename():
    ps = Players()

    tris = ps.add("tris")
    assert ps["tris"] == tris
    assert "tris" in ps

    ps["tris"].rename("nottris")
    assert ps["nottris"] is tris
    assert "tris" not in ps

    claire = ps.add("claire")
    assert tris != claire

    with pytest.raises(KeyError):
        ps["tris"]

def test_players_track_length():
    ps = Players()

    ps.add("tris")
    ps.add("claire")
    ps.add("bob")
    assert len(ps) == 3

    ps.remove("claire")
    assert len(ps) == 2

def test_players_shuffle():
    ps = Players()

    ps.add("tris")
    ps.add("claire")
    ps.add("bob")

    assert len(ps.shuffled()) == 3

def test_players_to_str():
    ps = Players()

    ps.add("tris")
    ps.add("claire")
    ps.add("bob")
    assert str(ps) == "3 players: tris, claire, bob"

    ps.remove("bob")
    ps.remove("tris")
    assert str(ps) == "1 player: claire"

def test_team_opposite():
    assert ~Team.GREEN == Team.PINK
    assert ~Team.PINK == Team.GREEN

def test_playingteam_normal():
    for t in [Team.GREEN, Team.PINK]:
        team = PlayingTeam(t)

        team.add(Player("tris"))
        team.add(Player("claire"))

        assert team.spymaster.name == "tris"
        assert len(team.guessers) == 1
        assert team.guessers[0].name == "claire"

        players = list(team)
        assert players == team.players

def test_playingteam_gray():
    team = PlayingTeam(Team.GRAY)

    team.add(Player("tris"))
    team.add(Player("claire"))

    assert team.spymaster is None
    assert len(team.guessers) == 2

def test_playingteam_length():
    team = PlayingTeam(Team.GRAY)

    team.add(Player("tris"))
    team.add(Player("claire"))

    assert len(team) == 2

def test_playingteam_contains():
    team = PlayingTeam(Team.GRAY)

    tris = Player("tris")
    claire = Player("claire")
    team.add(tris)

    assert tris in team
    assert claire not in team

def test_player_double_join():
    game = Game()
    game.join("tris")

    with pytest.raises(InvalidGameState):
        game.join("tris")

def test_player_double_leave():
    game = Game()

    with pytest.raises(InvalidGameState):
        game.leave("tris")

def test_assign_players_gray_small():
    game = Game()

    tris = game.join("tris")
    claire = game.join("claire")
    bob = game.join("bob")

    bob.team = Team.GREEN
    tris.team = Team.PINK

    game.mode = GameMode.GRAY
    game.assign_teams()

    assert tris.team == Team.PINK
    assert claire.team == Team.GRAY
    assert bob.team == Team.GREEN

    assert game.teams[Team.GREEN].spymaster == bob
    assert not game.teams[Team.GREEN].guessers

    assert game.teams[Team.PINK].spymaster == tris
    assert not game.teams[Team.PINK].guessers

    assert game.teams[Team.GRAY].guessers == [claire]

def test_assign_players_gray_big():
    for _ in range(100):
        game = Game()

        game.join("tris")
        game.join("claire")
        game.join("bob")
        alice = game.join("alice")
        eve = game.join("eve")

        # two players want to be on the green team
        # nobody in particular wants to be on the pink team
        alice.team = Team.GREEN
        eve.team = Team.GREEN

        game.mode = GameMode.GRAY
        game.assign_teams()

        # make sure alice and eve got on different teams, but one of them got on the green team
        green_preferred_resulting_teams = {alice.team, eve.team}
        assert len(green_preferred_resulting_teams) == 2
        assert Team.GREEN in green_preferred_resulting_teams

        # sanity check that
        assert game.teams[Team.GREEN].spymaster in {alice, eve}

        # make sure someone is the pink spymaster
        pink_spymaster = game.teams[Team.PINK].spymaster

        # make sure pink spymaster is not on gray team as well
        assert pink_spymaster.team == Team.PINK
        assert pink_spymaster not in game.teams[Team.GRAY].players

        # make sure gray team has exactly 3 players
        assert len(game.teams[Team.GRAY]) == 3

def test_assign_players_gray_spymaster_preference():
    for _ in range(100):
        game = Game()

        game.join("tris")
        game.join("claire")
        game.join("bob")
        alice = game.join("alice")
        eve = game.join("eve")

        alice.spymaster_preference = True
        eve.spymaster_preference = True

        game.mode = GameMode.GRAY
        game.assign_teams()

        # make sure alice and eve are the spymasters
        assert {game.teams[Team.GREEN].spymaster, game.teams[Team.PINK].spymaster} == {alice, eve}

def test_assign_players_gray_for_gray_preference():
    for _ in range(100):
        game = Game()

        tris = game.join("tris")
        claire = game.join("claire")
        bob = game.join("bob")
        alice = game.join("alice")

        bob.team = Team.GRAY
        alice.team = Team.GRAY

        game.mode = GameMode.GRAY
        game.assign_teams()

        assert {game.teams[Team.GREEN].spymaster, game.teams[Team.PINK].spymaster} == {tris, claire}

def test_assign_players_versus_all_team_preference():
    for _ in range(100):
        game = Game()

        tris = game.join("tris")
        claire = game.join("claire")
        bob = game.join("bob")
        alice = game.join("alice")
        eve = game.join("eve")

        tris.team = Team.GREEN
        bob.team = Team.GREEN
        claire.team = Team.PINK
        alice.team = Team.PINK
        eve.team = Team.PINK

        game.mode = GameMode.VERSUS
        game.assign_teams()

        assert tris.team == bob.team == Team.GREEN
        assert claire.team == alice.team == eve.team == Team.PINK

def test_assign_players_versus_two_spymasters():
    for _ in range(100):
        game = Game()

        tris = game.join("tris")
        claire = game.join("claire")
        game.join("bob")
        game.join("alice")
        game.join("eve")

        tris.spymaster_preference = True
        claire.spymaster_preference = True

        game.mode = GameMode.VERSUS
        game.assign_teams()

        spymaster_teams = {tris.team, claire.team}
        assert len(spymaster_teams) == 2

def test_assign_players_versus_team_preference_overrides_spymaster_preference():
    for _ in range(100):
        game = Game()

        tris = game.join("tris")
        claire = game.join("claire")
        game.join("bob")
        game.join("alice")
        game.join("eve")

        tris.spymaster_preference = True
        claire.spymaster_preference = True

        tris.team = Team.GREEN
        claire.team = Team.GREEN

        game.mode = GameMode.VERSUS
        game.assign_teams()

        assert tris.team == claire.team == Team.GREEN

def test_assign_players_versus_balanced_teams():
    for _ in range(100):
        game = Game()

        tris = game.join("tris")
        claire = game.join("claire")
        bob = game.join("bob")
        alice = game.join("alice")

        tris.team = claire.team = bob.team = Team.GREEN

        game.mode = GameMode.VERSUS
        game.assign_teams()

        assert alice.team == Team.PINK
        assert len(game.teams[Team.GREEN]) == 2
        assert len(game.teams[Team.PINK]) == 2

def test_assign_words():
    game = Game()

    game.join("tris")
    game.join("claire")
    game.join("bob")

    game.assign_words()

    assert len(game.words[Team.GREEN]) == 9
    assert len(game.words[Team.PINK]) == 8
    assert len(game.words[Team.GRAY]) == 7
    assert game.assassin is not None

def test_start_game():
    game = Game()

    game.join("tris")
    game.join("claire")
    game.join("bob")

    game.mode = GameMode.GRAY
    game.start_game()

    assert game.teams[Team.GREEN].spymaster is not None
    assert game.teams[Team.PINK].spymaster is not None
    assert len(game.teams[Team.GRAY].guessers) > 0

    assert game.words[Team.GREEN]
    assert game.words[Team.PINK]
    assert game.words[Team.GRAY]
    assert game.assassin is not None

def test_start_game_not_enough_players():
    game = Game()

    game.join("tris")
    game.join("claire")

    with pytest.raises(InvalidGameState):
        game.start_game()

def test_start_game_after_player_leaves():
    game = Game()

    game.join("tris")
    game.join("claire")
    game.join("bob")
    game.leave("bob")

    with pytest.raises(InvalidGameState):
        game.start_game()

def test_midgame_join_forced_gray():
    game = Game()

    game.join("tris")
    game.join("claire")
    game.join("bob")

    game.start_game()

    alice = game.join("alice")
    assert alice.team == Team.GRAY

    eve = game.join("eve", team=Team.GREEN)
    assert eve.team == Team.GRAY

def test_midgame_join_gray_in_versus():
    game = Game()

    game.join("tris")
    game.join("claire")
    game.join("bob")
    game.join("alice")

    game.mode = GameMode.VERSUS
    game.start_game()

    eve = game.join("eve", team=Team.GRAY)
    assert eve.team != Team.GRAY

def test_midgame_join_allowed_to_select_if_balanced():
    for _ in range(100):
        game = Game()

        game.join("tris")
        game.join("claire")
        game.join("bob")
        game.join("alice")

        game.start_game()

        eve = game.join("eve", team=Team.GREEN)
        assert eve.team == Team.GREEN

def test_midgame_join_forced_if_unbalanced():
    for _ in range(100):
        game = Game()

        game.join("tris", team=Team.GREEN)
        game.join("claire", team=Team.GREEN)
        game.join("bob", team=Team.GREEN)
        game.join("alice", team=Team.PINK)
        game.join("charlie", team=Team.PINK)

        game.start_game()

        eve = game.join("eve", team=Team.GREEN)
        assert eve.team == Team.PINK

    for _ in range(100):
        game = Game()

        game.join("tris", team=Team.PINK)
        game.join("claire", team=Team.PINK)
        game.join("bob", team=Team.PINK)
        game.join("alice", team=Team.GREEN)
        game.join("charlie", team=Team.GREEN)

        game.start_game()

        eve = game.join("eve", team=Team.PINK)
        assert eve.team == Team.GREEN

def test_midgame_join_assigned_if_unbalanced():
    game = Game()

    game.join("tris", team=Team.GREEN)
    game.join("claire", team=Team.GREEN)
    game.join("bob", team=Team.GREEN)
    game.join("alice", team=Team.PINK)
    game.join("charlie", team=Team.PINK)

    game.start_game()

    eve = game.join("eve")
    assert eve.team == Team.PINK

    game = Game()

    game.join("tris", team=Team.PINK)
    game.join("claire", team=Team.PINK)
    game.join("bob", team=Team.PINK)
    game.join("alice", team=Team.GREEN)
    game.join("charlie", team=Team.GREEN)

    game.start_game()

    eve = game.join("eve")
    assert eve.team == Team.GREEN

def setup_4p_game():
    game = Game()

    tris = game.join("tris", team=Team.GREEN)
    tris.spymaster_preference = True

    bob = game.join("bob", team=Team.PINK)
    bob.spymaster_preference = True

    game.join("claire", team=Team.GREEN)
    game.join("alice", team=Team.PINK)

    game.start_game()

    return game, tris

def test_hint_normal():
    game, tris = setup_4p_game()

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.GREEN

    game.hint(tris, "foo", 2)

    assert game.phase == GamePhase.GUESSING
    assert game.active_team == Team.GREEN
    assert game.current_hint == ("foo", 2)
    assert game.remaining_guesses == 3

def test_hint_from_disallowed_actors():
    game, tris = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.hint(game.players["claire"], "foo", 2)

    with pytest.raises(InvalidGameState):
        game.hint(game.players["bob"], "foo", 2)

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.GREEN

def test_hint_with_too_many_words():
    game, tris = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.hint(tris, "foo", 10)

def test_hint_is_active_word():
    game, tris = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.hint(tris, game.assassin, 1)

def test_hint_after_hint():
    game, tris = setup_4p_game()

    game.hint(tris, "foo", 1)

    assert game.phase == GamePhase.GUESSING

    with pytest.raises(InvalidGameState):
        game.hint(tris, "quux", 1)

def test_unlimited_hint():
    game, tris = setup_4p_game()

    game.hint(tris, "foo", UNLIMITED)

    assert game.current_hint == ("foo", UNLIMITED)
    assert game.remaining_guesses == UNLIMITED

def test_guess_civilian():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 4)

    word = game.words[Team.GRAY][0]
    game.guess(game.players["claire"], word)

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.PINK
    assert word not in game.all_words
    assert word not in game.remaining_words[Team.GRAY]
    assert word in game.words[Team.GRAY]

def test_guess_opposite_team():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 4)

    word = game.words[Team.PINK][0]
    game.guess(game.players["claire"], word)

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.PINK
    assert word not in game.all_words
    assert word not in game.remaining_words[Team.PINK]
    assert word in game.words[Team.PINK]

def test_guess_same_team():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 4)

    assert game.remaining_guesses == 5

    word = game.words[Team.GREEN][0]
    game.guess(game.players["claire"], word)

    assert game.phase == GamePhase.GUESSING
    assert game.active_team == Team.GREEN
    assert game.remaining_guesses == 4

    assert word not in game.all_words
    assert word not in game.remaining_words[Team.GREEN]
    assert word in game.words[Team.GREEN]

    with pytest.raises(InvalidGameState):
        game.guess(game.players["claire"], word)

    word = game.words[Team.GREEN][1]
    game.guess(game.players["claire"], word)

    assert game.remaining_guesses == 3

def test_guess_assassin():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 4)

    game.guess(game.players["claire"], game.assassin)

    assert game.phase == GamePhase.POST_GAME
    assert game.winner == Team.PINK

def test_running_out_of_guesses():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 1)

    game.guess(game.players["claire"], game.remaining_words[Team.GREEN][0])
    game.guess(game.players["claire"], game.remaining_words[Team.GREEN][0])

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.PINK

def test_stopping_guess_phase_early():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 1)

    game.guess(game.players["claire"], game.remaining_words[Team.GREEN][0])
    game.stop(game.players["claire"])

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.PINK

def test_spymaster_cannot_guess():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 1)

    with pytest.raises(InvalidGameState):
        game.guess(tris, game.remaining_words[Team.GREEN][0])

    assert game.phase == GamePhase.GUESSING
    assert game.active_team == Team.GREEN
    assert game.remaining_guesses == 2
    assert len(game.remaining_words[Team.GREEN]) == 9

def test_spymaster_cannot_stop_guessing():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 1)

    with pytest.raises(InvalidGameState):
        game.stop(tris)

    assert game.phase == GamePhase.GUESSING
    assert game.active_team == Team.GREEN

def test_cannot_guess_during_hint_phase():
    game, tris = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.guess(game.players["claire"], game.all_words[0])

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.GREEN

def test_cannot_stop_during_hint_phase():
    game, tris = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.stop(game.players["claire"])

    assert game.phase == GamePhase.HINTING
    assert game.active_team == Team.GREEN

def test_cannot_guess_for_other_team():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 1)

    with pytest.raises(InvalidGameState):
        game.guess(game.players["alice"], game.assassin)

    assert game.phase == GamePhase.GUESSING
    assert game.active_team == Team.GREEN
    assert game.remaining_guesses == 2

def test_cannot_stop_for_other_team():
    game, tris = setup_4p_game()
    game.hint(tris, "foo", 1)

    with pytest.raises(InvalidGameState):
        game.stop(game.players["alice"])

    assert game.phase == GamePhase.GUESSING
    assert game.active_team == Team.GREEN

def test_winning():
    game, tris = setup_4p_game()
    game.hint(tris, "pony", 9)

    for word in game.words[Team.GREEN]:
        game.guess(game.players["claire"], word)

    assert game.phase == GamePhase.POST_GAME
    assert game.winner == Team.GREEN

def test_winning_during_guessing_phase():
    game, tris = setup_4p_game()
    game.hint(tris, "pony", UNLIMITED)

    game.stop(game.players["claire"])
    game.hint(game.players["bob"], "pony", 7)

    for word in game.words[Team.PINK][:-1]:
        game.guess(game.players["alice"], word)

    game.stop(game.players["alice"])
    game.hint(tris, "pony", UNLIMITED)

    game.guess(game.players["claire"], game.remaining_words[Team.PINK][0])

    assert game.phase == GamePhase.POST_GAME
    assert game.winner == Team.PINK

def test_winning_during_guessing_phase_with_0_guesses_remaining():
    game, tris = setup_4p_game()
    game.hint(tris, "pony", UNLIMITED)

    game.stop(game.players["claire"])
    game.hint(game.players["bob"], "pony", 7)

    for word in game.words[Team.PINK][:-2]:
        game.guess(game.players["alice"], word)

    game.stop(game.players["alice"])
    game.hint(tris, "pony", 1)

    game.guess(game.players["claire"], game.words[Team.GRAY][0])

    game.hint(game.players["bob"], "pony", 1)

    game.guess(game.players["alice"], game.remaining_words[Team.PINK][0])
    game.guess(game.players["alice"], game.remaining_words[Team.PINK][0])

    assert game.phase == GamePhase.POST_GAME
    assert game.winner == Team.PINK

def test_leave_during_game():
    game, tris = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.leave(tris)

def test_join_after_game():
    # the game state needs to be reset externally before new joins are accepted
    game, tris = setup_4p_game()
    game.hint(tris, "pony", 9)
    game.guess(game.players["claire"], game.assassin)

    assert game.phase == GamePhase.POST_GAME

    with pytest.raises(InvalidGameState):
        game.join("steve")

def test_start_ongoing_game():
    game, _ = setup_4p_game()

    with pytest.raises(InvalidGameState):
        game.start_game()


"""users"""

from django.urls import path, include

from . import views

app_name = 'users'
urlpatterns = [
    # auth urls адрес по умолчанию.
    path('', include('django.contrib.auth.urls')),
    # Страница регистрации
    path('register/', views.register, name='register'),
]


#! /usr/bin/env python
# coding: utf-8

"""
calculate mfcc for each .wav file
dump numpy files of mfcc

configure [MFCC] section in "config/config.ini"
    SAMPLING_RATE
    FRAME_LENGTH
    HOP_LENGTH
"""

import glob
import librosa
import numpy as np
import os
import time

import vmlutil

def create_mfcc(wavFilename, samplingRate, frameLength, hopLength):
    wave, samplingRate = librosa.load(wavFilename, sr=samplingRate)
    splitTime_arr = librosa.effects.split(wave)   # split wave signal into non-silent interval

    concatenatedWave = []                         # non-silent wave signal
    for interval in splitTime_arr:                # interval: list of start([0]) and end([1])
        trimmedWave , index = librosa.effects.trim(wave[interval[0]:interval[1]])
        concatenatedWave = np.r_[concatenatedWave, trimmedWave]

    frameWave_arr = librosa.util.frame(concatenatedWave, frame_length=frameLength, hop_length=hopLength)

    mfcc_lst = []
    for i in range(frameWave_arr.shape[1]):
        mfcc = librosa.feature.mfcc(y=frameWave_arr[:, i], sr=samplingRate)
        mfcc_lst.append(mfcc[:])
    mfccFilename = wavFilename + ".mfcc"
    np.save(mfccFilename, mfcc_lst)

def main():
    configFilepath = "./config/config.ini"
    trainDirpath, testDirpath, train_lst, test_lst = vmlutil.read_config(configFilepath)
    samplingRate, frameLength, hopLength = vmlutil.read_config_mfcc(configFilepath)
    for wavFilename in glob.glob(trainDirpath + "*/*.wav"):
        print("START: calc mfcc of {}".format(wavFilename))
        create_mfcc(wavFilename, samplingRate=samplingRate, frameLength=frameLength, hopLength=hopLength)
        print("DONE: dump mfcc to {}.mfcc".format(wavFilename))
    for wavFilename in glob.glob(testDirpath + "*/*.wav"):
        print("START: calc mfcc of {}".format(wavFilename))
        create_mfcc(wavFilename, samplingRate=samplingRate, frameLength=frameLength, hopLength=hopLength)
        print("DONE: dump mfcc to {}.mfcc".format(wavFilename))

if __name__ == '__main__':
    start = time.time()
    main()
    elapse = time.time() - start
    print('\nelapse time: {}  sec'.format(elapse))


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from sklearn.metrics import mean_squared_error
from fastcore.all import *
import argparse

class NodeDetectionEvaluator:
    def __init__(self, ground_truth, participant, tolerance):
        self.ground_truth = ground_truth.copy()
        self.participant = participant.copy()
        self.tolerance = tolerance

    def evaluate(self, object_id):
        gt_object = self.ground_truth[(self.ground_truth['ObjectID'] == object_id) & \
                          (self.ground_truth['Direction'] != 'ES')].copy()
        p_object = self.participant[(self.participant['ObjectID'] == object_id) & \
                                    (self.participant['Direction'] != 'ES')].copy()
        p_object['matched'] = False
        p_object['classification'] = None
        p_object['distance'] = None
        gt_object['classification'] = None
        gt_object['distance'] = None
        tp = 0
        fp = 0
        fn = 0

        for gt_idx, gt_row in gt_object.iterrows():
            matching_participant_events = p_object[
                (p_object['TimeIndex'] >= gt_row['TimeIndex'] - self.tolerance) &
                (p_object['TimeIndex'] <= gt_row['TimeIndex'] + self.tolerance) &
                (p_object['Direction'] == gt_row['Direction'])
            ]

            if len(matching_participant_events) > 0:
                p_idx = matching_participant_events.index[0]
                p_row = matching_participant_events.iloc[0]
                distance = p_row['TimeIndex'] - gt_row['TimeIndex']
                if p_row['Node'] == gt_row['Node'] and p_row['Type'] == gt_row['Type']:
                    tp += 1
                    gt_object.loc[gt_idx, 'classification'] = 'TP'
                    gt_object.loc[gt_idx, 'distance'] = distance
                    p_object.loc[p_idx, 'classification'] = 'TP'
                    p_object.loc[p_idx, 'distance'] = distance
                else:
                    fp += 1
                    gt_object.loc[gt_idx, 'classification'] = 'FP'
                    gt_object.loc[gt_idx, 'distance'] = distance
                    p_object.loc[p_idx, 'classification'] = 'FP'
                    p_object.loc[p_idx, 'distance'] = distance
                p_object.loc[matching_participant_events.index[0], 'matched'] = True
            else:
                fn += 1
                gt_object.loc[gt_idx, 'classification'] = 'FN'

        additional_fp = p_object[~p_object['matched']].copy()
        fp += len(additional_fp)
        p_object.loc[additional_fp.index, 'classification'] = 'FP'

        return tp, fp, fn, gt_object, p_object

    def score(self):
        total_tp = 0
        total_fp = 0
        total_fn = 0
        total_distances = []

        for object_id in self.ground_truth['ObjectID'].unique():
            _, _, _, gt_object, p_object = self.evaluate(object_id)

            total_tp += len(p_object[p_object['classification'] == 'TP'])
            total_fp += len(p_object[p_object['classification'] == 'FP'])
            total_fn += len(gt_object[gt_object['classification'] == 'FN'])
            total_distances.extend(
                p_object[p_object['classification'] == 'TP']['distance'].tolist()
            )

        precision = total_tp / (total_tp + total_fp) \
            if (total_tp + total_fp) != 0 else 0
        recall = total_tp / (total_tp + total_fn) \
            if (total_tp + total_fn) != 0 else 0
        f2 = (5 * total_tp) / (5 * total_tp + 4 * total_fn + total_fp) \
            if (5 * total_tp + 4 * total_fn + total_fp) != 0 else 0
        rmse = np.sqrt((sum(d ** 2 for d in total_distances) / len(total_distances))) if total_distances else 0

        return precision, recall, f2, rmse

    def plot(self, object_id):
        tp, fp, fn, gt_object, p_object = self.evaluate(object_id)
        tp_distances = p_object[p_object['classification'] == 'TP']['distance']
        mse = np.sqrt(mean_squared_error([0] * len(tp_distances), tp_distances)) if tp_distances.size else 0
        fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 6), sharex=True)
        ground_truth_EW = gt_object[gt_object['Direction'] == 'EW']
        ground_truth_NS = gt_object[gt_object['Direction'] == 'NS']
        participant_EW = p_object[p_object['Direction'] == 'EW']
        participant_NS = p_object[p_object['Direction'] == 'NS']
        self._plot_type_timeline(ground_truth_EW, participant_EW, ax1, 'EW')
        self._plot_type_timeline(ground_truth_NS, participant_NS, ax2, 'NS')
        plt.xlabel('TimeIndex')
        title_info = f"Object {object_id}: TPs={tp}, FPs={fp}, FNs={fn}"
        fig.suptitle(title_info, fontsize=10)
        plt.tight_layout(rect=[0, 0.08, 1, 0.96])
        legend_elements = [
            plt.Line2D([0], [0], color='green', linestyle='dashed', label='True Positive (TP)'),
            plt.Line2D([0], [0], color='blue', linestyle='dashed', label='False Positive (FP)'),
            plt.Line2D([0], [0], color='red', linestyle='dashed', label='False Negative (FN)'),
            patches.Patch(color='grey', alpha=0.2, label='Tolerance Interval')
        ]
        plt.legend(handles=legend_elements, loc='lower center', bbox_to_anchor=(0.5, -0.5), ncol=4)
        plt.show()

    def _plot_type_timeline(self, ground_truth_type, participant_type, ax,
                            type_label):
        for _, row in ground_truth_type.iterrows():
            label = row['Node'] + '-' + row['Type']
            ax.scatter(row['TimeIndex'], 2, color='black')
            ax.text(row['TimeIndex'] + 3, 2.05, label, rotation=45)
            ax.fill_betweenx([1, 2], row['TimeIndex'] - self.tolerance,
                             row['TimeIndex'] + self.tolerance, color='grey',
                             alpha=0.2)
            if row['classification'] == 'TP':
                ax.text(row['TimeIndex'] + self.tolerance + .5, 1.5,
                        str(row['distance']),
                        color='black')
                ax.plot([row['TimeIndex'],
                         row['TimeIndex'] + row['distance']], [2, 1],
                         color='green', linestyle='dashed')
            elif row['classification'] == 'FP':
                ax.plot([row['TimeIndex'],
                         row['TimeIndex'] + row['distance']], [2, 1],
                         color='blue', linestyle='dashed')
            elif row['classification'] == 'FN':
                ax.plot([row['TimeIndex'],
                         row['TimeIndex']], [2, 2.2], color='red',
                         linestyle='dashed')

        for _, row in participant_type.iterrows():
            label = row['Node'] + '-' + row['Type']
            ax.scatter(row['TimeIndex'], 1, color='black')
            ax.text(row['TimeIndex'] + 3, 1.05, label, rotation=45)
            if row['classification'] == 'FP' and row['matched'] == False:
                ax.plot([row['TimeIndex'], row['TimeIndex']], [1, 0.8],
                        color='blue', linestyle='dashed')

        ax.spines['left'].set_color('none')
        ax.spines['right'].set_color('none')
        ax.spines['top'].set_color('none')
        ax.yaxis.grid(True)
        ax.set_yticks([1, 2])
        ax.set_yticklabels(['Participant', 'Ground truth'])
        ax.set_title(type_label)

def merge_label_files(label_folder):
    """
    Merges all label files in a given folder into a single pandas DataFrame.
    The filenames must be in the format <ObjectID>.csv, and the object id will
    be extracted from the filename and added as a column to the DataFrame.

    Args:
        label_folder (str): The path to the folder containing the label files.

    Returns:
        pandas.DataFrame: A DataFrame containing the merged label data.
    """
    label_data = []
    label_folder = Path(label_folder).expanduser()
    for file_path in label_folder.ls():
        df = pd.read_csv(file_path)
        oid_s = os.path.basename(file_path).split('.')[0]  # Extract ObjectID from filename
        df['ObjectID'] = int(oid_s)
        label_data.append(df)

    label_data = pd.concat(label_data)
    label_data = label_data[['ObjectID'] + list(label_data.columns[:-1])]
    return label_data

def run_evaluator(ground_truth_path=None, participant_path=None, plot_object=None):

    if participant_path is None:
        participant_df = pd.read_csv('participant_toy.csv')
    else:
        participant_path = Path(participant_path).expanduser()
        if participant_path.is_dir():
            participant_df = merge_label_files(participant_path)
        else:
            participant_df = pd.read_csv(participant_path)

    if ground_truth_path is None:
        ground_truth_df = pd.read_csv('ground_truth_toy.csv')
    else:
        ground_truth_path = Path(ground_truth_path).expanduser()
        if ground_truth_path.is_dir():
            ground_truth_df = merge_label_files(ground_truth_path)
        else:
            ground_truth_df = pd.read_csv(ground_truth_path)

    # Create a NodeDetectionEvaluator instance
    evaluator = NodeDetectionEvaluator(ground_truth_df, participant_df, tolerance=6)
    precision, recall, f2, rmse = evaluator.score()
    print(f'Precision: {precision:.2f}')
    print(f'Recall: {recall:.2f}')
    print(f'F2: {f2:.2f}')
    print(f'RMSE: {rmse:.2f}')

    # Plot the evaluation for the selected object (if any)
    if plot_object:
        evaluator.plot(object_id=plot_object)
    return precision, recall, f2, rmse

if __name__ == "__main__":
    if 'ipykernel' in sys.modules:
        run_evaluator(plot_object=True)
    else:
        # Parse the arguments
        parser = argparse.ArgumentParser()
        parser.add_argument('--participant', type=str, required=False,
                            help='Path to the participant file or folder. \
                                If a folder is provided, all the CSV files in the \
                                    folder will be used. If none, the toy example \
                                        will be used.')
        parser.add_argument('--ground_truth', type=str, required=False,
                            help='Path to the ground truth file. If none, the toy \
                                example will be used.')
        parser.add_argument('--plot_object', type=int, required=False,
                    help='Object ID to plot.')
        args = parser.parse_args()
        run_evaluator(args.ground_truth, args.participant, args.plot_object)


#!/usr/bin/env python3
import sys
import logging
from page_loader import download
from page_loader.cli import get_args
from page_loader import logger

def main():
    logger.set_logger()
    args = get_args()
    try:
        logging.warning(f'Downloading page from "{args.url}"...')
        page_local_path = download(args.url, args.output, args.log)
    except Exception as exc:
        logging.warning(exc)
        logging.warning('Page wasn\'t download!')
        sys.exit(42)
    logging.warning(f'Page was successfully downloaded to "{page_local_path}".')
    sys.exit(0)

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn import tree
from sklearn.tree import _tree

def feature_stat(data_, feature, target_name):
    data = data_.copy()
    print('Counts:')
    print(data.groupby(feature)[target_name].count())
    print('Frequencies:')
    print(data[feature].value_counts(normalize=True, dropna=False))
    x = [i for i in data.groupby(feature)[target_name].count().index]

    if data[feature].isnull().any():
        if str(data[feature].dtype) == 'category':
            data[feature].cat.add_categories(['None'], inplace=True)
        data[feature].fillna('None', inplace=True)

    y1 = [i for i in data.groupby(feature)[target_name].count().values]
    y2 = [i for i in data.groupby(feature)[target_name].mean().values]
    ind = np.arange(len(data[feature].unique()))
    fig, ax1 = plt.subplots(figsize=(8, 6))
    ax1.bar(ind, y1, align='center', width=0.4, alpha=0.7)
    ax1.set_xlabel(feature)
    ax1.set_ylabel('Counts', color='b')
    ax1.tick_params('y1', colors='b')
    ax2 = ax1.twinx()
    ax2.plot(ind, y2, 'r')
    ax2.set_ylabel('Mean rate', color='r')
    ax2.tick_params('y2', colors='r')
    plt.xticks(ind, x, rotation=45)
    ax1.set_xticklabels(x, rotation=35)
    plt.grid(False)
    plt.show()
    _, iv = calc_iv(data, target_name, feature)
    print('IV: ', iv)

def cont_split(data, feature, target, leafs, auto_calc=False):
    x = data[feature]
    y = data[target]

    clf = tree.DecisionTreeClassifier(criterion='entropy', min_samples_split=0.05, min_samples_leaf=0.05, max_features=1, min_weight_fraction_leaf=0.1,  max_leaf_nodes=leafs, class_weight='balanced')
    clf = clf.fit(x.values.reshape(-1, 1), y)

    h = tree_to_thresholds(clf, 'a')
    h = list(h)
    h.append(max(x))
    if feature == 'AGE':
        d = {k: np.round(v, 0) for k,v in zip(range(len(h)),h)}
    elif data[feature].max() > 1000:
        d = {k: np.round(v, -2) for k,v in zip(range(len(h)),h)}
    else:
        d = {k: v for k,v in zip(range(len(h)),h)}

    if len(set(d.values())) < len(list(d.values())):
        return None

    if auto_calc == True:
        temp_series = pd.DataFrame({feature: pd.cut(data[feature], bins=[v for v in d.values()]), target: data[target]})
        temp_series[feature].cat.add_categories(['No value'], inplace=True)
        return calc_iv(temp_series, target, feature)[1]

    #data.groupby(pd.cut(x, bins=[v for v in d.values()]))[target].mean().plot(kind='bar')
    temp_series = pd.DataFrame({feature: pd.cut(data[feature], bins=[v for v in d.values()]), target: data[target]})
    temp_series[feature].cat.add_categories(['No value'], inplace=True)

    print(pd.cut(x, bins=[v for v in d.values()]).value_counts(normalize=True, dropna=False))

    print('IV: ', calc_iv(temp_series, target, feature)[1])

    return pd.cut(data[feature], bins=[v for v in d.values()])

def tree_to_thresholds(tree, feature_names):
    '''
    Get thresholds from decision tree.
    '''

    tr = [0]
    tree_ = tree.tree_
    feature_name = [
        feature_names[i] if i != _tree.TREE_UNDEFINED else "undefined!"
        for i in tree_.feature
    ]

    def recurse(node, depth):
        indent = "  " * depth
        if tree_.feature[node] != _tree.TREE_UNDEFINED:
            name = feature_name[node]
            threshold = tree_.threshold[node]
            if threshold not in tr: tr.append(threshold)
            recurse(tree_.children_left[node], depth + 1)
            if threshold not in tr: tr.append(threshold)
            recurse(tree_.children_right[node], depth + 1)
        else:
            pass

    recurse(0, 1)
    return sorted(tr)

def calc_iv(data, target, feature):
    df = pd.DataFrame(index = data[feature].unique(),
                      data={'% responders': data.groupby(feature)[target].sum() / np.sum(data[target])})
    df['% non-responders'] = (data.groupby(feature)[target].count() - data.groupby(feature)[target].sum()) / (len(data[target]) - np.sum(data[target]))
    df['WOE'] = np.log(df['% responders'] / df['% non-responders'])
    df['DG-DB'] = df['% responders'] - df['% non-responders']
    df['IV'] = df['WOE'] * df['DG-DB']
    return df, np.sum(df['IV'])

def split_best_iv(data, feature, target_name):
    best_iv = 0
    for i in range(2, 20):
        iv_temp = cont_split(data, feature, target_name, i, auto_calc=True)
        if iv_temp == None:
            return cont_split(data, feature, target_name, i - 1)
        if iv_temp > best_iv:
            best_iv = iv_temp
        else:
            return cont_split(data, feature, target_name, i - 1)