import unittest
import responses
import digitalocean

from .BaseTest import BaseTest

class TestImage(BaseTest):

    def setUp(self):
        super(TestImage, self).setUp()
        self.image = digitalocean.Image(id=449676856, token=self.token)

    @responses.activate
    def test_load(self):
        data = self.load_from_file('images/single.json')

        responses.add(responses.GET,
                      '{}images/{}'.format(self.base_url, self.image.id),
                      body=data,
                      status=200,
                      content_type='application/json')

        self.image.load()

        self.assertEqual(responses.calls[0].request.url,
                         self.base_url + 'images/449676856')
        self.assertEqual(self.image.id, 449676856)
        self.assertEqual(self.image.name, 'My Snapshot')
        self.assertEqual(self.image.distribution, 'Ubuntu')
        self.assertEqual(self.image.public, False)
        self.assertEqual(self.image.created_at, "2014-08-18T16:35:40Z")
        self.assertEqual(self.image.size_gigabytes, 2.34)
        self.assertEqual(self.image.min_disk_size, 20)

    @responses.activate
    def test_destroy(self):
        responses.add(responses.DELETE,
                      '{}images/{}/'.format(self.base_url, self.image.id),
                      status=204,
                      content_type='application/json')

        self.image.destroy()

        self.assertEqual(responses.calls[0].request.url,
                         self.base_url + 'images/449676856/')

    @responses.activate
    def test_transfer(self):
        data = self.load_from_file('images/transfer.json')

        responses.add(responses.POST,
                      '{}images/{}/actions/'.format(
                        self.base_url, self.image.id),
                      body=data,
                      status=201,
                      content_type='application/json')

        res = self.image.transfer(new_region_slug='lon1')

        self.assertEqual(responses.calls[0].request.url,
                         self.base_url + 'images/449676856/actions/')
        self.assertEqual(res['action']['type'], 'transfer')
        self.assertEqual(res['action']['status'], 'in-progress')
        self.assertEqual(res['action']['id'], 68212728)

    @responses.activate
    def test_rename(self):
        data = self.load_from_file('images/rename.json')

        responses.add(responses.PUT,
                      '{}images/{}'.format(self.base_url, self.image.id),
                      body=data,
                      status=200,
                      content_type='application/json')

        res = self.image.rename(new_name='Descriptive name')

        self.assertEqual(responses.calls[0].request.url,
                         self.base_url + 'images/449676856')
        self.assertEqual(res['image']['name'], 'Descriptive name')

if __name__ == '__main__':
    unittest.main()


import re
from typing import Any
from pyspark.sql.dataframe import DataFrame
from pyathena import connect as athena_connect
from spark_pipeline_framework.logger.yarn_logger import get_logger
from spark_pipeline_framework.utilities.athena.athena_source_file_type import (
    AthenaSourceFileType,
)

class Athena:
    # S3_STAGING_PATH = "s3://bwell-helix-log-scratchpad-ue1/athena_query_results/"

    @staticmethod
    def drop_create_table(
        table_name: str,
        schema_name: str,
        df: DataFrame,
        s3_source_path: str,
        source_file_type: AthenaSourceFileType,
        s3_temp_folder: str,
    ) -> None:
        drop_ddl: str = Athena._generate_athena_drop_table_ddl(table_name)
        Athena._execute_ddl(schema_name, drop_ddl, s3_temp_folder=s3_temp_folder)
        create_ddl: str = Athena._generate_athena_create_table_ddl(
            df, table_name, s3_source_path, source_file_type
        )
        Athena._execute_ddl(schema_name, create_ddl, s3_temp_folder=s3_temp_folder)

    @staticmethod
    def _generate_athena_drop_table_ddl(athena_table_name: str) -> str:
        return f"DROP TABLE IF EXISTS `{athena_table_name}`"

    # noinspection PyPep8Naming,PyMissingOrEmptyDocstring
    @staticmethod
    def _generate_athena_create_table_ddl(
        df: DataFrame,
        athena_table_name: str,
        s3_location: str,
        source_file_type: AthenaSourceFileType,
    ) -> str:
        s3_location = s3_location.replace("s3a://", "s3://")
        ddl = []
        fields = df.schema

        ddl.append(f"CREATE EXTERNAL TABLE IF NOT EXISTS `{athena_table_name}` (")

        for field in fields.fieldNames()[:-1]:
            ddl.append(
                f"\t`{fields[field].name}` {Athena._get_ddl_data_type(fields[field])},"
            )
        last = fields[fields.fieldNames()[-1]]
        ddl.append(f"\t`{last.name}` {Athena._get_ddl_data_type(last)}")
        ddl.append(")")

        if source_file_type == AthenaSourceFileType.CSV_QUOTED:
            ddl.append("ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'")
            ddl.append("WITH SERDEPROPERTIES (")
            ddl.append("  'separatorChar' = ',',")
            ddl.append("  'quoteChar' = '\"',")
            ddl.append("  'escapeChar' = '\\\\'")
            ddl.append("  )")
        elif source_file_type == AthenaSourceFileType.CSV_UNQUOTED:
            ddl.append("ROW FORMAT DELIMITED")
            ddl.append("  FIELDS TERMINATED BY ','")
            ddl.append("  ESCAPED BY '\\\\'")
            ddl.append("  LINES TERMINATED BY '\\n'")
        elif source_file_type == AthenaSourceFileType.JSONL:
            ddl.append("ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'")
        elif source_file_type == AthenaSourceFileType.PARQUET:
            ddl.append("STORED AS PARQUET")

        ddl.append(f"LOCATION '{s3_location}'")

        if source_file_type == AthenaSourceFileType.PARQUET:
            ddl.append('tblproperties ("parquet.compression"="SNAPPY")')

        return "\n".join(ddl)

    @staticmethod
    def _get_ddl_data_type(field: Any) -> str:
        # wrap all field names in backticks to avoid reserved word collisions
        return str(
            re.sub("(\\w*?)?:(\\w*?)", "`\\1`:\\2", field.dataType.simpleString())
        )

    @staticmethod
    def _execute_ddl(schema_name: str, ddl: str, s3_temp_folder: str) -> None:
        try:
            cursor = athena_connect(
                schema_name=schema_name, s3_staging_dir=s3_temp_folder
            ).cursor()
            cursor.execute(ddl)
        except Exception as e:
            logger = get_logger(__name__)
            logger.exception(
                f"ERROR executing Athena query on schema: {schema_name} using temp_folder:{s3_temp_folder}"
            )
            logger.error("---------- Query ------------")
            logger.error(ddl)
            logger.error("------------------------------")
            logger.error(e)


import pygame
from sys import exit
from config import *
from debug import debug
from level import Level
from player import Player

class Game:
    """Create and run the entire game."""

    def __init__(self):
        # General setup
        pygame.init()
        self.screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
        main_icon = pygame.image.load('./Graphics/Images/player.png')
        pygame.display.set_caption('Schmoo')
        pygame.display.set_icon(main_icon)
        self.clock = pygame.time.Clock()

        # Creating player
        self.player = Player()

        # Creating the level
        self.running = True
        self.tavern_0 = Level("./Graphics/Levels/Tavern_0", self.player)

    def main(self):
        """Main game loop. Displays the gameplay."""
        while self.running:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    self.running = False

            # Updating the screen
            self.screen.fill(BLACK)
            self.tavern_0.run()
            pygame.display.update()
            self.clock.tick(FPS)

            # if BATTLE:
                # im thkinking this is where we check if there is a fight happening and update the stuff.

0
if __name__ == '__main__':
    # Creating the game
    game = Game()
    game.main()

    # Closing the code
    pygame.quit()
    exit()


from os import name
from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name="home"),
    path('login/', views.login_user, name="login"),
    path('register/', views.register_user, name="register"),
    path('logout/', views.logout_user, name="logout"),
    path('group/<str:pk>/', views.group, name="group"),
    path('create-group/', views.create_group, name="create-group"),
    path('update-group/<str:pk>/', views.update_group, name="update-group"),
    path('delete-group/<str:pk>/', views.delete_group, name="delete-group"),
    path('delete-message/<str:pk>/', views.delete_message, name="delete-message"),
    path('profile/<str:pk>/', views.user_profile, name="profile"),
    path('update-profile/', views.update_user_profile, name="update-profile"),
    path('all-subjects/', views.all_subjects, name="all-subjects"),
    path('all-activity/', views.all_activity, name="all-activity")
]


#!/usr/bin/env python

'''
Python implementation of SATZ, a sentence
boundary disambiguator using a feed
forward neural network.

Created by David Palmer (Edited by us)
https://arxiv.org/pdf/cmp-lg/9503019.pdf
'''

from __future__ import absolute_import, division
from __future__ import print_function

import os
import sys
import pdb
import numpy as np
import nltk
import cPickle

from constants import *
import create_toy_data as ctd
from metrics import get_auc
from collections import defaultdict

local_ref = lambda x: os.path.join(os.path.dirname(__file__),  x)

sys.path.append(local_ref('../common'))
from util import split_data

sys.path.append(local_ref('../models'))
import nn

def get_loc_in_array(value, array):
    find = np.where(array == value)[0]
    # print("value: {}, array: {}, find: {}".format(value, array, find))
    if find.shape[0] > 0:
        return find[0]
    raise RuntimeError("Couldn't find value: {} in array".format(value))

def init_prior_pos_proba(
        lexicon=None, simplify_tags=True, save_to_disk=False):
    ''' Set prior proba probabilities using
        the part-of-speech frequencies for
        every word in a given lexicon. Default
        lexicon used is the BROWN corpus.

        Args
        ----
        lexicon : list of tuples
                  each tuple is (word, part-of-speech)

        simplify_tags : boolean
                        many of the tags have additional info
                        like IN-HL or HVZ*. Setting simplify_tags
                        to True, ignores these additional tags.

        save_to_disk : boolean
                       write frequencies to a pickle file

        Returns
        -------
        freqs : dict
                dictionary of word mapping to
                its frequency
    '''
    if lexicon is None:
        lexicon = ctd.load_data(
            ctd.brown_generator(), return_tags=True)

    descriptor_array = np.array([
        NOUN,
        VERB,
        ARTICLE,
        MODIFIER,
        CONJUNCTION,
        PRONOUN,
        PREPOSITION,
        PROPER_NOUN,
        NUMBER,
        COMMA_SEMICOLON,
        LEFT_PAREN,
        RIGHT_PAREN,
        NON_PUNC_CHAR,
        POSSESSIVE,
        COLON_DASH,
        ABBREV,
        EOS_PUNC,
        OTHERS,
        IS_UPPER,
        FOLLOWS_EOS_PUNC
    ])

    cat_lookup = group_categories()
    num_tags = descriptor_array.shape[0]
    tag_counts = defaultdict(lambda: np.zeros(num_tags))

    # loop through words and fill out freq
    for word, tag_bunch in lexicon:
        if tag_bunch == '--':
            tag_idx = get_loc_in_array(COLON_DASH, descriptor_array)
            tag_counts[word][tag_idx] += 1
            continue

        if '*' in tag_bunch:
            tag_idx = get_loc_in_array(CONJUNCTION, descriptor_array)
            tag_counts[word][tag_idx] += 1
            tag_bunch = tag_bunch.replace('*', '')

        for tt in tag_bunch.split('-'):
            for single_tag in tt.split('+'):
                # print("tag_bunch: {} || single_tag: {}".format(tag_bunch, single_tag))

                reduced_tags = cat_lookup(single_tag)

                if reduced_tags is None and single_tag in ['``', "''", '', "'", 'HL', 'FW', 'NC', 'TL']:
                    # tags known to be weird, but also okay
                    continue
                elif reduced_tags is None and \
                    single_tag in ['RB$', 'DT$', 'NR$', 'AP$', 'JJ$', 'CD$', 'N', 'T', 'PP', 'NIL']:
                    # odd tags, probably should really figure them out
                    # datasets/brown/raw/cj31 -> PPSS+BER-N
                    # datasets/brown/raw/cj37 -> FW-IN+AT-T
                    # datasets/brown/raw/ce01 -> WDT+BER+PP, PP is probably personal pronoun
                    continue
                elif reduced_tags is None:
                    # not a "known exception", print
                    print("word: {}, tag_bunch not found: {}".format(word, tag_bunch))
                    continue

                for reduced_tag in reduced_tags:
                    tag_idx = get_loc_in_array(reduced_tag, descriptor_array)
                    tag_counts[word][tag_idx] += 1

    if save_to_disk:
        with open(local_ref('../storage/brown_tag_distribution.pkl'), 'wb') as f:
            cPickle.dump(dict(tag_counts), f)
        with open(local_ref('../storage/brown_tag_order.pkl'), 'wb') as f:
            cPickle.dump(descriptor_array, f)

    return tag_counts

def get_descriptor_arrays(
        tokens, tag_counts=None, tag_order=None):
    ''' The context around a word can be approximated
        by a single part-of-speech (POS) per word. We
        approximate this part-of-speech using the
        prior probabilities for all parts-of-speech
        per word.

        If unknown, follow a prescribed list of
        assumptions.

        Args
        ----
        tokens : list
                 tokens abstracted from text

        tag_counts : dict of word-to-subdict mapping
                     subdict is P-O-S to counts mapping

        tag_order : list of strings
                    order of P-O-S in frequency array

        Returns
        -------
        probas : 2D array
                 probabilities per token per P-O-S
    '''

    if tag_counts is None:
        tag_counts = cPickle.load(
            open(local_ref('../storage/brown_tag_distribution.pkl'), 'rb'))

    if tag_order is None:
        tag_order = cPickle.load(
            open(local_ref('../storage/brown_tag_order.pkl'), 'rb'))

    num_tags = tag_order.shape[0]

    # heuristics
    lemmatizer = nltk.stem.WordNetLemmatizer()
    has_number = lambda x: any(
        char.isdigit() for char in x)
    has_eos_punc = lambda x: x[0] in ".!?".split()
    reduce_morpho = lambda x: lemmatizer.lemmatize(x)
    is_abbrev = lambda x: '.' in x
    has_hyphen = lambda x: '-' in x

    def is_plural(x):
        lemma = lemmatizer.lemmatize(x, 'n')
        plural = True if word is not lemma else False # aka plural != lemma
        return plural

    def is_upper(x):
        if len(x) >= 0:
            return x[0].isupper()
        return False

    desc_arrays = []
    prev_token = None
    for token_cnt, token in enumerate(tokens):
        # if token_cnt % 10000 == 0:
        #     print('----------------------')
        #     print('processing: token ({}/{})'.format(token_cnt, len(tokens)))
        token_in_lexicon = False
        if token in tag_counts:
            token_in_lexicon = True
            cur_tag_count = tag_counts[token]
        elif reduce_morpho(token) in tag_counts:
            token_in_lexicon = True
            cur_tag_count = tag_counts[reduce_morpho(token)]
        else:
            ''' If a word is not in the lexicon, use a
                list of heuristics to try to approx the
                tag probabilities.

                - if contains a 0-9, assume number
                - if begins with a ".!?", should be
                  end-of-sentence punctuation
                - handle morphological endings
                - internal period is an abbreviation
                - if capital word
                    - 0.9 pr of being a proper noun
                - uniform distribution over tags
            '''
            cur_tag_count = np.zeros(num_tags)
            if has_number(token):
                cur_tag_count[get_loc_in_array(NUMBER, tag_order)] += 1
            elif has_eos_punc(token):
                cur_tag_count[get_loc_in_array(EOS_PUNC, tag_order)] += 1
            elif is_abbrev(token):
                cur_tag_count[get_loc_in_array(ABBREV, tag_order)] += 1
            elif has_hyphen(token):
                cur_tag_count[get_loc_in_array(OTHERS, tag_order)] += 1
                # TODO: don't know how to lookup this...
            else:
                cur_tag_count = np.ones(num_tags)

        # zero out IS_UPPER and FOLLOWS_EOS_PUNC before normalize
        cur_tag_count[get_loc_in_array(IS_UPPER, tag_order)] = 0
        cur_tag_count[get_loc_in_array(FOLLOWS_EOS_PUNC, tag_order)] = 0

        # divide counts to get probabilities
        cur_tag_distrib = cur_tag_count / np.sum(cur_tag_count)

        ''' capitalized words in lexicon but not
            registered as proper nouns can still be
            proper nouns with 0.5 probability. If not
            in lexicon, use 0.9 probability.
        '''
        if is_upper(token):
            proper_pr = 0.5 if token_in_lexicon else 0.9
            cur_tag_distrib *= (1 - proper_pr)
            cur_tag_distrib[get_loc_in_array(
                PROPER_NOUN, tag_order)] = proper_pr

            cur_tag_distrib[get_loc_in_array(IS_UPPER, tag_order)] = 1

        if prev_token and has_eos_punc(prev_token):
            cur_tag_distrib[get_loc_in_array(FOLLOWS_EOS_PUNC, tag_order)] = 1

        prev_token = token
        desc_arrays.append(cur_tag_distrib)

    return np.array(desc_arrays)

def get_dummies(outputs):
    # only works for 1-D outputs
    uniq_outputs = np.unique(outputs)
    num_outputs = outputs.shape[0]

    dummies = np.zeros((num_outputs, uniq_outputs.shape[0]))
    for i, row in enumerate(outputs):
        col_idx = np.where(uniq_outputs == row)[0][0]
        dummies[i, col_idx] = 1

    return dummies

def group_categories():
    '''
    Mapping into these words:

    noun, verb, article, modifier,
    conjunction, pronoun, preposition,
    proper noun, number, comma or semicolon,
    left parentheses, right parentheses,
    non-punctuation character, possessive,
    colon or dash, abbreviation,
    sentence-ending punctuation, others
    '''
    mapper = dict()
    mapper["."] = [EOS_PUNC]                      # ending punctuation
    mapper["("] = [LEFT_PAREN]                    # left parentheses
    mapper[")"] = [RIGHT_PAREN]                   # right parentheses
    mapper["*"] = [CONJUNCTION]                   # conjunction
    mapper["--"] = [COLON_DASH]                   # dash
    mapper[","] = [COMMA_SEMICOLON]               # comma
    mapper[":"] = [COLON_DASH]                    # colon
    mapper["ABL"] = [MODIFIER]                    # pre-qualifier [quite, rather]
    mapper["ABN"] = [MODIFIER]                    # pre-quantifier [half, all]
    mapper["ABR"] = [ABBREV]                      # abbreviation CUSTOM TAG
    mapper["ABX"] = [MODIFIER]                    # pre-quantifier [both]
    mapper["AP"] = [OTHERS]                       # post-determiner [many, several, next]
    mapper["AT"] = [ARTICLE]                      # article
    mapper["BE"] = [VERB]                         # be
    mapper["BED"] = [VERB]                        # were
    mapper["BEDZ"] = [VERB]                       # was
    mapper["BEG"] = [VERB]                        # being
    mapper["BEM"] = [VERB]                        # am
    mapper["BEN"] = [VERB]                        # been
    mapper["BER"] = [VERB]                        # are, art
    mapper["BEZ"] = [VERB]                        # is
    mapper["CC"] = [CONJUNCTION]                  # coordinating conjunction
    mapper["CD"] = [NUMBER]                       # cardinal numeral
    mapper["CS"] = [CONJUNCTION]                  # subordinating conjunction
    mapper["DO"] = [VERB]                         # do
    mapper["DOD"] = [VERB]                        # did
    mapper["DOZ"] = [VERB]                        # does
    mapper["DT"] = [OTHERS]                       # plural determiner [these, those]
    mapper["DTI"] = [OTHERS]                      # singular/plural determiner/quantifier [some, any]
    mapper["DTS"] = [OTHERS]                      # plural determiner
    mapper["DTX"] = [CONJUNCTION]                 # determiner/double conjunction [either]
    mapper["EX"] = [OTHERS]                       # existentil there
    mapper["FW"] = [OTHERS]                       # foreign word (hyphenated before regular tag)
    mapper["HL"] = [OTHERS]                       # word occurring in headline (hyphenated after regular tag)
    mapper["HV"] = [VERB]                         # have
    mapper["HVD"] = [VERB]                        # had (past tense)
    mapper["HVG"] = [VERB]                        # having
    mapper["HVN"] = [VERB]                        # had (past participle)
    mapper["HVZ"] = [VERB]                        # has
    mapper["IN"] = [PREPOSITION]                  # preposition
    mapper["JJ"] = [MODIFIER]                     # adjective
    mapper["JJR"] = [MODIFIER]                    # comparative adjective
    mapper["JJS"] = [MODIFIER]                    # semantically superlative adjective [chief, top]
    mapper["JJT"] = [MODIFIER]                    # morphologically superlative adjective [ biggest ]
    mapper["MD"] = [OTHERS]                       # modal auxiliary [can, should, will]
    mapper["NC"] = [OTHERS]                       # cited word (hypenated after regular tag)
    mapper["NN"] = [NOUN]                         # singular or mass noun
    mapper["NN$"] = [NOUN, POSSESSIVE]            # possessive singular noun
    mapper["NNS"] = [NOUN]                        # plural noun
    mapper["NNS$"] = [NOUN, POSSESSIVE]           # possessive plural noun
    mapper["NP"] = [PROPER_NOUN]                  # proper noun or part of name phrase
    mapper["NP$"] = [PROPER_NOUN, POSSESSIVE]     # possessive proper noun
    mapper["NPS"] = [PROPER_NOUN]                 # proper plural noun
    mapper["NPS$"] = [PROPER_NOUN, POSSESSIVE]    # possessive plural proper noun
    mapper["NR"] = [NOUN, MODIFIER]               # adverbial noun [home, today, west]
    mapper["NRS"] = [NOUN, MODIFIER]              # plural adverbial noun
    mapper["OD"] = [NUMBER]                       # ordinal numeral [first, 2nd]
    mapper["PN"] = [PRONOUN]                      # nominal pronoun [everybody, nothing]
    mapper["PN$"] = [PRONOUN, POSSESSIVE]         # possessive nominal pronoun
    mapper["PP$"] = [PRONOUN, POSSESSIVE]         # possessive personal pronoun [my, our]
    mapper["PP$$"] = [PRONOUN, POSSESSIVE]        # second (nominal) possessive pronoun [mine, ours]
    mapper["PPL"] = [PRONOUN]                     # singular reflexive/intensive personal pronoun [myself]
    mapper["PPLS"] = [PRONOUN]                    # plural reflexive/intensive personal pronoun [ourselves]
    mapper["PPO"] = [PRONOUN]                     # objective personal pronoun [me, him, it, them]
    mapper["PPS"] = [PRONOUN]                     # 3rd. singular nominative pronoun  [he, she, it, one]
    mapper["PPSS"] = [PRONOUN]                    # other nominative personal pronoun [I, we, they, you]
    mapper["QL"] = [MODIFIER]                     # qualifier [very, fairly]
    mapper["QLP"] = [MODIFIER]                    # post-qualifier [enough, indeed]
    mapper["RB"] = [MODIFIER]                     # adverb
    mapper["RBR"] = [MODIFIER]                    # comparative adverb
    mapper["RBT"] = [MODIFIER]                    # superlative adverb
    mapper["RN"] = [MODIFIER]                     # nominal adverb [here then, inddors]
    mapper["RP"] = [MODIFIER]                     # adverb/participle [about, off, up]
    mapper["TL"] = [OTHERS]                       # word occuring in title (hyphenated after regular tag)
    mapper["TO"] = [OTHERS]                       # infinitive marker to
    mapper["UH"] = [OTHERS]                       # interjection, exclamation
    mapper["UNK"] = [OTHERS]                      # unknown CUSTOM TAG
    mapper["VB"] = [VERB]                         # verb, base form
    mapper["VBD"] = [VERB]                        # verb, past tense
    mapper["VBG"] = [VERB]                        # verb, present participle/gerund
    mapper["VBN"] = [VERB]                        # verb, past participle
    mapper["VBZ"] = [VERB]                        # verb, 3rd. singular present
    mapper["WDT"] = [OTHERS]                      # wh- determiner [what, which]
    mapper["WP$"] = [PRONOUN, POSSESSIVE]         # possessive wh- pronoun [whose]
    mapper["WPO"] = [PRONOUN]                     # objective wh- pronoun [whom, which, that]
    mapper["WPS"] = [PRONOUN]                     # nominative wh- pronoun [who, which, that]
    mapper["WQL"] = [MODIFIER]                    # wh- qualifier [how]
    mapper["WRB"] = [MODIFIER]                    # wh- adverb [how, where, when]

    f = lambda x: mapper[x] if x in mapper else None
    return f

def safe_index(A, s, e, pad=False):
    # safe indexing
    num_col = A.shape[1]
    pre = np.zeros((0, num_col))
    post = np.zeros((0, num_col))

    if s < 0:
        pre = np.zeros((-s, num_col))
        s = 0

    if e > A.shape[0]:
        post = np.zeros((e - A.shape[0], num_col))
        e = A.shape[0]

    if pad:
        return np.concatenate((pre, A[s:e], post))
    return A[s:e]

def make_grams(darrays, num_grams, labels=None, target_tag=EOS_PUNC, tag_order=None):
    ''' Given darrays, only take the ones that have a certain
        label. Then take k/2 neighbors from either side.

        Args
        ----
        darrays : np array
                 descriptor array

        labels : np array
                 output array

        num_grams : integer
                    num of surrounding grams
                    if num_grams = 3, 3 on each side
                    will be taken.

        target_tag : string
                     what tag to index to look for

        tag_order : array
                    list of tags

        Returns
        -------
        new_darrays : np array
                     gram'd descriptor array

        new_labels : np array
                     gram'd labels
    '''

    if tag_order is None:
        tag_order = cPickle.load(
            open(local_ref('../storage/sentence_disambiguation/brown_tag_order.pkl'), 'rb'))

    if not target_tag in tag_order:
        raise ValueError('Target tag not found in tag order.')

    target_idx = np.where(tag_order == target_tag)[0][0]
    eos_idx = np.where(darrays[:,target_idx] > 0)[0]

    new_darrays = np.zeros((eos_idx.shape[0],darrays.shape[1]*(2*num_grams + 1)))
    if not labels is None:
        new_labels = np.zeros((eos_idx.shape[0], labels.shape[1]))

    for i, idx in enumerate(eos_idx):
        # if i % 5000 == 0:
        #     print('---------------------')
        #     print('finding: end-of-sentence ({}/{})'.format(i, eos_idx.shape[0]))

        sliced_darrays = safe_index(
            darrays, idx-num_grams, idx+num_grams+1, pad=True).flatten()
        new_darrays[i, :] = sliced_darrays
        if not labels is None:
            new_labels[i, :] = labels[idx, :]

    if not labels is None:
        return (eos_idx, new_darrays, new_labels)
    return eos_idx, new_darrays

def create_features_labels(save_to_disk=False):
    ''' Use pipeline to generate the (input, output)
        pairs for machine learning

        Args
        ----
        save_to_disk : boolean
                       write frequencies to a pickle file

    '''

    lexicon = ctd.load_data(
        ctd.brown_generator(), return_sent_labels=True)

    # not efficient --> change me
    data = np.array([[t, l] for t, l in lexicon])
    tokens = data[:, 0]
    labels = data[:, 1]

    # get features
    darrays = get_descriptor_arrays(tokens)
    labels = get_dummies(labels)

    # put into grams (give context)
    didx, darrays, labels = make_grams(
        darrays, 3, labels=labels, target_tag=EOS_PUNC)

    (tr_inputs, te_inputs), (tr_outputs, te_outputs) = split_data(
        darrays, out_data=labels, frac=0.80)

    if save_to_disk:
        np.save(local_ref('../storage/sentence_disambiguation/X_train.npy', tr_inputs))
        np.save(local_ref('../storage/sentence_disambiguation/X_test.npy', te_inputs))
        np.save(local_ref('../storage/sentence_disambiguation/y_train.npy', tr_outputs))
        np.save(local_ref('../storage/sentence_disambiguation/y_test.npy', te_outputs))

    return (tr_inputs, te_inputs), (tr_outputs, te_outputs)

def predict_from_tokens(tokens, trained_weights, tag_counts, tag_order):
    darrays = get_descriptor_arrays(tokens, tag_counts=tag_counts,
                                            tag_order=tag_order)
    didx, darrays = make_grams(darrays, 3, target_tag=EOS_PUNC,
                                           tag_order=tag_order)
    y_pred = nn.neural_net_predict(trained_weights, darrays)
    proba = np.exp(y_pred)
    return didx, proba

def split_into_sentences(tokens, didx, proba):
    labels = np.argmax(proba, axis=1)
    split_indexes = didx[np.where(labels==1)[0]]
    split_arr = np.split(tokens, split_indexes+1)
    return [list(i) for i in split_arr if len(i) > 0]

def main():
    ''' Paper found parameters to be efficient:
        6 word contexts --> 3 num_grams
        2 hidden_units
    '''

    X_train = np.load(local_ref('../storage/sentence_disambiguation/X_train.npy'))
    X_test = np.load(local_ref('../storage/sentence_disambiguation/X_test.npy'))
    y_train = np.load(local_ref('../storage/sentence_disambiguation/y_train.npy'))
    y_test = np.load(local_ref('../storage/sentence_disambiguation/y_test.npy'))

    trained_weights = nn.train_nn(
        X_train, y_train, [50, 10],
        batch_size=256, param_scale=0.1,
        num_epochs=20, step_size=0.001, L2_reg=1.0)

    # save the weights
    np.save(local_ref('../storage/trained_weights.npy'), trained_weights)

    y_pred = nn.neural_net_predict(trained_weights, X_test)
    # don't forget to exp
    print("auc: {}".format(
        get_auc(y_test[:, 1], np.exp(y_pred))))


from openai import OpenAI
import openai
from config import OPENAI_KEY
import os
openai.api_key = os.getenv("OPENAI_API_KEY")
# openai.api_key = OPENAI_KEY

client = OpenAI()

def img2txt(url, data_name="weibo"):
  print("Generating Image Captioning...")
  if "http:" in url:
      response = client.chat.completions.create(
        model="gpt-4-vision-preview",
        messages=[
          {
            "role": "user",
            "content": [
              {"type": "text", "text": "Let's think step by step. First, extract all the text in the image (OCR). Then, describe this picture. If it is a photo, how many characters are in this pic? what are they doing? what are their relations. what is the background? what activity is this? If it is a chart, what kind of chart it is? what interesting data does it have? what is the title or purpose of this chart? use 200 words to answer"},
              {
                "type": "image_url",
                "image_url": {
                  "url": f"{url}",
                },
              },
            ],
          }
        ],
        max_tokens=300,
      )
      return response.choices[0].message.content
  else:
      import base64
      import requests
      # OpenAI API Key
      # api_key = OPENAI_KEY
      api_key = os.getenv("OPENAI_API_KEY")

      # Function to encode the image
      def encode_image(image_path):
        if data_name == "weibo":
          rumor_img_dir = "../nlp-project/Data/weibo/nonrumor_images/"
          rumor_img_dir = os.path.abspath(rumor_img_dir)

          non_rumor_img_dir = "../nlp-project/Data/weibo/rumor_images/"
          non_rumor_img_dir = os.path.abspath(non_rumor_img_dir)

          for filename in os.listdir(rumor_img_dir):
            if filename == image_path:
              image_path = rumor_img_dir + "/" + image_path
              break

          for filename in os.listdir(non_rumor_img_dir):
            if filename == image_path:
              image_path = non_rumor_img_dir + "/" + image_path
              break

          with open(image_path, "rb") as image_file:
            return base64.b64encode(image_file.read()).decode('utf-8')

        elif data_name == "twitter":
          img_dir = "../nlp-project/Data/twitter/Mediaeval2016_TestSet_Images"

          for filename in os.listdir(img_dir):
            if filename[:-4] == image_path:
              image_path = img_dir + "/" + image_path + ".jpg"
              break

          with open(image_path, "rb") as image_file:
            return base64.b64encode(image_file.read()).decode('utf-8')

      # Path to your image
      image_path = url

      # Getting the base64 string
      base64_image = encode_image(image_path)

      headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {api_key}"
      }

      payload = {
        "model": "gpt-4-vision-preview",
        "messages": [
          {
            "role": "user",
            "content": [
              {
                "type": "text",
                "text": "Let's think step by step. First, extract all the text in the image (OCR). Then, describe this picture. If it is a photo, how many characters are in this pic? what are they doing? what are their relations. what is the background? what activity is this? If it is a chart, what kind of chart it is? what interesting data does it have? what is the title or purpose of this chart? use 200 words to answer"
              },
              {
                "type": "image_url",
                "image_url": {
                  "url": f"data:image/jpeg;base64,{base64_image}"
                }
              }
            ]
          }
        ],
        "max_tokens": 300
      }

      response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)

      return eval(response.text)["choices"][0]["message"]["content"]


import argparse
import pandas as pd
import functools
import waterbirds.data_builder as wb
import tensorflow as tf
import numpy as np
from causallearn.utils.KCI.KCI import KCI_CInd

# NOTE: the original causallearn code was modified to take epsilon as an argument.
# in the original code, its value is hardcoded.

def get_waterbirds_high_dim_x(random_seed, base_dir, pixel=128,
	weighted=False, batch_size=1000):
	"""Function to get the data."""
	experiment_directory = (
			f"{base_dir}/experiment_data/rs{random_seed}")
	train_data = pd.read_csv(
			f'{experiment_directory}/train.txt'
			).values.tolist()
	train_data = [
			tuple(train_data[i][0].split(',')) for i in range(len(train_data))
	]

	map_to_image_label_given_pixel = functools.partial(wb.map_to_image_label,
					pixel=pixel, weighted=weighted)

	train_dataset = tf.data.Dataset.from_tensor_slices(train_data)
	train_dataset = train_dataset.map(map_to_image_label_given_pixel, num_parallel_calls=1)
	train_dataset = train_dataset.batch(batch_size,
			drop_remainder=False).repeat(1)

	tr_data = []
	for batch_id, examples in enumerate(train_dataset):
			print(batch_id)
			x, labels_weights = examples
			x = x.numpy()
			x = x.reshape(x.shape[0], pixel*pixel*3)
			pix_cols = [f'x{i}' for i in range(x.shape[1])]
			labels = labels_weights['labels'].numpy()
			label_cols = [f'y{i}' for i in range(labels.shape[1])]
			x = pd.DataFrame(np.hstack([x, labels]),
											columns = pix_cols + label_cols)
			tr_data.append(x)

	tr_data = pd.concat(tr_data)
	return tr_data

def get_high_dim_x(experiment_name, random_seed, base_dir,
	pixel=128, weighted=False, batch_size=1000):
	if experiment_name == "waterbirds":
		return	get_waterbirds_high_dim_x(
			random_seed=random_seed,
			base_dir=base_dir,
			pixel=pixel,
			weighted=weighted,
			batch_size=batch_size)
	else:
		raise NotImplementedError("not yet!")

def get_y_waterbirds(random_seed, base_dir):
	experiment_directory = (
		f"{base_dir}/experiment_data/rs{random_seed}")

	tr_data = pd.read_csv(f'{experiment_directory}/train.txt')
	tr_data
	tr_data = tr_data['0'].str.split(",", expand=True)
	D = tr_data.shape[1]-4
	tr_data.columns = ['bird_img', 'bird_seg', 'back_img', 'noise_img'] + \
			[f'y{i}' for i in range(D)]
	tr_data.drop(['bird_img', 'bird_seg', 'back_img', 'noise_img'], axis=1, inplace=True)
	for i in range(D):
			tr_data[f'y{i}'] = tr_data[f'y{i}'].astype(np.float32)

	return tr_data

def get_y(experiment_name, random_seed, base_dir):
	if experiment_name == 'waterbirds':
		return get_y_waterbirds(random_seed=random_seed,
			base_dir=base_dir)
	else:
		raise NotImplementedError("not yet!")

def y_independence_test(random_seed, v_dim, base_dir, experiment_name):
	df = get_y(experiment_name=experiment_name,
		random_seed=random_seed,
		base_dir=base_dir
		)
	D = v_dim + 1

	var_variance = df.var(axis=0)
	var_variance = (var_variance == 0.0).tolist()
	if sum(var_variance) > 0.0:
		drop_vars = [i for i, i_true in enumerate(var_variance) if i_true]
		print(f'the following variables had 0 variance {df.columns[drop_vars]}')
		non_zero_var_labels = list(set(range(D)) - set(drop_vars))
	else:
		non_zero_var_labels = [i for i in range(D)]

	p_values = []
	for i in range(1, D):
		if i in non_zero_var_labels:
			kci_obj = KCI_CInd(kernelX='Gaussian',
								 kernelY='Gaussian',
								 kernelZ='Gaussian',
								 est_width='median',
								 epsilon=1e-3)

			p, _ = kci_obj.compute_pvalue(
				data_x=df.values[:, 0][:, np.newaxis],
				data_y=df.values[:, i][:, np.newaxis],
				data_z = df.values[:,[j for j in non_zero_var_labels if ((j!=0) & (j!=i))]]
				)
		else:
			p = 0.0
		p_values.append(p)
	return p_values

def x_independence_test(random_seed, pixel, batch_size,
	 model_name, xv_mode, v_dim, base_dir, x_mode, experiment_name):
	if x_mode == 'sufficient_stat':
		df = pd.read_csv((f'{base_dir}/final_models/opt_pred_rs{random_seed}_{model_name}_{xv_mode}'
			f'_pix{pixel}_bs{batch_size}_vdim{v_dim}.csv'))
		df.columns = df.columns.str.replace("pred", "x")
		df.drop(['model', 'dist'], inplace=True, axis=1)
	else:
		df = get_high_dim_x(
			experiment_name=experiment_name,
			random_seed=random_seed,
			base_dir=base_dir,
			pixel=128,
			weighted=False,
			batch_size=1000)

	D = v_dim + 1
	y_cols = [i for i in range(df.shape[1] - D, df.shape[1])]
	x_cols = list(set(range(df.shape[1])) - set(y_cols))

	# if any of the variables have 0 variance, the test fails
	# need to remove them first.
	var_variance = df.var(axis=0)

	var_variance = (var_variance == 0.0).tolist()
	if sum(var_variance) > 0.0:
		drop_vars = [i for i, i_true in enumerate(var_variance) if i_true]
		print(f'the following variables had 0 variance {df.columns[drop_vars]}')
		drop_y_vars = list(set(y_cols) & set(drop_vars))
		y_cols_reduced = list(set(y_cols) - set(drop_vars))
		x_cols = list(set(x_cols) - set(drop_vars))
	else:
		y_cols_reduced = y_cols

	p_values = []
	for i in y_cols[1:]:
		if i in y_cols_reduced:
			kci_obj = KCI_CInd(kernelX='Gaussian',
												 kernelY='Gaussian',
												 kernelZ='Gaussian',
												 est_width='median',
												 epsilon=1e-3)

			p, _ = kci_obj.compute_pvalue(
				data_x=df.values[:, x_cols],
				data_y=df.values[:, i][:, np.newaxis],
				data_z = df.values[:,[j for j in y_cols_reduced if (j!=i)]]
					)
		else:
			p = 0.0
		p_values.append(p)

	return p_values

def main(seed_list, x_mode, test_pval, experiment_name, pixel,
	batch_size, model_name, xv_mode, v_dim, base_dir, **args):
	seed_list = [int(i) for i in seed_list.split(",")]

	results = []
	for random_seed in seed_list:
		print(f'======= Random seed :{random_seed} ========')
		x_pvals = x_independence_test(
			random_seed = random_seed,
			pixel = pixel,
			batch_size = batch_size,
			model_name = model_name,
			xv_mode = xv_mode,
			v_dim = v_dim,
			base_dir = base_dir,
			x_mode = x_mode,
			experiment_name=experiment_name
			)

		y_pvals = y_independence_test(
			random_seed = random_seed,
			v_dim=v_dim,
			base_dir = base_dir,
			experiment_name=experiment_name
			)

		x_rel_bool = [x_pval <= test_pval for x_pval in x_pvals]
		x_rel = [i + 1 for i, i_true in enumerate(x_rel_bool) if i_true]

		y_rel_bool = [y_pval <= test_pval for y_pval in y_pvals]
		y_rel = [i + 1 for i, i_true in enumerate(y_rel_bool) if i_true]

		rel_aux = list(set(x_rel) & set(y_rel))

		rs_results = [random_seed, x_mode] + x_pvals + y_pvals + [rel_aux]
		results.append(rs_results)

	results = pd.DataFrame(results)
	results.columns = ['random_seed', 'x_mode'] + \
		[f'x_ci_y{i}' for i in range(1, len(x_pvals) + 1)] + \
		[f'y0_ci_y{i}' for i in range(1, len(y_pvals) + 1)] + \
		['relevant_aux_labs']

	print(results[['random_seed', 'x_mode', 'relevant_aux_labs']].head(10))

	results.to_csv(
		(f'{base_dir}/final_models/indep_tests_{x_mode}'
		f'_pix{pixel}_bs{batch_size}.csv'),
		index=False)

if __name__ == "__main__":

	parser = argparse.ArgumentParser()

	parser.add_argument('--experiment_name', '-experiment_name',
		help="which dataset?",
		choices=['waterbirds', 'chexpert', 'dr'],
		type=str)

	parser.add_argument('--base_dir', '-base_dir',
		help="Base directory where the final model will be saved",
		type=str)

	parser.add_argument('--random_seed', '-random_seed',
		help="Random seed for which we want to get predictions",
		default=-1,
		type=int)

	parser.add_argument('--seed_list', '-seed_list',
		help=("Comma separated list of seeds to get predictions for. "
				"overrides random seed"),
		type=str)

	parser.add_argument('--batch_size', '-batch_size',
		help="batch size",
		type=int)

	parser.add_argument('--pixel', '-pixel',
		help="pixels",
		type=int)

	parser.add_argument('--v_dim', '-v_dim',
		help="number of additional dimensions",
		type=int)

	parser.add_argument('--model_name', '-model_name',
		default='first_step',
		help="Which model to predict for",
		type=str)

	parser.add_argument('--xv_mode', '-xv_mode',
		default='classic',
		choices=['classic', 'two_step'],
		help=("which cross validation algorithm do you want to get preds for"),
		type=str)

	parser.add_argument('--x_mode', '-x_mode',
		default='sufficient_stat',
		choices=['sufficient_stat', 'high_dim'],
		help=("use high dim raw x or reduce using sufficient statistic?"),
		type=str)

	parser.add_argument('--test_pval', '-test_pval',
		default=0.001,
		help=("what is on the pval for the independence test?"),
		type=float)

	args = vars(parser.parse_args())
	main(**args)


from typing import Dict
from pathlib import Path
from datetime import datetime

from nonebot import get_bot
from nonebot.adapters import Bot
from nonebot.adapters.onebot.v11 import GroupMessageEvent

from ATRI.utils import FileDealer
from ATRI.service import ServiceTools
from ATRI.message import MessageBuilder
from ATRI.exceptions import load_error

from .models import RequestList

MANAGE_DIR = Path(".") / "data" / "plugins" / "manage"
MANAGE_DIR.mkdir(parents=True, exist_ok=True)

_TRACEBACK_FORMAT = (
    MessageBuilder("追踪ID：{trace_id}")
    .text("关键词：{prompt}")
    .text("时间：{time}")
    .text("{content}")
    .done()
)

class BotManager:
    async def __load_data(self, file_name: str) -> dict:
        path = MANAGE_DIR / file_name
        dealer = FileDealer(path)
        if not path.is_file():
            await dealer.write_json(dict())

        try:
            data = dealer.json()
        except Exception:
            data = dict()
        return data

    async def __store_data(self, file_name: str, data: dict) -> None:
        path = MANAGE_DIR / file_name
        dealer = FileDealer(path)
        if not path.is_file():
            await dealer.write_json(dict())

        await dealer.write_json(data)

    async def __load_block_group(self) -> dict:
        return await self.__load_data("block_group.json")

    async def __store_block_group(self, data: dict) -> None:
        await self.__store_data("block_group.json", data)

    async def __load_block_user(self) -> dict:
        return await self.__load_data("block_user.json")

    async def __store_block_user(self, data: dict) -> None:
        await self.__store_data("block_user.json", data)

    async def load_friend_req(self) -> RequestList:
        return RequestList.parse_obj(await self.__load_data("friend_add.json"))

    async def store_friend_req(self, data: dict) -> None:
        await self.__store_data("friend_add.json", data)

    async def load_group_req(self) -> RequestList:
        return RequestList.parse_obj(await self.__load_data("group_invite.json"))

    async def store_group_req(self, data: dict) -> None:
        await self.__store_data("group_invite.json", data)

    async def block_group(self, group_id: str) -> None:
        data = await self.__load_block_group()
        data[group_id] = {"time": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
        try:
            await self.__store_block_group(data)
        except Exception:
            raise Exception("写入文件时失败")

    async def unblock_group(self, group_id: str) -> None:
        data = await self.__load_block_group()
        if group_id not in data:
            raise Exception("群不存在于封禁名单")

        try:
            data.pop(group_id)
            await self.__store_block_group(data)
        except Exception:
            raise Exception("写入文件时失败")

    async def block_user(self, user_id: str) -> None:
        data = await self.__load_block_user()
        data[user_id] = {"time": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
        try:
            await self.__store_block_user(data)
        except Exception:
            raise Exception("写入文件时失败")

    async def unblock_user(self, user_id: str) -> None:
        data = await self.__load_block_user()
        if user_id not in data:
            raise Exception("用户不存在于封禁名单")

        try:
            data.pop(user_id)
            await self.__store_block_user(data)
        except Exception:
            raise Exception("写入文件时失败")

    def toggle_global_service(self, service: str) -> bool:
        serv = ServiceTools(service)
        try:
            data = serv.load_service()
        except Exception as e:
            error_msg = str(e)
            raise Exception(error_msg)

        data.enabled = not data.enabled
        serv.save_service(data)
        return data.enabled

    def toggle_group_service(self, service: str, event) -> bool:
        if isinstance(event, GroupMessageEvent):
            group_id = str(event.group_id)
            serv = ServiceTools(service)
            try:
                data = serv.load_service()
            except Exception as e:
                error_msg = str(e)
                raise Exception(error_msg)

            if group_id in data.disable_group:
                data.disable_group.remove(group_id)
                result = True
            else:
                data.disable_group.append(group_id)
                result = False
            serv.save_service(data)
            return result
        raise Exception("该功能只能在群聊中使用")

    def toggle_user_service(self, service: str, user_id: str) -> bool:
        serv = ServiceTools(service)
        try:
            data = serv.load_service()
        except Exception as e:
            error_msg = str(e)
            raise Exception(error_msg)

        if user_id in data.disable_user:
            data.disable_user.remove(user_id)
            result = True
        else:
            data.disable_user.append(user_id)
            result = False
        serv.save_service(data)
        return result

    async def track_error(self, trace_id: str) -> str:
        try:
            data = load_error(trace_id)
        except Exception:
            raise Exception("未找到对应ID的信息")

        return _TRACEBACK_FORMAT.format(
            trace_id=data.track_id,
            prompt=data.prompt,
            time=data.time,
            content=data.content,
        )

    def __get_bot(self) -> Bot:
        try:
            return get_bot()
        except Exception:
            raise Exception("无法获取 bot 实例")

    async def apply_friend_req(self, code: str) -> None:
        bot = self.__get_bot()
        try:
            await bot.call_api("set_friend_add_request", flag=code, approve=True)
        except Exception:
            raise Exception("同意失败，请尝试手动同意")
        raw_data = await self.load_friend_req()
        data = raw_data.data
        data.pop(code)
        raw_data.data = data
        await self.store_friend_req(raw_data.dict())

    async def reject_friend_req(self, code: str) -> None:
        bot = self.__get_bot()
        try:
            await bot.call_api("set_friend_add_request", flag=code, approve=False)
        except Exception:
            raise Exception("拒绝失败，请尝试手动拒绝")
        raw_data = await self.load_friend_req()
        data = raw_data.data
        data.pop(code)
        raw_data.data = data
        await self.store_friend_req(raw_data.dict())

    async def apply_group_req(self, code: str) -> None:
        bot = self.__get_bot()
        try:
            await bot.call_api(
                "set_group_add_request", flag=code, sub_type="invite", approve=True
            )
        except Exception:
            raise Exception("同意失败，请尝试手动同意")
        raw_data = await self.load_group_req()
        data = raw_data.data
        data.pop(code)
        raw_data.data = data
        await self.store_group_req(raw_data.dict())

    async def reject_group_req(self, code: str) -> None:
        bot = self.__get_bot()
        try:
            await bot.call_api(
                "set_group_add_request", flag=code, sub_type="invite", approve=False
            )
        except Exception:
            raise Exception("拒绝失败，请尝试手动拒绝")
        raw_data = await self.load_group_req()
        data = raw_data.data
        data.pop(code)
        raw_data.data = data
        await self.store_group_req(raw_data.dict())


import numpy as np
import difflib
import synonyms
import spacy

nlp = spacy.load("ru_core_news_lg")
# Позроверка поздаровался ли
def is_greet_in_phrase(doc):
    greet_list = synonyms.greet_list
    for token in doc:
        if token.lemma_ in greet_list and (
            token.dep_ == "amod" or token.dep_ == "ROOT"
        ):
            return True
    return False

# Проверка представился ли
def is_introduce_in_phrase(doc):
    intriduce_list = synonyms.intriduce_list
    for token in doc:
        # Проверяем на наличие именнованной сущиности Person, смотрим отношение nsubj к главному слову из списка для фраз "меня зовут", "моё имя", "с вами говорит", на всякий случай проверяем по леммам
        if token.ent_type_ == "PER" and token.head.lemma_ in intriduce_list:
            return True, token.text
        elif (
            token.lemma_ == "это"
            and token.dep_ == "nsubj"
            and token.head.dep_ == "ROOT"
        ):
            return True, token.head.text
    return False

# Получаем название компании
def similarity(s1, s2):
    normalized1 = s1.lower()
    normalized2 = s2.lower()
    matcher = difflib.SequenceMatcher(None, normalized1, normalized2)
    return matcher.ratio()

def get_company_name(doc):
    company_list = synonyms.company_list
    for i, token in enumerate(doc):
        count = i
        if token.lemma_ == "компания":
            company_name_list = list()
            for c, item in enumerate(company_list):
                company_name_list.append(similarity(doc[count + 1].text, item))
            company_index = company_name_list.index(max(company_name_list))
            if max(company_name_list) < 0.6:
                company_name = doc[count + 1].text
            else:
                company_name = company_list[company_index]
            return True, company_name
    return False

# проверка прощания
def is_bye(doc):
    byes_list = synonyms.byes_list
    for token in doc:
        if token.lemma_ in byes_list[0] or (
            token.lemma_ in byes_list[1] and token.dep_ == "case"
        ):
            return True
    return False

# получаем фарзы которые проверяем на наличие сущностей (dlg_len определяет n - первых и последних)
def get_phrases(arr, dlg_len, c, manager_phrases_index):
    if dlg_len > 0:
        phrases = arr[:, 3][manager_phrases_index[c][:dlg_len]]
        phrase_index = [manager_phrases_index[c][:dlg_len]][0]
        return phrases, phrase_index
    if dlg_len < 0:
        phrases = arr[:, 3][manager_phrases_index[c][dlg_len:]]
        phrase_index = [manager_phrases_index[c][dlg_len:]][0]
        return phrases, phrase_index

# проверяем фразы на наличие сущностей
def chek_phrase(phrases, phrase_index, function):
    for n, phrase in enumerate(phrases):
        doc = nlp(str(phrase))
        if function(doc):
            index = phrase_index[n]
            return (index, function(doc))
    return False

def get_result(arr, dlg_ids, dlg_len, manager_phrases_index):
    # проходимся по всем диалогам
    for c, item in enumerate(dlg_ids):
        # dlg_len = 3 #сколько фраз проверять

        phrases, phrase_index = get_phrases(arr, dlg_len, c, manager_phrases_index)

        greet = chek_phrase(phrases, phrase_index, is_greet_in_phrase)
        intro = chek_phrase(phrases, phrase_index, is_introduce_in_phrase)
        company = chek_phrase(phrases, phrase_index, get_company_name)
        dlg_len = dlg_len * -1  # сберем три последние

        phrases, phrase_index = get_phrases(arr, dlg_len, c, manager_phrases_index)
        bye = chek_phrase(phrases, phrase_index, is_bye)

        insight_txt = ""
        intro_n_comp = ""

        if greet:
            arr[:, 4][greet[0]] = "greeting=True"
            arr[:, 5][greet[0]] = "greeting=True"
            insight_txt += "greeting=True, "
        if not greet:
            ins_index = manager_phrases_index[c][0]  # первая фраза менеджера
            arr[:, 4][ins_index] = "greeting=False"
            arr[:, 5][ins_index] = "greeting=False"
            insight_txt += "greeting=False, "

        if intro:
            arr[:, 4][intro[0]] = "introduction=True"
            arr[:, 6][intro[0]] = "introduction=True"
            intro_n_comp += "introduction=True, "
            insight_txt += "introduction=True, "
            arr[:, 7][intro[0]] = intro[1][1]
        if not intro:
            intro_n_comp += "introduction=False, "
            insight_txt += "introduction=False, "
            arr[:, 6][ins_index] = "introduction=False"

        if company:
            intro_n_comp += "company=True"
            insight_txt += "company=True, "
            arr[:, 4][company[0]] = intro_n_comp
            arr[:, 8][company[0]] = company[1][1]
        if not company:
            insight_txt += "company=False, "
            intro_n_comp += "company=False"

        if bye:
            arr[:, 4][bye[0]] = "bying=True"
            arr[:, 9][bye[0]] = "bying=True"
            insight_txt += "bying=True"
        if not bye:
            insight_txt += "bying=False"
            ins_index = np.where(arr[:, 0] == item)[0][-1]
            arr[:, 4][ins_index] = "bying=False"  # последняя фраза
            arr[:, 9][ins_index] = "bying=False"

        if greet and bye:
            arr[:, 10][greet[0]] = "greet-bye=True"
        else:
            arr[:, 10][manager_phrases_index[c][0]] = "greet-bye=False"

        ins_index = np.where(arr[:, 0] == item)[0][0]  # индекс первой фразы диалога
        arr[:, 4][ins_index] = insight_txt
    return arr


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from sklearn import ensemble, neighbors, linear_model, metrics, preprocessing
from datetime import datetime
import glob, re
data = {
    'ar': pd.read_csv('../input/air_reserve.csv'),
    'as': pd.read_csv('../input/air_store_info.csv'),
    'tra': pd.read_csv('../input/air_visit_data.csv'),
    'hol': pd.read_csv('../input/date_info.csv').rename(columns={'calendar_date': 'visit_date'}),
    'hr': pd.read_csv('../input/hpg_reserve.csv'),
    'hs': pd.read_csv('../input/hpg_store_info.csv'),
    'tes': pd.read_csv('../input/sample_submission.csv'),
    'id': pd.read_csv('../input/store_id_relation.csv')
}
data['id'].sample(3)
data['hr'] = data['hr'].merge(data['id'], how='inner', on=['hpg_store_id'])
data['hr'].sample(5)
# Notice datetime initially an object. We will fix that
np.dtype(data['hr']['visit_datetime'])
for df in ['ar', 'hr']:
    data[df]['visit_datetime'] = pd.to_datetime(data[df]['visit_datetime'])
    data[df]['visit_datetime'] = data[df]['visit_datetime'].dt.date
    data[df]['reserve_datetime'] = pd.to_datetime(data[df]['reserve_datetime'])
    data[df]['reserve_datetime'] = data[df]['reserve_datetime'].dt.date
    data[df]['reserve_datetime_diff'] = data[df].apply(
        lambda r: (r['visit_datetime'] - r['reserve_datetime']).days, axis=1)
    data[df] = data[df].groupby(['air_store_id', 'visit_datetime'], as_index=False)[['reserve_datetime_diff',
                                                                                'reserve_visitors']].sum().rename(columns={'visit_datetime':'visit_date'})
data['hr'].sample(5)
data['ar'].sample(4)
data['as'].sample(5)
lbl = preprocessing.LabelEncoder()
data['as']['air_genre_name'] = lbl.fit_transform(data['as']['air_genre_name'])
data['as']['air_area_name'] = lbl.fit_transform(data['as']['air_area_name'])
data['as'].sample(3)
data['tra'].sample(4)
data['tra']['visit_date'] = pd.to_datetime(data['tra']['visit_date'])
data['tra']['day_of_week'] = data['tra']['visit_date'].dt.dayofweek
data['tra']['month'] = data['tra']['visit_date'].dt.month
data['tra']['year'] = data['tra']['visit_date'].dt.year
data['tra']['visit_date'] = data['tra']['visit_date'].dt.date

data['tra'].sample(4)
data['hol'].sample(5)
data['hol'] = data['hol'].drop(columns=['day_of_week'])
data['hol']['visit_date'] = pd.to_datetime(data['hol']['visit_date'])
data['hol']['visit_date'] = data['hol']['visit_date'].dt.date
data['hol'].sample(4)
data['hs'].sample(4)
data['tes'].sample(3)
data['tes']['visit_date'] = data['tes']['id'].map(lambda x: str(x).split('_')[2])
data['tes']['air_store_id'] = data['tes']['id'].map(lambda x: '_'.join(x.split('_')[:2]))
data['tes']['visit_date'] = pd.to_datetime(data['tes']['visit_date'])
data['tes']['day_of_week'] = data['tes']['visit_date'].dt.dayofweek
data['tes']['month'] = data['tes']['visit_date'].dt.month
data['tes']['year'] = data['tes']['visit_date'].dt.year
data['tes']['visit_date'] = data['tes']['visit_date'].dt.date
data['tes'].sample(4)
# Store each unique restaurant in an array
unique_stores = data['tes']['air_store_id'].unique()
# Break each restaurant into 7 rows to track each day of the week
stores = pd.concat([pd.DataFrame({'air_store_id': unique_stores,
                                  'day_of_week': [i]*len(unique_stores)}) for i in range(7)],
                   axis=0, ignore_index=True)
# Make a temporary variable to store new 'tra' features on visitors
# Then merge that new feature into our stores dataframe
tmp = data['tra'].groupby(['air_store_id', 'day_of_week'],
                          as_index=False)['visitors'].min().rename(columns={'visitors':'min_visitors'})
stores = stores.merge(tmp, how='left', on=['air_store_id','day_of_week'])
# Continue this process for mean, max, and count of visitors
tmp = data['tra'].groupby(['air_store_id', 'day_of_week'],
                         as_index=False)['visitors'].mean().rename(columns={'visitors':'mean_visitors'})
stores = stores.merge(tmp, how='left', on=['air_store_id', 'day_of_week'])
tmp = data['tra'].groupby(['air_store_id', 'day_of_week'],
                          as_index=False)['visitors'].median().rename(columns={'visitors':'median_visitors'})
stores = stores.merge(tmp, how='left', on=['air_store_id', 'day_of_week'])
tmp = data['tra'].groupby(['air_store_id', 'day_of_week'],
                          as_index=False)['visitors'].max().rename(columns={'visitors':'max_visitors'})
stores = stores.merge(tmp, how='left', on=['air_store_id', 'day_of_week'])
tmp = data['tra'].groupby(['air_store_id', 'day_of_week'],
                          as_index=False)['visitors'].count().rename(columns={'visitors':'visitor count'})
stores = stores.merge(tmp, how='left', on=['air_store_id', 'day_of_week'])
# Now we'll merge 'as' in
stores = stores.merge(data['as'], how='left', on=['air_store_id'])
# Check everything looks good and is machine readable
stores.head(4)
train = pd.merge(data['tra'], data['hol'], how='left', on=['visit_date'])
train = pd.merge(data['tra'], stores, how='left', on=['air_store_id', 'day_of_week'])
test = pd.merge(data['tes'], data['hol'], how='left', on=['visit_date'])
test = pd.merge(data['tes'], stores, how='left', on=['air_store_id', 'day_of_week'])
test.sample(3)
data['ar'] = data['ar'].merge(data['hr'], how='left', on=['air_store_id', 'visit_date', 'reserve_datetime_diff', 'reserve_visitors'])
data['ar'].sample(4)
train = train.merge(data['ar'], how='left', on=['air_store_id', 'visit_date'])
test = test.merge(data['ar'], how='left', on=['air_store_id', 'visit_date'])
train.sample(5)
col = [c for c in train if c not in ['id', 'air_store_id', 'visit_date', 'visitors']]
train = train.fillna(-1)
test = test.fillna(-1)
train.sample(4)
def RMSLE(y, pred):
    return metrics.mean_squared_error(y, pred)**0.5
etc = ensemble.ExtraTreesRegressor(n_estimators=225, max_depth=5, n_jobs=-1, random_state=3)
knn = neighbors.KNeighborsRegressor(n_jobs=-1, n_neighbors=4)
etc.fit(train[col], np.log1p(train['visitors'].values))
knn.fit(train[col], np.log1p(train['visitors'].values))
test['visitors'] = (etc.predict(test[col]) / 2) + (knn.predict(test[col]) / 2)
test['visitors'] = np.expm1(test['visitors']).clip(lower=0.)
sub1 = test[['id', 'visitors']].copy()
# Create a glob expression for finding and reading all .csv files in the data warehouse
dfs = {re.search('/([^/\.]*)\.csv', fn).group(1):
      pd.read_csv(fn) for fn in glob.glob('../input/*.csv')}

# store the CSVs locally for quick access
for k, v in dfs.items(): locals()[k] = v
#date_info['weight'] = ((date_info.index + 1) / len(date_info))       # LB 0.509
#date_info['weight'] = ((date_info.index + 1) / len(date_info)) ** 2  # LB 0.503
#date_info['weight'] = ((date_info.index + 1) / len(date_info)) ** 3  # LB 0.500
#date_info['weight'] = ((date_info.index + 1) / len(date_info)) ** 4  # LB 0.498
date_info['weight'] = ((date_info.index + 1) / len(date_info)) ** 5 # LB 0.497
visit_data = air_visit_data.merge(date_info, left_on = 'visit_date', right_on = 'calendar_date', how = 'left')
visit_data.drop('calendar_date', axis=1, inplace = True)
visit_data['visitors'] = visit_data.visitors.map(pd.np.log1p)
visit_data.head()
wmean = lambda x:((x.weight * x.visitors).sum() / x.weight.sum())
visitors = visit_data.groupby(['air_store_id', 'day_of_week', 'holiday_flg']).apply(wmean).rename('visitors').reset_index()
visitors.head()
sample_submission['air_store_id'] = sample_submission.id.map(lambda x: '_'.join(x.split('_')[:-1]))
sample_submission['calendar_date'] = sample_submission.id.map(lambda x: x.split('_')[2])
sample_submission.drop('visitors', axis=1, inplace=True)
sample_submission = sample_submission.merge(date_info, on='calendar_date', how='left')
sample_submission = sample_submission.merge(visitors, on=[
    'air_store_id', 'day_of_week', 'holiday_flg'], how='left')
sample_submission.head()
sample_submission.visitors.isnull().sum()
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
    visitors[visitors.holiday_flg == 0], on=('air_store_id', 'day_of_week'),
    how='left')['visitors_y'].values
sample_submission.isnull().sum()
missings = sample_submission.visitors.isnull()
sample_submission.loc[missings, 'visitors'] = sample_submission[missings].merge(
    visitors[['air_store_id', 'visitors']].groupby('air_store_id').mean().reset_index(),
    on='air_store_id', how='left')['visitors_y'].values
# Double check we filled all the missing values
sample_submission.isnull().sum()
sample_submission['visitors'] = sample_submission.visitors.map(pd.np.expm1)
sub2 = sample_submission[['id', 'visitors']].copy()
sub_merge = pd.merge(sub1, sub2, on='id', how='inner')
## Geometric Mean
sub_merge['visitors'] = (sub_merge['visitors_x'] * sub_merge['visitors_y']) ** (1/2)
sub_merge[['id', 'visitors']].to_csv('sub_geo_mean.csv', index = False)
sub_merge[['id', 'visitors']].head()
## Harmonic Mean
sub_merge['visitors'] = 2/(1/sub_merge['visitors_x'] + 1/sub_merge['visitors_y'])
sub_merge[['id', 'visitors']].to_csv('sub_hrm_mean.csv', index = False)
sub_merge[['id', 'visitors']].head()


from django.test import TestCase
from django.urls import reverse
from yogapp.forms import QuestionAndAnswerForm, RegisterLessonForm, PoseCreateForm, AnswerForm
from yogapp.models import LessonStyle, Instructer, Lesson, Pose, PoseCollection
from registration.models import User
from django.utils import timezone

import datetime
import random

class QandAFormTest(TestCase):

    def test_send_question_form(self):
        contents = {
            "category": "1",
            "detail": "hello world. this is form test. this test returns True."
        }
        form = QuestionAndAnswerForm(data=contents)
        self.assertTrue(form.is_valid())

    def test_send_question_form_unusable_detail(self):
        contents1 = {
            "category": "1",
            "detail": "helloworl"
        }
        contents2 = {
            "category": "1",
            "detail": "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901"
        }
        form1 = QuestionAndAnswerForm(data=contents1)
        form2 = QuestionAndAnswerForm(data=contents2)
        self.assertFalse(form1.is_valid())
        self.assertFalse(form2.is_valid())

class RegisterLessonFormTest(TestCase):

    def setUp(self):
        past = timezone.now() + datetime.timedelta(-1)
        future = timezone.now() + datetime.timedelta(1)
        far_future = timezone.now() + datetime.timedelta(367)
        self.past = Utils.create_datetime_string(past)
        self.future = Utils.create_datetime_string(future)
        self.far_future = Utils.create_datetime_string(far_future)
        Utils.create_pose_collection()

    # Testing for datetime
    def test_register_lesson_form(self):
        content1 = {
            "style": "normal",
            "lesson_date": self.future,
            "poses": Pose.objects.all(),
            "price": "1000"
        }
        content2 = {
            "style": "premium",
            "lesson_date": self.future,
            "poses": Pose.objects.all(),
            "price": "2000"
        }
        form1 = RegisterLessonForm(data=content1)
        form2 = RegisterLessonForm(data=content2)
        self.assertTrue(form1.is_valid())
        self.assertTrue(form2.is_valid())

    def test_register_past_lesson(self):
        content = {
            "style": "normal",
            "lesson_date": self.past,
            "poses": Pose.objects.all(),
            "price": "400"
        }
        form = RegisterLessonForm(data=content)
        self.assertFalse(form.is_valid())

    def test_register_far_future_lesson(self):
        content = {
            "style": "normal",
            "lesson_date": self.far_future,
            "poses": Pose.objects.all(),
            "price": "500"
        }
        form = RegisterLessonForm(data=content)
        self.assertFalse(form.is_valid())

    # Testing for style
    def test_register_unusable_style_lesson(self):
        content = {
            "style": "unusable",
            "lesson_date": self.future,
            "poses": Pose.objects.all(),
            "price": "300"
        }
        form = RegisterLessonForm(data=content)
        self.assertFalse(form.is_valid())

    def test_register_unusable_price_format(self):
        content = {
            "style": "normal",
            "lesson_date": self.future,
            "poses": Pose.objects.all(),
            "price": "testprice"
        }
        form = RegisterLessonForm(data=content)
        self.assertFalse(form.is_valid())

class PoseCreateFormTest(TestCase):
    def setUp(self):
        Pose.objects.create(name="TestPose", detail="this is test pose")

    def test_create_pose(self):
        content = {
            "pose_name": "test",
            "pose_detail": "this is test pose."
        }
        form = PoseCreateForm(data=content)
        self.assertTrue(form.is_valid())

    def test_create_pose_name_over_20(self):
        content = {
            "pose_name": "testtesttesttesttesttest",
            "pose_detail": "this is test pose."
        }
        form = PoseCreateForm(data=content)
        self.assertFalse(form.is_valid())

    def test_same_name_pose(self):
        content = {
            "pose_name": "TestPose",
            "pose_detail": "this is test pose"
        }
        form = PoseCreateForm(data=content)
        self.assertFalse(form.is_valid())

class AnswerFormTest(TestCase):
    def test_answer_the_question(self):
        content = {
            "detail": "this is test answer"
        }
        form = AnswerForm(data=content)
        self.assertTrue(form.is_valid())

    def test_small_answer(self):
        content = {
            "detail": "testtest"
        }
        form = AnswerForm(data=content)
        self.assertFalse(form.is_valid())

    def test_big_answer(self):
        content = {
            "detail": "this is test answer string 30.this is test answer string 30.this is test answer string 30.this is test answer string 30.this is test answer string 30.this is test answer string 30.this is test answer string 30."
        }
        form = AnswerForm(data=content)
        self.assertFalse(form.is_valid())

class Utils:

    @staticmethod
    def create_pose():
        name = Utils.generate_random_string(10)
        detail = Utils.generate_random_string(30)
        return Pose.objects.create(name=name, detail=detail)

    @staticmethod
    def create_pose_collection():
        pose1 = Utils.create_pose()
        pose2 = Utils.create_pose()
        collection = PoseCollection.objects.create()
        collection.poses.add(pose1)
        collection.poses.add(pose2)
        return collection

    @staticmethod
    def create_datetime_string(dt):
        # Convert datetime to %Y-%m-%dT%H:%M string
        st_date = str(dt.year) + "-" + str(dt.month) + "-" + str(dt.day) + "T" + str(dt.hour) + ":" + str(dt.minute)
        return st_date

    @staticmethod
    def generate_random_string(size=15):
        string = "abcdefghijklmnopqrstuvwxyz0123456789"
        random_string = ""
        for i in range(size):
            random_string += random.choice(string)
        return random_string


from flask import Flask, request, render_template
import IntentLoader

app = Flask(__name__)

@app.route('/')
def my_form():
    return render_template('my-form.html')

@app.route('/', methods=['POST'])
def my_form_post():
    text = request.form['text']
    processed_text = IntentLoader.getResult(text)
    return processed_text


import logging
import os
import numpy as np
from pytorch_lightning.callbacks import ModelCheckpoint

def count_trainable_parameters(model):
    model_parameters = filter(lambda p: p.requires_grad, model.parameters())
    params = sum([np.prod(p.size()) for p in model_parameters])
    return params

logger = logging.getLogger(__name__)

def get_checkpoint_callback(output_dir, metric):
    """Saves the best model by validation ROUGE2 score."""
    if metric == "rouge2":
        exp = "{val_avg_rouge2:.4f}-{step_count}"
    elif metric == "bleu":
        exp = "{val_avg_bleu:.4f}-{step_count}"
    elif metric == "sari_avgaddscore":
        exp = "{val_avg_sari_avgaddscore:.4f}-{step_count}"
    elif metric == "accuracy":
        exp = "{val_avg_accuracy:.4f}-{step_count}"
    else:
        raise NotImplementedError(
            f"seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this function."
        )

    checkpoint_callback = ModelCheckpoint(
        filepath=os.path.join(output_dir, exp),
        monitor=f"val_{metric}",
        mode="max",
        save_top_k=1,
        period=0,  # maybe save a checkpoint every time val is run, not just end of epoch.
    )
    return checkpoint_callback


#
# @lc app=leetcode id=34 lang=python3
#
# [34] Find First and Last Position of Element in Sorted Array
#

# @lc code=start
from typing import List

class Solution:
    def searchRange(self, nums: List[int], target: int) -> List[int]:
        if not nums:
            return [-1, -1]

        begin = 0
        end = len(nums) - 1

        if begin == end and nums[0] == target:
            return [0, 0]

        left = -1
        right = -1

        while begin < end:
            mid = (begin + end) // 2
            if nums[mid] < target:
                begin = mid + 1
            elif nums[mid] > target:
                end = mid
            else:
                # if mid < mid+1 or mid is last element
                # mid is right bound
                if (mid == len(nums) - 1) or (nums[mid+1] > target):
                    right = mid
                    break
                begin = mid + 1

        # begin = mid + 1 may out of loop
        if right == -1 and nums[begin] == target:
            right = begin

        begin = 0
        end = len(nums) - 1

        while begin < end:
            mid = (begin + end) // 2
            if nums[mid] < target:
                begin = mid + 1
            elif nums[mid] > target:
                end = mid
            else:
                # if mid > mid-1 or mid is first element
                # mid is left bound
                if (mid == 0) or (nums[mid-1] < target):
                    left = mid
                    break
                end = mid

        # end = mid may out of loop
        if left == -1 and nums[end] == target:
            left = end

        return [left, right]

if __name__ == "__main__":
    s = Solution()
    # print(s.searchRange([5, 7, 7, 8, 8, 10], 8))
    print(s.searchRange([2, 2], 2))
# @lc code=end


import os
import time
import errno
import select
import threading
import libvirt

from .utils import version_compare, Is_Fedora

class eventListenerThread(threading.Thread):
    def __init__(self, event_source, event_id, event_type, event_detail, logger, rand=None):
        """
        A thread that will keep looping until callback is called with a event matches
        given event id, detail and type.

        use join(timeout) and stop to wait for a event
        eg:
            thread = eventListenerThread("domain_name",
                                         libvirt.VIR_DOMAIN_EVENT_ID_LIFECYCLE,
                                         libvirt.VIR_DOMAIN_EVENT_DEFINED,
                                         libvirt.VIR_DOMAIN_EVENT_DEFINED_ADDED,
                                         logger,
                                         "ksNsd*2s1."
            )
            try:
                thread.start()
                # Test Code here
                # Remember to add thread.callback to event handlers
                thread.join(10)
                if thread.result:
                    # Got wanted event
                else:
                    # Didn't get the event :(
            finally:
                thread.stop()

        :param event_source: Where the waiting event should come from
            could be a domain/node device/others.
        :param event_id: (Not actually used yet).
        :param event_type: Event type to wait for.
        :param event_detail: Event detail to wait for.
        :param rand: A random string, you can use it to verify opaque's integrity.
            If not given, callback's opaque will be ignored.
            If given, callback's opaque need to be the same string.
        """
        threading.Thread.__init__(self, name="libvirtEventListener")
        self.result = None
        self.rand = rand
        self.logger = logger
        self.event_id = event_id
        self.event_type = event_type
        self.event_detail = event_detail
        self.event_source = event_source

    def callback(self, conn, source, event_type, event_detail, opaque):
        random = opaque
        logger = self.logger
        if hasattr(source, 'name'):
            src_name = source.name()
        else:
            src_name = str(source)
        if hasattr(source, 'UUIDString'):
            src_idtype, src_id = 'UUID: ', source.UUIDString()
        elif hasattr(source, 'ID'):
            src_idtype, src_id = 'ID: ', str(source.ID())
        else:
            src_idtype, src_id = 'NO ID Avaliable', ''

        logger.info("Got EVENT: From %s (%s%s), Type:%s, Detail:%s" %
                    (src_name, src_idtype, src_id, event_type, event_detail))

        if random != self.rand:
            logger.error("Callback's opaque is currupted !")
            return
        if self.event_type is not None and event_type != self.event_type:
            logger.error("Wrong event type!")
            return
        if self.event_detail is not None and event_detail != self.event_detail:
            logger.error("Wrong event detail!")
            return
        if self.event_source is not None and source.name() != self.event_source:
            logger.error("Wrong event source!")
            return
        logger.info("Just as expected!")
        self.result = True
        return

    def run(self):
        while self.result is None:
            time.sleep(1)

    def stop(self):
        self.result = False

class eventListenerThreadThreshold(threading.Thread):
    def __init__(self, event_source, event_id, threshold, logger, rand=None):
        threading.Thread.__init__(self, name="libvirtEventListener")
        self.result = None
        self.rand = rand
        self.logger = logger
        self.event_id = event_id
        self.event_source = event_source
        self.threshold = threshold

    def callbackThreshold(self, conn, source, dev, path, threshold, excess, opaque):
        random = opaque
        logger = self.logger
        if hasattr(source, 'name'):
            src_name = source.name()
        else:
            src_name = str(source)
        if hasattr(source, 'UUIDString'):
            src_idtype, src_id = 'UUID: ', source.UUIDString()
        elif hasattr(source, 'ID'):
            src_idtype, src_id = 'ID: ', str(source.ID())
        else:
            src_idtype, src_id = 'NO ID Avaliable', ''

        logger.info("Got EVENT: From %s (%s%s), Threshold:%s" %
                    (src_name, src_idtype, src_id, threshold))

        if random != self.rand:
            logger.error("Callback's opaque is currupted !")
            return
        if self.event_source is not None and source.name() != self.event_source:
            logger.error("Wrong event source!")
            return
        if self.threshold is not None and str(threshold) != self.threshold:
            logger.error("Wrong threshold!")
            return

        logger.info("Just as expected!")
        self.result = True

    def run(self):
        while self.result is None:
            time.sleep(1)

    def stop(self):
        self.result = False

###############################################################
# Following class is a wrap up of the event loop
# from libvirt-python/examples/event-test.py
###############################################################
#
# This general purpose event loop will support waiting for file handle
# I/O and errors events, as well as scheduling repeatable timers with
# a fixed interval.
#
# It is a pure python implementation based around the poll() API
#
class virEventLoopPureThread(threading.Thread):
    """
    To use this standalone pure event loop,
    Init a instance, regist and run it. Remember to stop the running thread
    before test ends.
    eg:
        loop = virEventLoopPure(logger)
        try:
            loop.regist(libvirt)
            loop.start()
            # Run test here
        finally:
            loop.stop()
    """
    # This class contains the data we need to track for a
    # single file handle

    class virEventLoopPureHandle:

        def __init__(self, handle, fd, events, cb, opaque):
            self.handle = handle
            self.fd = fd
            self.events = events
            self.cb = cb
            self.opaque = opaque

        def get_id(self):
            return self.handle

        def get_fd(self):
            return self.fd

        def get_events(self):
            return self.events

        def set_events(self, events):
            self.events = events

        def dispatch(self, events):
            self.cb(self.handle,
                    self.fd,
                    events,
                    self.opaque)

    # This class contains the data we need to track for a
    # single periodic timer
    class virEventLoopPureTimer:

        def __init__(self, timer, interval, cb, opaque):
            self.timer = timer
            self.interval = interval
            self.cb = cb
            self.opaque = opaque
            self.lastfired = 0

        def get_id(self):
            return self.timer

        def get_interval(self):
            return self.interval

        def set_interval(self, interval):
            self.interval = interval

        def get_last_fired(self):
            return self.lastfired

        def set_last_fired(self, now):
            self.lastfired = now

        def dispatch(self):
            self.cb(self.timer,
                    self.opaque)

    def __init__(self, logger):
        threading.Thread.__init__(self, name="libvirtEventLoop-Pure")
        self.poll = select.poll()
        self.pipetrick = os.pipe()
        self.pendingWakeup = False
        self.runningPoll = False
        self.nextHandleID = 1
        self.nextTimerID = 1
        self.handles = []
        self.timers = []
        self.quit = False
        self.logger = logger
        if Is_Fedora() or version_compare("libvirt-python", 3, 7, 0, logger):
            self.cleanup = []

        # The event loop can be used from multiple threads at once.
        # Specifically while the main thread is sleeping in poll()
        # waiting for events to occur, another thread may come along
        # and add/update/remove a file handle, or timer. When this
        # happens we need to interrupt the poll() sleep in the other
        # thread, so that it'll see the file handle / timer changes.
        #
        # Using OS level signals for this is very unreliable and
        # hard to implement correctly. Thus we use the real classic
        # "self pipe" trick. A anonymous pipe, with one end registered
        # with the event loop for input events. When we need to force
        # the main thread out of a poll() sleep, we simple write a
        # single byte of data to the other end of the pipe.
        self.logger.debug("Self pipe watch %d write %d" % (self.pipetrick[0],
                                                           self.pipetrick[1]))
        self.poll.register(self.pipetrick[0], select.POLLIN)

    # Calculate when the next timeout is due to occurr, returning
    # the absolute timestamp for the next timeout, or 0 if there is
    # no timeout due
    def next_timeout(self):
        next = 0
        for t in self.timers:
            last = t.get_last_fired()
            interval = t.get_interval()
            if interval < 0:
                continue
            if next == 0 or (last + interval) < next:
                next = last + interval

        return next

    # Lookup a virEventLoopPureHandle object based on file descriptor
    def get_handle_by_fd(self, fd):
        for h in self.handles:
            if h.get_fd() == fd:
                return h
        return None

    # Lookup a virEventLoopPureHandle object based on its event loop ID
    def get_handle_by_id(self, handleID):
        for h in self.handles:
            if h.get_id() == handleID:
                return h
        return None

    # This is the heart of the event loop, performing one single
    # iteration. It asks when the next timeout is due, and then
    # calcuates the maximum amount of time it is able to sleep
    # for in poll() pending file handle events.
    #
    # It then goes into the poll() sleep.
    #
    # When poll() returns, there will zero or more file handle
    # events which need to be dispatched to registered callbacks
    # It may also be time to fire some periodic timers.
    #
    # Due to the coarse granularity of schedular timeslices, if
    # we ask for a sleep of 500ms in order to satisfy a timer, we
    # may return upto 1 schedular timeslice early. So even though
    # our sleep timeout was reached, the registered timer may not
    # technically be at its expiry point. This leads to us going
    # back around the loop with a crazy 5ms sleep. So when checking
    # if timeouts are due, we allow a margin of 20ms, to avoid
    # these pointless repeated tiny sleeps.
    def run_once(self):
        sleep = -1
        self.runningPoll = True

        if version_compare("libvirt-python", 3, 7, 0, self.logger):
            for opaque in self.cleanup:
                libvirt.virEventInvokeFreeCallback(opaque)
            self.cleanup = []

        try:
            next = self.next_timeout()
            #self.logger.debug("Next timeout due at %d" % next)
            if next > 0:
                now = int(time.time() * 1000)
                if now >= next:
                    sleep = 0
                else:
                    sleep = (next - now) / 1000.0

            #self.logger.debug("Poll with a sleep of %d" % sleep)
            events = self.poll.poll(sleep)

            # Dispatch any file handle events that occurred
            for (fd, revents) in events:
                # See if the events was from the self-pipe
                # telling us to wakup. if so, then discard
                # the data just continue
                if fd == self.pipetrick[0]:
                    self.pendingWakeup = False
                    data = os.read(fd, 1)
                    continue

                h = self.get_handle_by_fd(fd)
                if h:
                    #self.logger.debug("Dispatch fd %d handle %d events %d" %
                    #                  (fd, h.get_id(), revents))
                    h.dispatch(self.events_from_poll(revents))

            now = int(time.time() * 1000)
            for t in self.timers:
                interval = t.get_interval()
                if interval < 0:
                    continue

                want = t.get_last_fired() + interval
                # Deduct 20ms, since scheduler timeslice
                # means we could be ever so slightly early
                if now >= (want - 20):
                    #self.logger.debug("Dispatch timer %d now %s want %s" %
                    #                  (t.get_id(), str(now), str(want)))
                    t.set_last_fired(now)
                    t.dispatch()

        except (os.error, select.error) as e:
            if e.args[0] != errno.EINTR:
                raise
        finally:
            self.runningPoll = False

    # Actually the event loop forever
    def run_loop(self):
        self.quit = False
        while not self.quit:
            self.run_once()

    def interrupt(self):
        if self.runningPoll and not self.pendingWakeup:
            self.pendingWakeup = True
            os.write(self.pipetrick[1], 'c'.encode())

    # Registers a new file handle 'fd', monitoring  for 'events' (libvirt
    # event constants), firing the callback  cb() when an event occurs.
    # Returns a unique integer identier for this handle, that should be
    # used to later update/remove it
    def add_handle(self, fd, events, cb, opaque):
        handleID = self.nextHandleID + 1
        self.nextHandleID = self.nextHandleID + 1

        h = self.virEventLoopPureHandle(handleID, fd, events, cb, opaque)
        self.handles.append(h)

        self.poll.register(fd, self.events_to_poll(events))
        self.interrupt()

        #self.logger.debug("Add handle %d fd %d events %d" %
        #                  (handleID, fd, events))

        return handleID

    # Registers a new timer with periodic expiry at 'interval' ms,
    # firing cb() each time the timer expires. If 'interval' is -1,
    # then the timer is registered, but not enabled
    # Returns a unique integer identier for this handle, that should be
    # used to later update/remove it
    def add_timer(self, interval, cb, opaque):
        timerID = self.nextTimerID + 1
        self.nextTimerID = self.nextTimerID + 1

        h = self.virEventLoopPureTimer(timerID, interval, cb, opaque)
        self.timers.append(h)
        self.interrupt()

        #self.logger.debug("Add timer %d interval %d" % (timerID, interval))

        return timerID

    # Change the set of events to be monitored on the file handle
    def update_handle(self, handleID, events):
        h = self.get_handle_by_id(handleID)
        if h:
            h.set_events(events)
            self.poll.unregister(h.get_fd())
            self.poll.register(h.get_fd(), self.events_to_poll(events))
            self.interrupt()

            #self.logger.debug("Update handle %d fd %d events %d" %
            #                  (handleID, h.get_fd(), events))

    # Change the periodic frequency of the timer
    def update_timer(self, timerID, interval):
        for h in self.timers:
            if h.get_id() == timerID:
                h.set_interval(interval)
                self.interrupt()

                #self.logger.debug("Update timer %d interval %d" %
                #                  (timerID, interval))
                break

    # Stop monitoring for events on the file handle
    def remove_handle(self, handleID):
        handles = []
        for h in self.handles:
            if h.get_id() == handleID:
                self.poll.unregister(h.get_fd())
                if version_compare("libvirt-python", 3, 7, 0, self.logger):
                    self.cleanup.append(h.opaque)
                #self.logger.debug("Remove handle %d fd %d" %
                #                  (handleID, h.get_fd()))
            else:
                handles.append(h)
        self.handles = handles
        self.interrupt()

    # Stop firing the periodic timer
    def remove_timer(self, timerID):
        timers = []
        for h in self.timers:
            if version_compare("libvirt-python", 3, 7, 0, self.logger):
                if h.get_id() != timerID:
                    timers.append(h)
                else:
                    self.cleanup.append(h.opaque)
            else:
                if h.get_id() != timerID:
                    timers.append(h)
        self.timers = timers
        self.interrupt()

    # Convert from libvirt event constants, to poll() events constants
    def events_to_poll(self, events):
        ret = 0
        if events & libvirt.VIR_EVENT_HANDLE_READABLE:
            ret |= select.POLLIN
        if events & libvirt.VIR_EVENT_HANDLE_WRITABLE:
            ret |= select.POLLOUT
        if events & libvirt.VIR_EVENT_HANDLE_ERROR:
            ret |= select.POLLERR
        if events & libvirt.VIR_EVENT_HANDLE_HANGUP:
            ret |= select.POLLHUP
        return ret

    # Convert from poll() event constants, to libvirt events constants
    def events_from_poll(self, events):
        ret = 0
        if events & select.POLLIN:
            ret |= libvirt.VIR_EVENT_HANDLE_READABLE
        if events & select.POLLOUT:
            ret |= libvirt.VIR_EVENT_HANDLE_WRITABLE
        if events & select.POLLNVAL:
            ret |= libvirt.VIR_EVENT_HANDLE_ERROR
        if events & select.POLLERR:
            ret |= libvirt.VIR_EVENT_HANDLE_ERROR
        if events & select.POLLHUP:
            ret |= libvirt.VIR_EVENT_HANDLE_HANGUP
        return ret

    def regist(self, libvirt):
        # Now glue an instance of the general event loop into libvirt's event loop
        # These next set of 6 methods are the glue between the official
        # libvirt events API, and our particular impl of the event loop
        #
        # There is no reason why the 'virEventLoopPure' has to be used.
        # An application could easily may these 6 glue methods hook into
        # another event loop such as GLib's, or something like the python
        # Twisted event framework.

        def virEventAddHandleImpl(fd, events, cb, opaque):
            return self.add_handle(fd, events, cb, opaque)

        def virEventUpdateHandleImpl(handleID, events):
            return self.update_handle(handleID, events)

        def virEventRemoveHandleImpl(handleID):
            return self.remove_handle(handleID)

        def virEventAddTimerImpl(interval, cb, opaque):
            return self.add_timer(interval, cb, opaque)

        def virEventUpdateTimerImpl(timerID, interval):
            return self.update_timer(timerID, interval)

        def virEventRemoveTimerImpl(timerID):
            return self.remove_timer(timerID)

        # This tells libvirt what event loop implementation it
        # should use
        libvirt.virEventRegisterImpl(virEventAddHandleImpl,
                                     virEventUpdateHandleImpl,
                                     virEventRemoveHandleImpl,
                                     virEventAddTimerImpl,
                                     virEventUpdateTimerImpl,
                                     virEventRemoveTimerImpl)

    def unregist(self, libvirt):
        # Not used yet
        pass

    def start(self):
        self.setDaemon(True)
        threading.Thread.start(self)

    def run(self):
        self.run_loop()

    def stop(self):
        self.quit = True
        self.interrupt()

def eventLoopPure(logger):
    eventLoop = virEventLoopPureThread(logger)
    eventLoop.regist(libvirt)
    eventLoop.start()

    return 0


from django.db import models
from django.utils import dates

class GenderChoices(models.TextChoices):
    MALE = 'M', 'Male'
    FEMALE = 'F', 'Female'
    OTHER = 'O', 'Other'

class Employee(models.Model):
    first_name = models.CharField(max_length=30, blank=False, null=False)
    last_name = models.CharField(max_length=30, blank=False, null=False)
    email_address = models.EmailField(blank=False, null=False, unique=True)
    gender = models.CharField(max_length=1, choices=GenderChoices.choices, default=GenderChoices.OTHER)

    def __str__(self):
        return str(self.email_address)

class EmployeeEvents(models.Model):
    employee = models.ForeignKey(to=Employee, on_delete=models.CASCADE)
    event_type = models.CharField(max_length=10, blank=False, null=False)
    event_date = models.DateField(blank=False, null=False)
    event_month = models.IntegerField(choices=dates.MONTHS.items(), blank=True, null=True)

    class Meta:
        unique_together = ('employee', 'event_type')

    @classmethod
    def get_data_by_month(cls, month, email_sent_for_today_event_ids):
        return cls.objects.filter(event_month=month).exclude(pk__in=email_sent_for_today_event_ids)


from random import random
import numpy as np
import soundfile as sf
import ipywidgets as widgets
import matplotlib.pyplot as plt
from IPython.display import (
    Audio, display, clear_output)

default_widgets_params = {
    'min': 0.1,
    'max': 1.0,
    'step': 0.01,
    'disabled': False,
    'continuous_update': False,
    'orientation': 'horizontal',
    'readout': True,
    'readout_format': '.2f',
}

attack_widgets_params = {
    'value': 0.05,
    'min': 0.01,
    'max': 0.4,
    'step': 0.01,
    'disabled': False,
    'continuous_update': False,
    'orientation': 'horizontal',
    'readout': True,
    'readout_format': '.2f',
    'description': 'Attack'
}

release_widgets_params = {
    'value': 0.5,
    'min': 0.05,
    'max': 0.6,
    'step': 0.01,
    'disabled': False,
    'continuous_update': False,
    'orientation': 'horizontal',
    'readout': True,
    'readout_format': '.2f',
    'description': 'Release'
}

DEFAULT_MODEL = 'log_kicks_high/'
model_widgets_params = {
    'options': [('Kicks norm', 'log_kicks/'), ('Kicks high', 'log_kicks_high/'), ('Kicks full', 'log_kicks_full'),
                ('FS norm', 'log_free/'), ('FS high', 'log_free_high/'), ('FS full', 'log_free_full/')],
    'value': DEFAULT_MODEL,
    'description': 'Model',
    'disabled': False,
}

CURRENT_MODEL_NAME = DEFAULT_MODEL

# not used, just here for keeping good example values
DEFAULT_KICKS_VALUES = [0.46533436, 0.6132435, 0.6906892, 0.5227648, 0.6955591, 0.733622, 0.4321724]
DEFAULT_FS_VALUES = [0.7279065, 0.7787189, 0.4476448, 0.7108625, 0.39033762, 0.4943667, 0.5660683]

def generate_envelope(attack, release, sr=16000):
    envelope = []
    if attack + release <= 1 :
        envelope = np.concatenate((np.linspace(0, 1, int(attack*sr)), np.linspace(1, 0, int(release*sr))))
        result = np.zeros(16000)
        result[:len(envelope)] =  envelope
    return result

def return_widgets(callback):
    model_name = widgets.Dropdown(**model_widgets_params)
    attack = widgets.FloatSlider(**attack_widgets_params)
    release = widgets.FloatSlider(**release_widgets_params)
    brightness = widgets.FloatSlider(description='Brightness', value=0.46, **default_widgets_params)
    hardness = widgets.FloatSlider(description='Hardness', value=0.61, **default_widgets_params)
    depth = widgets.FloatSlider(description='Depth', value=0.69, **default_widgets_params)
    roughness = widgets.FloatSlider(description='Roughness', value=0.52, **default_widgets_params)
    boominess = widgets.FloatSlider(description='Boominess', value=0.69, **default_widgets_params)
    warmth = widgets.FloatSlider(description='Warmth', value=0.73, **default_widgets_params)
    sharpness = widgets.FloatSlider(description='Sharpness', value=0.43, **default_widgets_params)

    out = widgets.interactive_output(callback, {
        'model_name': model_name,
        'attack': attack,
        'release': release,
        'brightness': brightness,
        'hardness': hardness,
        'depth': depth,
        'roughness': roughness,
        'boominess': boominess,
        'warmth': warmth,
        'sharpness': sharpness
    })

    return model_name, attack, release, brightness, hardness, depth, roughness, boominess, warmth, sharpness, out