import json
from pathlib import Path

import h5py
import numpy as np
from tensorflow.keras import backend as K
import tensorflow.keras as keras
import tensorflow as tf
import os
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.optimizers import Adam

class WBAttackGenerator:
    """
    Class to generate WB Train/Test Data
    WB-Input layer consists of several components including:
    - gradient
    - Output Hidden Layers
    - Loss
    - Label
    """
    DS_NAME = 'attack_data'

    def __init__(self, model, X, Y,
                 train_indices, test_indices, batch_size,
                 num_classes, last_layer_only=True, one_hot=False):
        """
        train_data_file: Train data from target model
        test_data_file: Test data from target model
        """
        assert len(X) == len(Y), f"size of X and Y must match {len(X)} != {len(Y)}"
        assert batch_size > 0, "batch size cannot be negative"
        assert num_classes > 0, "num classes must be greater 0"
        self.X = X
        self.Y = Y
        self.batch_size = batch_size
        self.num_classes = num_classes
        self.one_hot = one_hot
        self.one_hot_Y = keras.utils.to_categorical(self.Y, self.num_classes)
        self.train_indices = train_indices
        self.test_indices = test_indices
        self.last_layer_only = last_layer_only
        self.layers_used = []
        self.model = model
        loss = SparseCategoricalCrossentropy(reduction=tf.compat.v1.losses.Reduction.NONE)
        self.model.compile(optimizer=Adam(0.001), loss=loss, metrics=['accuracy'])
        # Init gradient extraction
        self.keras_get_gradients = self._init_gradient_extraction()

    def write_attack_info(self, folder_path, name):
        """
        Write a json file containing the used layer information (# neurons)
        And the file paths to the train/test-files
        """

        file_name_train = f'{folder_path}/{name}_target_train_attack_data.h5'
        file_name_test = f'{folder_path}/{name}_target_test_attack_data.h5'

        assert not os.path.exists(file_name_train), "training file already exists"
        assert not os.path.exists(file_name_test), "test file already exists"
        self._create_attack_data(file_name_train, self.train_indices)

        self._create_attack_data(file_name_test, self.test_indices)
        data_inf = {}
        data_inf['last_layer_only'] = self.last_layer_only
        data_inf['attack_train_file'] = file_name_train
        data_inf['attack_test_file'] = file_name_test
        data_inf['layers_used'] = self.layers_used

        data_inf_file = f'{folder_path}/{name}_data_inf.json'
        with open(data_inf_file, 'w') as outfile:
            json.dump(data_inf, outfile, indent=4, sort_keys=True)

    def _create_attack_data(self, output_file, indices):
        """
        extract and serialize the Features from the Target Model for the White Box Attacker Model
        """
        total_elements = len(indices)
        percent = 0
        print('Creating ' + output_file)
        print("get layer outputs")
        l_outs = self.get_layer_outputs()
        print("get losses")
        losses = self.get_losses()
        # loop datapoints in set
        print("get gradients")
        for i, ind in enumerate(indices):
            per = (i * 100) // total_elements
            if percent < per:
                percent = per
                print('Progress ' + str(per) + '%')

            # get gradients for datapoint
            gradients = self.get_gradients(self.X[ind:ind + 1], self.Y[ind:ind + 1])

            grads = None
            # at this point gradients are still in shape (n x m)
            # (where n is the previous layer size and m the current layer size)
            # if reshape_grads is set to True,
            # the gradient matrix will be transposed to get a (m x n) matrix before flattening
            reshape_grads = True
            for g in gradients:
                # append bias gradients to weight gradient matrix
                g_l = np.concatenate((g[0], [g[1]]))
                if reshape_grads:
                    g_l = g_l.T

                if grads is None:
                    grads = g_l.flatten()
                else:
                    grads = np.concatenate([grads, g_l.flatten()])

            # get outputs for datapoint
            l_out = None
            for l in l_outs:
                if l_out is None:
                    l_out = l[ind]
                else:
                    l_out = np.concatenate([l_out, l[ind]])

            # concatenate all components data
            if self.one_hot:
                output = np.concatenate(
                    [grads, l_out, np.array([losses[ind]]), self.one_hot_Y[ind]])
            else:
                output = np.concatenate([grads, l_out, np.array([losses[ind]]), self.Y[ind]])

            # write data to output file
            WBAttackGenerator.write_to_file(output_file, '%s' % WBAttackGenerator.DS_NAME, output)

    def get_layer_outputs(self):
        """
        get the layer outputs
        """
        layer_outputs = []
        percent = 0

        for i in range(0, self._get_iterations()):
            per = (i * 100) // self._get_iterations()
            if percent < per:
                percent = per
                print('Progress ' + str(per) + '%')
            start_i = i * self.batch_size
            end_i = (i + 1) * self.batch_size
            if i > self._get_batches():
                end_i = start_i + self._get_rest()

            layer_outputs.append(self._get_layer_outputs(self.X[start_i:end_i]))

        return np.concatenate(layer_outputs[:], axis=1)

    def get_losses(self):
        """
        Retrieve the losses by evaluating the model
        """
        all_losses = []
        percent = 0
        for i in range(0, self._get_iterations()):
            per = (i * 100) // self._get_iterations()
            if percent < per:
                percent = per
                print('Progress ' + str(per) + '%')

            start_i = i * self.batch_size
            end_i = (i + 1) * self.batch_size
            if i > self._get_batches():
                end_i = start_i + self._get_rest()

            loss = \
                self.model.evaluate(self.X[start_i:end_i], self.Y[start_i:end_i], batch_size=self.batch_size,
                                    verbose=0)[
                    0]
            all_losses.append(loss)
        return np.concatenate(all_losses[:])

    def get_gradients(self, x, y):
        """
        Return the gradient of every trainable weight in model
        """

        """
        gradients_tensor = K.gradients(self.model.output, weights_tensor)
        input_tensors = (
                    self.model._feed_inputs + self.model._feed_targets + self.model._feed_sample_weights)  # TODO: This was changed
        iterate = K.function(input_tensors, gradients_tensor)

        grads = np.array(iterate([x, y]))
        """
        grads = self.keras_get_gradients([x, y])
        grad_w_b = np.empty((len(grads) // 2, 2), dtype=object)
        # grad_biases = np.empty((grads/2, 1))

        # gradients always consist of two matrices
        # for each layer (one matrix for weights (n x m) and one for biases(1 x m))
        for i, l in enumerate(grads):
            if not i % 2:
                # l = [item for sublist in l for item in sublist]
                grad_w_b[i // 2, 0] = np.array(l)
            else:
                grad_w_b[i // 2, 1] = np.array(l)

        return grad_w_b

    def _init_gradient_extraction(self):
        """
        Set the trainable weights and the gradient function
        """
        weights_tensor = self.model.trainable_weights
        if self.last_layer_only:
            weights_tensor = weights_tensor[-2:]
        gradients_tensor = tf.gradients(self.model.total_loss, weights_tensor)
        input_tensors = self.model.inputs + self.model._targets
        return K.function(inputs=input_tensors, outputs=gradients_tensor)

    def _get_batches(self):
        return self.Y.shape[0] // self.batch_size

    def _get_rest(self):
        return self.Y.shape[0] % self.batch_size

    def _get_iterations(self):
        if self._get_rest() > 0:
            iterations = self._get_batches() + 1
        else:
            iterations = self._get_batches()
        return iterations

    def _get_layer_output(self, layer=1):
        return K.function([self.model.layers[0].input], [self.model.layers[layer].output])

    def _get_layer_outputs(self, X):
        layer_outputs = []
        if self.last_layer_only:
            layer_outputs.append(self._get_layer_output(-1)([X])[0])
            if not self.layers_used:
                self.layers_used.append(self.model.layers[-2].units)
                self.layers_used.append(self.model.layers[-1].units)
        else:
            for i in range(0, len(self.model.layers)):
                layer_outputs.append(self._get_layer_output(i)([X])[0])
                if not self.layers_used:
                    self.layers_used.append(self.model.layers[i].units)

        return np.asarray(layer_outputs)

    @staticmethod
    def write_to_file(file_path, ds_name, payload):
        """
        Write the features to a file
        """
        my_file = Path(file_path)

        if my_file.is_file():
            with h5py.File(file_path, 'a') as hf:
                hf[ds_name].resize((hf[ds_name].shape[0] + 1), axis=0)
                hf[ds_name][hf[ds_name].shape[0] - 1, :] = payload
        else:
            with h5py.File(file_path, 'w') as hf:
                hf.create_dataset(ds_name, data=payload[np.newaxis], compression='gzip', chunks=True, maxshape=(
                    None, payload.shape[0]))


import os
import cv2

import torch

class ShopeeDataset(torch.utils.data.Dataset):
    '''
    Define Shopee Dataset
    '''
    def __init__(self, df, transform=None):
        self.df = df
        self.root_dir = 'train_images'
        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):

        row = self.df.iloc[idx]
        # define image path
        image_path = os.path.join(self.root_dir, row.image)
        # read input image and transform
        transformed_image = cv2.imread(image_path)
        transformed_image = cv2.cvtColor(transformed_image, cv2.COLOR_BGR2RGB)
        label = row.label_group

        augmented_image = self.transform(image=transformed_image)
        transformed_image = augmented_image['image']

        return {
            'image': transformed_image,
            'label': torch.tensor(label).long()
        }


import tensorflow as tf
import math
import time
from datetime import datetime
import os
from PIL import Image, ImageDraw, ImageFont
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt

class Data:
    def __init__(self, batch_size, num_classes, data_path, val_data):
        """
        :param batch_size: batch size
        :param num_classes: number of classes for datasets
        :param data_path: training set path
        :param val_data: validation set path
        """
        self.batch_size = batch_size
        self.num_classes = num_classes
        self.data_path = data_path
        self.labels_name = []
        self.val_data = val_data
        self.image_names = os.listdir(self.data_path)
        for name in tqdm(self.image_names):
            class_name = name.split('.')[0].split('_')[-1]
            self.labels_name.append(class_name)
        class_set = set(self.labels_name)
        self.labels_dict = {}
        for v, k in enumerate(class_set):
            self.labels_dict[k] = v
        print("Data Loading finished!")
        print("Label dict: ", self.labels_dict)
        self.labels = [self.labels_dict.get(k) for k in self.labels_name]
        self.inverse_dict = dict((val, key) for key, val in self.labels_dict.items())

    # convert label to one hot form
    def getOneHotLabel(self, labels, depth):
        label = np.zeros([len(labels), depth])
        for i in range(len(labels)):
            label[i][labels[i]] = 1
        return label

    def get_batch(self, count):
        start = count * self.batch_size
        end = (count + 1) * self.batch_size
        start_pos = max(0, start)
        end_pos = min(end, len(self.labels))
        images_name_batch = self.image_names[start_pos: end_pos]
        images = []
        for images_name in images_name_batch:
            image_path = os.path.join(self.data_path, images_name)
            image = Image.open(image_path)
            image = np.array(image) / 255.0
            images.append(image)
        labels = self.labels[start_pos: end_pos]
        datas = np.array(images)
        labels = np.array(labels)
        labels = self.getOneHotLabel(labels, self.num_classes)
        return datas, labels

    def get_batch_num(self):
        return len(self.labels) // self.batch_size

    def get_batch_size(self):
        return self.batch_size

    def get_val_data(self):
        val_names = os.listdir(self.val_data)
        val_images = []
        val_labels = []
        for name in val_names:
            image_path = os.path.join(self.val_data, name)
            image = Image.open(image_path)
            image = np.array(image) / 255.0
            val_images.append(image)
            class_name_val = name.split('.')[0].split('_')[-1]
            val_labels.append(class_name_val)
        val_images = np.array(val_images)
        val_labels = [self.labels_dict.get(k) for k in val_labels]
        val_labels = np.array(val_labels)
        val_labels = self.getOneHotLabel(val_labels, self.num_classes)
        return val_images, val_labels

    def get_label_dict(self):
        return self.labels_dict


"""Tests of email marketing signal handlers."""
import datetime
import logging

import ddt
from django.contrib.auth.models import AnonymousUser
from django.contrib.sites.models import Site
from django.test import TestCase
from django.test.client import RequestFactory
from freezegun import freeze_time
from mock import ANY, patch
from opaque_keys.edx.keys import CourseKey
from sailthru.sailthru_error import SailthruClientError
from sailthru.sailthru_response import SailthruResponse
from testfixtures import LogCapture

from email_marketing.models import EmailMarketingConfiguration
from email_marketing.signals import (
    add_email_marketing_cookies,
    email_marketing_register_user,
    email_marketing_user_field_changed
)
from email_marketing.tasks import (
    _create_user_list,
    _get_list_from_email_marketing_provider,
    _get_or_create_user_list,
    update_user,
    update_user_email
)
from student.tests.factories import UserFactory, UserProfileFactory
from util.json_request import JsonResponse

log = logging.getLogger(__name__)

LOGGER_NAME = "email_marketing.signals"

TEST_EMAIL = "test@edx.org"

def update_email_marketing_config(enabled=True, key='badkey', secret='badsecret', new_user_list='new list',
                                  template='Activation', enroll_cost=100, lms_url_override='http://testserver'):
    """
    Enable / Disable Sailthru integration
    """
    return EmailMarketingConfiguration.objects.create(
        enabled=enabled,
        sailthru_key=key,
        sailthru_secret=secret,
        sailthru_new_user_list=new_user_list,
        sailthru_activation_template=template,
        sailthru_enroll_template='enroll_template',
        sailthru_lms_url_override=lms_url_override,
        sailthru_get_tags_from_sailthru=False,
        sailthru_enroll_cost=enroll_cost,
        sailthru_max_retries=0,
        welcome_email_send_delay=600
    )

@ddt.ddt
class EmailMarketingTests(TestCase):
    """
    Tests for the EmailMarketing signals and tasks classes.
    """

    def setUp(self):
        update_email_marketing_config(enabled=False)
        self.request_factory = RequestFactory()
        self.user = UserFactory.create(username='test', email=TEST_EMAIL)
        self.profile = self.user.profile
        self.profile.year_of_birth = 1980
        self.profile.save()

        self.request = self.request_factory.get("foo")
        update_email_marketing_config(enabled=True)

        # create some test course objects
        self.course_id_string = 'edX/toy/2012_Fall'
        self.course_id = CourseKey.from_string(self.course_id_string)
        self.course_url = 'http://testserver/courses/edX/toy/2012_Fall/info'

        self.site = Site.objects.get_current()
        self.site_domain = self.site.domain
        self.request.site = self.site
        super(EmailMarketingTests, self).setUp()

    @freeze_time(datetime.datetime.now())
    @patch('email_marketing.signals.crum.get_current_request')
    @patch('email_marketing.signals.SailthruClient.api_post')
    def test_drop_cookie(self, mock_sailthru, mock_get_current_request):
        """
        Test add_email_marketing_cookies
        """
        response = JsonResponse({
            "success": True,
            "redirect_url": 'test.com/test',
        })
        self.request.COOKIES['anonymous_interest'] = 'cookie_content'
        mock_get_current_request.return_value = self.request
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'keys': {'cookie': 'test_cookie'}}))
        cookie_log = "Sending to Sailthru the user interest cookie [{'anonymous_interest': 'cookie_content'}]" \
                     ' for user [test@edx.org]'

        with LogCapture(LOGGER_NAME, level=logging.INFO) as logger:
            add_email_marketing_cookies(None, response=response, user=self.user)
            logger.check(
                (LOGGER_NAME, 'INFO', cookie_log),
                (LOGGER_NAME, 'INFO',
                    'Started at {start} and ended at {end}, time spent:{delta} milliseconds'.format(
                        start=datetime.datetime.now().isoformat(' '),
                        end=datetime.datetime.now().isoformat(' '),
                        delta=0)
                 )
            )
        mock_sailthru.assert_called_with('user',
                                         {'fields': {'keys': 1},
                                          'cookies': {'anonymous_interest': 'cookie_content'},
                                          'id': TEST_EMAIL,
                                          'vars': {'last_login_date': ANY}})
        self.assertTrue('sailthru_hid' in response.cookies)
        self.assertEquals(response.cookies['sailthru_hid'].value, "test_cookie")

    @patch('email_marketing.signals.SailthruClient.api_post')
    def test_drop_cookie_error_path(self, mock_sailthru):
        """
        test that error paths return no cookie
        """
        response = JsonResponse({
            "success": True,
            "redirect_url": 'test.com/test',
        })
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'keys': {'cookiexx': 'test_cookie'}}))
        add_email_marketing_cookies(None, response=response, user=self.user)
        self.assertFalse('sailthru_hid' in response.cookies)

        mock_sailthru.return_value = SailthruResponse(JsonResponse({'error': "error", "errormsg": "errormsg"}))
        add_email_marketing_cookies(None, response=response, user=self.user)
        self.assertFalse('sailthru_hid' in response.cookies)

        mock_sailthru.side_effect = SailthruClientError
        add_email_marketing_cookies(None, response=response, user=self.user)
        self.assertFalse('sailthru_hid' in response.cookies)

    @patch('email_marketing.tasks.log.error')
    @patch('email_marketing.tasks.SailthruClient.api_post')
    @patch('email_marketing.tasks.SailthruClient.api_get')
    def test_add_user(self, mock_sailthru_get, mock_sailthru_post, mock_log_error):
        """
        test async method in tasks that actually updates Sailthru
        """
        site_dict = {'id': self.site.id, 'domain': self.site.domain, 'name': self.site.name}
        mock_sailthru_post.return_value = SailthruResponse(JsonResponse({'ok': True}))
        mock_sailthru_get.return_value = SailthruResponse(JsonResponse({'lists': [{'name': 'new list'}], 'ok': True}))
        update_user.delay(
            {'gender': 'm', 'username': 'test', 'activated': 1}, TEST_EMAIL, site_dict, new_user=True
        )
        self.assertFalse(mock_log_error.called)
        self.assertEquals(mock_sailthru_post.call_args[0][0], "user")
        userparms = mock_sailthru_post.call_args[0][1]
        self.assertEquals(userparms['key'], "email")
        self.assertEquals(userparms['id'], TEST_EMAIL)
        self.assertEquals(userparms['vars']['gender'], "m")
        self.assertEquals(userparms['vars']['username'], "test")
        self.assertEquals(userparms['vars']['activated'], 1)
        self.assertEquals(userparms['lists']['new list'], 1)

    @patch('email_marketing.tasks.SailthruClient.api_post')
    @patch('email_marketing.tasks.SailthruClient.api_get')
    def test_add_user_list_existing_domain(self, mock_sailthru_get, mock_sailthru_post):
        """
        test non existing domain name updates Sailthru user lists with default list
        """
        existing_site = Site.objects.create(domain='testing.com', name='testing.com')
        site_dict = {'id': existing_site.id, 'domain': existing_site.domain, 'name': existing_site.name}
        mock_sailthru_post.return_value = SailthruResponse(JsonResponse({'ok': True}))
        mock_sailthru_get.return_value = SailthruResponse(
            JsonResponse({'lists': [{'name': 'new list'}, {'name': 'testing_com_user_list'}], 'ok': True})
        )
        update_user.delay(
            {'gender': 'm', 'username': 'test', 'activated': 1}, TEST_EMAIL, site=site_dict, new_user=True
        )
        self.assertEquals(mock_sailthru_post.call_args[0][0], "user")
        userparms = mock_sailthru_post.call_args[0][1]
        self.assertEquals(userparms['lists']['testing_com_user_list'], 1)

    @patch('email_marketing.tasks.SailthruClient.api_post')
    @patch('email_marketing.tasks.SailthruClient.api_get')
    def test_user_activation(self, mock_sailthru_get, mock_sailthru_post):
        """
        test send of activation template
        """
        mock_sailthru_post.return_value = SailthruResponse(JsonResponse({'ok': True}))
        mock_sailthru_get.return_value = SailthruResponse(JsonResponse({'lists': [{'name': 'new list'}], 'ok': True}))
        expected_schedule = datetime.datetime.utcnow() + datetime.timedelta(seconds=600)
        update_user.delay({}, self.user.email, new_user=True, activation=True)
        # look for call args for 2nd call
        self.assertEquals(mock_sailthru_post.call_args[0][0], "send")
        userparms = mock_sailthru_post.call_args[0][1]
        self.assertEquals(userparms['email'], TEST_EMAIL)
        self.assertEquals(userparms['template'], "Activation")
        self.assertEquals(userparms['schedule_time'], expected_schedule.strftime('%Y-%m-%dT%H:%M:%SZ'))

    @patch('email_marketing.tasks.log.error')
    @patch('email_marketing.tasks.SailthruClient.api_post')
    def test_update_user_error_logging(self, mock_sailthru, mock_log_error):
        """
        Ensure that error returned from Sailthru api is logged
        """
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'error': 100, 'errormsg': 'Got an error'}))
        update_user.delay({}, self.user.email)
        self.assertTrue(mock_log_error.called)

        # force Sailthru API exception
        mock_log_error.reset_mock()
        mock_sailthru.side_effect = SailthruClientError
        update_user.delay({}, self.user.email, self.site_domain)
        self.assertTrue(mock_log_error.called)

        # force Sailthru API exception on 2nd call
        mock_log_error.reset_mock()
        mock_sailthru.side_effect = [SailthruResponse(JsonResponse({'ok': True})), SailthruClientError]
        update_user.delay({}, self.user.email, activation=True)
        self.assertTrue(mock_log_error.called)

        # force Sailthru API error return on 2nd call
        mock_log_error.reset_mock()
        mock_sailthru.side_effect = [SailthruResponse(JsonResponse({'ok': True})),
                                     SailthruResponse(JsonResponse({'error': 100, 'errormsg': 'Got an error'}))]
        update_user.delay({}, self.user.email, activation=True)
        self.assertTrue(mock_log_error.called)

    @patch('email_marketing.tasks.update_user.retry')
    @patch('email_marketing.tasks.log.error')
    @patch('email_marketing.tasks.SailthruClient.api_post')
    def test_update_user_error_retryable(self, mock_sailthru, mock_log_error, mock_retry):
        """
        Ensure that retryable error is retried
        """
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'error': 43, 'errormsg': 'Got an error'}))
        update_user.delay({}, self.user.email)
        self.assertTrue(mock_log_error.called)
        self.assertTrue(mock_retry.called)

    @patch('email_marketing.tasks.update_user.retry')
    @patch('email_marketing.tasks.log.error')
    @patch('email_marketing.tasks.SailthruClient.api_post')
    def test_update_user_error_nonretryable(self, mock_sailthru, mock_log_error, mock_retry):
        """
        Ensure that non-retryable error is not retried
        """
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'error': 1, 'errormsg': 'Got an error'}))
        update_user.delay({}, self.user.email)
        self.assertTrue(mock_log_error.called)
        self.assertFalse(mock_retry.called)

    @patch('email_marketing.tasks.log.error')
    @patch('email_marketing.tasks.SailthruClient.api_post')
    def test_just_return_tasks(self, mock_sailthru, mock_log_error):
        """
        Ensure that disabling Sailthru just returns
        """
        update_email_marketing_config(enabled=False)

        update_user.delay(self.user.username)
        self.assertFalse(mock_log_error.called)
        self.assertFalse(mock_sailthru.called)

        update_user_email.delay(self.user.username, "newemail2@test.com")
        self.assertFalse(mock_log_error.called)
        self.assertFalse(mock_sailthru.called)

        update_email_marketing_config(enabled=True)

    @patch('email_marketing.signals.log.error')
    def test_just_return_signals(self, mock_log_error):
        """
        Ensure that disabling Sailthru just returns
        """
        update_email_marketing_config(enabled=False)

        add_email_marketing_cookies(None)
        self.assertFalse(mock_log_error.called)

        email_marketing_register_user(None)
        self.assertFalse(mock_log_error.called)

        update_email_marketing_config(enabled=True)

        # test anonymous users
        anon = AnonymousUser()
        email_marketing_register_user(None, user=anon)
        self.assertFalse(mock_log_error.called)

        email_marketing_user_field_changed(None, user=anon)
        self.assertFalse(mock_log_error.called)

    @patch('email_marketing.tasks.SailthruClient.api_post')
    def test_change_email(self, mock_sailthru):
        """
        test async method in task that changes email in Sailthru
        """
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'ok': True}))
        update_user_email.delay(TEST_EMAIL, "old@edx.org")
        self.assertEquals(mock_sailthru.call_args[0][0], "user")
        userparms = mock_sailthru.call_args[0][1]
        self.assertEquals(userparms['key'], "email")
        self.assertEquals(userparms['id'], "old@edx.org")
        self.assertEquals(userparms['keys']['email'], TEST_EMAIL)

    @patch('email_marketing.tasks.SailthruClient')
    def test_get_or_create_sailthru_list(self, mock_sailthru_client):
        """
        Test the task the create sailthru lists.
        """
        mock_sailthru_client.api_get.return_value = SailthruResponse(JsonResponse({'lists': []}))
        _get_or_create_user_list(mock_sailthru_client, 'test1_user_list')
        mock_sailthru_client.api_get.assert_called_with("list", {})
        mock_sailthru_client.api_post.assert_called_with(
            "list", {'list': 'test1_user_list', 'primary': 0, 'public_name': 'test1_user_list'}
        )

        # test existing user list
        mock_sailthru_client.api_get.return_value = \
            SailthruResponse(JsonResponse({'lists': [{'name': 'test1_user_list'}]}))
        _get_or_create_user_list(mock_sailthru_client, 'test2_user_list')
        mock_sailthru_client.api_get.assert_called_with("list", {})
        mock_sailthru_client.api_post.assert_called_with(
            "list", {'list': 'test2_user_list', 'primary': 0, 'public_name': 'test2_user_list'}
        )

        # test get error from Sailthru
        mock_sailthru_client.api_get.return_value = \
            SailthruResponse(JsonResponse({'error': 43, 'errormsg': 'Got an error'}))
        self.assertEqual(_get_or_create_user_list(
            mock_sailthru_client, 'test1_user_list'), None
        )

        # test post error from Sailthru
        mock_sailthru_client.api_post.return_value = \
            SailthruResponse(JsonResponse({'error': 43, 'errormsg': 'Got an error'}))
        mock_sailthru_client.api_get.return_value = SailthruResponse(JsonResponse({'lists': []}))
        self.assertEqual(_get_or_create_user_list(mock_sailthru_client, 'test2_user_list'), None)

    @patch('email_marketing.tasks.SailthruClient')
    def test_get_sailthru_list_map_no_list(self, mock_sailthru_client):
        """Test when no list returned from sailthru"""
        mock_sailthru_client.api_get.return_value = SailthruResponse(JsonResponse({'lists': []}))
        self.assertEqual(_get_list_from_email_marketing_provider(mock_sailthru_client), {})
        mock_sailthru_client.api_get.assert_called_with("list", {})

    @patch('email_marketing.tasks.SailthruClient')
    def test_get_sailthru_list_map_error(self, mock_sailthru_client):
        """Test when error occurred while fetching data from sailthru"""
        mock_sailthru_client.api_get.return_value = SailthruResponse(
            JsonResponse({'error': 43, 'errormsg': 'Got an error'})
        )
        self.assertEqual(_get_list_from_email_marketing_provider(mock_sailthru_client), {})

    @patch('email_marketing.tasks.SailthruClient')
    def test_get_sailthru_list_map_exception(self, mock_sailthru_client):
        """Test when exception raised while fetching data from sailthru"""
        mock_sailthru_client.api_get.side_effect = SailthruClientError
        self.assertEqual(_get_list_from_email_marketing_provider(mock_sailthru_client), {})

    @patch('email_marketing.tasks.SailthruClient')
    def test_get_sailthru_list(self, mock_sailthru_client):
        """Test fetch list data from sailthru"""
        mock_sailthru_client.api_get.return_value = \
            SailthruResponse(JsonResponse({'lists': [{'name': 'test1_user_list'}]}))
        self.assertEqual(
            _get_list_from_email_marketing_provider(mock_sailthru_client),
            {'test1_user_list': {'name': 'test1_user_list'}}
        )
        mock_sailthru_client.api_get.assert_called_with("list", {})

    @patch('email_marketing.tasks.SailthruClient')
    def test_create_sailthru_list(self, mock_sailthru_client):
        """Test create list in sailthru"""
        mock_sailthru_client.api_post.return_value = SailthruResponse(JsonResponse({'ok': True}))
        self.assertEqual(_create_user_list(mock_sailthru_client, 'test_list_name'), True)
        self.assertEquals(mock_sailthru_client.api_post.call_args[0][0], "list")
        listparms = mock_sailthru_client.api_post.call_args[0][1]
        self.assertEqual(listparms['list'], 'test_list_name')
        self.assertEqual(listparms['primary'], 0)
        self.assertEqual(listparms['public_name'], 'test_list_name')

    @patch('email_marketing.tasks.SailthruClient')
    def test_create_sailthru_list_error(self, mock_sailthru_client):
        """Test error occurrence while creating sailthru list"""
        mock_sailthru_client.api_post.return_value = SailthruResponse(
            JsonResponse({'error': 43, 'errormsg': 'Got an error'})
        )
        self.assertEqual(_create_user_list(mock_sailthru_client, 'test_list_name'), False)

    @patch('email_marketing.tasks.SailthruClient')
    def test_create_sailthru_list_exception(self, mock_sailthru_client):
        """Test exception raised while creating sailthru list"""
        mock_sailthru_client.api_post.side_effect = SailthruClientError
        self.assertEqual(_create_user_list(mock_sailthru_client, 'test_list_name'), False)

    @patch('email_marketing.tasks.log.error')
    @patch('email_marketing.tasks.SailthruClient.api_post')
    def test_error_logging(self, mock_sailthru, mock_log_error):
        """
        Ensure that error returned from Sailthru api is logged
        """
        mock_sailthru.return_value = SailthruResponse(JsonResponse({'error': 100, 'errormsg': 'Got an error'}))
        update_user_email.delay(self.user.username, "newemail2@test.com")
        self.assertTrue(mock_log_error.called)

        mock_sailthru.side_effect = SailthruClientError
        update_user_email.delay(self.user.username, "newemail2@test.com")
        self.assertTrue(mock_log_error.called)

    @patch('email_marketing.signals.crum.get_current_request')
    @patch('lms.djangoapps.email_marketing.tasks.update_user.delay')
    def test_register_user(self, mock_update_user, mock_get_current_request):
        """
        make sure register user call invokes update_user
        """
        mock_get_current_request.return_value = self.request
        email_marketing_register_user(None, user=self.user, profile=self.profile)
        self.assertTrue(mock_update_user.called)

    @patch('lms.djangoapps.email_marketing.tasks.update_user.delay')
    def test_register_user_no_request(self, mock_update_user):
        """
        make sure register user call invokes update_user
        """
        email_marketing_register_user(None, user=self.user, profile=self.profile)
        self.assertTrue(mock_update_user.called)

    @patch('email_marketing.signals.crum.get_current_request')
    @patch('lms.djangoapps.email_marketing.tasks.update_user.delay')
    @ddt.data(('auth_userprofile', 'gender', 'f', True),
              ('auth_user', 'is_active', 1, True),
              ('auth_userprofile', 'shoe_size', 1, False))
    @ddt.unpack
    def test_modify_field(self, table, setting, value, result, mock_update_user, mock_get_current_request):
        """
        Test that correct fields call update_user
        """
        mock_get_current_request.return_value = self.request
        email_marketing_user_field_changed(None, self.user, table=table, setting=setting, new_value=value)
        self.assertEqual(mock_update_user.called, result)

    @patch('lms.djangoapps.email_marketing.tasks.update_user_email.delay')
    def test_modify_email(self, mock_update_user):
        """
        Test that change to email calls update_user_email
        """
        email_marketing_user_field_changed(None, self.user, table='auth_user', setting='email', old_value='new@a.com')
        mock_update_user.assert_called_with(self.user.email, 'new@a.com')

        # make sure nothing called if disabled
        mock_update_user.reset_mock()
        update_email_marketing_config(enabled=False)
        email_marketing_user_field_changed(None, self.user, table='auth_user', setting='email', old_value='new@a.com')
        self.assertFalse(mock_update_user.called)


import requests
from bs4 import BeautifulSoup

def scrape_cyberpuerta():
    url = "https://www.cyberpuerta.mx/Computo-Hardware/Componentes/Tarjetas-de-Video/"
    response = requests.get(url)
    if response.status_code != 200:
        raise Exception("No se pudo acceder a la página.")

    soup = BeautifulSoup(response.text, 'html.parser')
    products = soup.find_all('a', class_='emproduct_right_title')

    data = []
    for product in products:
        name = product.text.strip()
        price = product.find_next('span', class_='price').text.strip()
        data.append((name, price))

    return data

if __name__ == "__main__":
    scraped_data = scrape_cyberpuerta()

    # Imprime la información en la terminal
    for name, price in scraped_data:
        print(f"Nombre: {name}")
        print(f"Precio: {price}")
        print("=" * 30)


import stanza
from pymongo import MongoClient
import pandas as pd
from player_name_comparator import PlayerNameComparator
from team_master_provider import TeamMasterProvider
from name_matcher import PlayerNameMatcher, CoachNameMatcher
from pos_tagging import PosTagger
from conferences_list import ConferencesList
from fuzzywuzzy import fuzz

mongo_url = 'mongodb://athlytetstdau:dQN*yR#jE6c1B@52.15.122.211:27017/?authSource=athlyte_test&authMechanism=SCRAM-SHA-1'
db_name = 'athlyte_test'
game_notes_collection_name = 'GameNotes'
game_schedules_collection_name = 'GameSchedules_New'

shared_mongo_url= 'localhost:27017'
shared_mongo_db= 'athlyte_new'
team_master_collection_name = 'TeamMaster'
team_games_collection_name = 'teamGames'

total_PERSON_tags = 0
total_COACH_tags = 0
total_PLAYER_tags = 0
total_PER_RESIDUALS = 0
total_RESIDUALS = 0
total_TEAM_tags = 0
total_CONF_tags = 0
total_OPPO_tags = 0
total_HOME_tags = 0
total_AWAY_tags = 0
total_NEUTRAL_tags = 0
total_STADIUM_tags = 0

game_notes_team_name = ''
game_notes_oppo_team_name = ''
game_notes_stadium = ''
game_notes_all_stadiums = []

class StanzaTagging(object):

    def __init__(self, team_code, season, sport_code, game_date):
        self.mongo_client = MongoClient(mongo_url)
        self.mongo_db = self.mongo_client[db_name]
        self.mongo_collection_gn = self.mongo_db[game_notes_collection_name]
        self.mongo_collection_gs = self.mongo_db[game_schedules_collection_name]
        self.team_code = team_code
        self.sport_code = sport_code
        self.game_date = game_date
        self.total_PERSON_tags = total_PERSON_tags
        self.total_COACH_tags = total_COACH_tags
        self.total_PLAYER_tags = total_PLAYER_tags
        self.total_PER_RESIDUALS = total_PER_RESIDUALS
        self.total_RESIDUALS = total_RESIDUALS
        self.total_TEAM_tags = total_TEAM_tags
        self.total_CONF_tags = total_CONF_tags
        self.total_OPPO_tags = total_OPPO_tags
        self.total_HOME_tags = total_HOME_tags
        self.total_AWAY_tags = total_AWAY_tags
        self.total_NEUTRAL_tags = total_NEUTRAL_tags
        self.total_STADIUM_tags = total_STADIUM_tags
        self.game_notes_team_name = game_notes_team_name
        self.game_notes_oppo_team_name = game_notes_oppo_team_name
        self.game_notes_stadium = game_notes_stadium
        self.game_notes_all_stadiums = game_notes_all_stadiums
        self.season = season

    # get the stories from GameNotes collection
    def get_game_note_stories(self):   #Reads stories from the Game notes
        game_notes = self.mongo_collection_gn.find(
            {'$and': [{'teamCode': self.team_code}, {'sportCode': self.sport_code}, {'gameDate': self.game_date}]})
        game_notes = pd.DataFrame(game_notes)
        game_notes_stories = game_notes.loc[0, 'notesText ']
        return game_notes_stories

    # get the team_name, oppo_name, and other details from GameNotes collection
    def get_game_notes_details(self):
        game_notes = self.mongo_collection_gn.find(
            {'$and': [{'teamCode': self.team_code}, {'sportCode': self.sport_code}, {'gameDate': self.game_date}]})
        game_notes = pd.DataFrame(game_notes)
        self.game_notes_team_name = game_notes.loc[0, 'teamName']
        self.game_notes_oppo_team_name = game_notes.loc[0, 'oppoTeamName']
        self.game_notes_stadium = game_notes.loc[0, 'stadium']
        game_notes_all = self.mongo_collection_gn.find({"stadium":{'$exists':True}})
        game_notes_all = pd.DataFrame(game_notes_all)
        for index in range(game_notes_all.shape[0]):
            stadium = game_notes_all.loc[index, 'stadium']
            self.game_notes_all_stadiums.append(stadium)

    # get the team_location, team_state, and other details from TeamMaster collection
    def get_team_master_details(self):
        mongo_client = MongoClient(shared_mongo_url)
        mongo_db = mongo_client[shared_mongo_db]
        mongo_collection_tm = mongo_db[team_master_collection_name]
        team_details = mongo_collection_tm.find({"teamName": self.game_notes_team_name})
        team_details = pd.DataFrame(team_details)
        team_loc = team_details.loc[0, 'teamLocation']
        team_state = team_details.loc[0, 'teamState']
        return team_loc, team_state

    # get the home and visitor team details from TeamGames collection
    def get_team_games_details(self):
        mongo_client = MongoClient(shared_mongo_url)
        mongo_db = mongo_client[shared_mongo_db]
        mongo_collection_tg = mongo_db[team_games_collection_name]

        team_games_details = mongo_collection_tg.find({"name": self.game_notes_team_name,"actualDate":self.game_date})
        team_details = pd.DataFrame(team_games_details)
        neutral_location= team_details.loc[0, 'neutralLocation']
        home_team = team_details.loc[0, 'homeTeam']
        home_team_name = home_team['name']
        visitor_team = team_details.loc[0, 'visitorTeam']
        visitor_team_name = visitor_team['name']
        return neutral_location, home_team_name, visitor_team_name

    def remove_whitespace(self, text):
        return " ".join(text.split())

    # Performs some basic clean up on the stories
    def clean_game_note_story(self, short_story):
        short_story = short_story.replace('-', ' ')
        short_story = short_story.replace("'", '')
        short_story = short_story.replace('.,', ' ,')
        short_story = short_story.replace('(', ' ')
        short_story = short_story.replace(')', ' ')
        short_story = short_story.replace('fi ', 'fi')
        short_story = StanzaTagging.remove_whitespace(self, short_story)
        return short_story

    # count all persons tags in the stories
    def count_person_tags(self, tagged_df):
        words = tagged_df.shape[0]
        for word in range(words):
            if tagged_df.loc[word, 'Tags'] in (':S-PERSON', ':B-PERSON'):
                self.total_PERSON_tags = self.total_PERSON_tags + 1
        return tagged_df

    # Remove additional tags from the stories
    def clean_unwanted_tags(self, tagged_df):
        words = tagged_df.shape[0]
        for word in range(words):
            if tagged_df.loc[word, 'Tags'] == ':O':
                tagged_df.loc[word, 'Tags'] = ''

            if tagged_df.loc[word, 'Tags'] in (
                    ':B-CARDINAL', ':I-CARDINAL', ':E-CARDINAL', ':S-CARDINAL', ':B-EVENT', ':I-EVENT', ':E-EVENT',
                    ':S-DATE', ':B-DATE', ':I-DATE', ':E-DATE', ':S-TIME',':B-TIME',':E-TIME', ':S-ORDINAL',':B-QUANTITY',':E-QUANTITY',':I-QUANTITY',':S-QUANTITY'):
                tagged_df.loc[word, 'Tags'] = ''
        return tagged_df

    # tag single word PERSON tag to COACH/PLAYER
    def tag_single_word_person(self, tag_name, tagged_df, persontype):
        tags = []
        mapper = PlayerNameComparator()
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':S-PERSON':
                    matched_name = mapper.compare_and_find_best_match(tagged_df.loc[word, 'Words'], persontype, 'NA', 'NA')
                    if matched_name:
                        tagged_df.loc[word, 'Tags'] = tag_name
                        tags.append(tagged_df.loc[word, 'Words'])
                        if tag_name == ':COACH':
                            self.total_COACH_tags+=1
                        else:
                            self.total_PLAYER_tags+=1
            except:
                pass
        return tagged_df, tags

    # tag multi word PERSON tags to COACH/PLAYER
    def tag_multi_word_person(self, tag_name, tagged_df, persontype):
        tags = []
        mapper = PlayerNameComparator()
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':B-PERSON':
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == ':E-PERSON':
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    matched_coach_name = mapper.compare_and_find_best_match(word_seq, persontype, 'NA', 'NA')
                    if matched_coach_name:
                        for i in range(count - 1):
                            tagged_df.loc[word + i, 'Tags'] = ''
                        tagged_df.loc[word + count - 1, 'Tags'] = tag_name
                        tags.append(word_seq)
                        if tag_name == ':COACH':
                            self.total_COACH_tags += 1
                        else:
                            self.total_PLAYER_tags += 1
            except:
                pass
        return tagged_df, tags

    # custom tag the PERSON tags based on Single word or Multiple Words as COACH/PLAYER/RESIDUAL
    def change_person_tags(self, tagged_df):

        player_matcher = PlayerNameMatcher(self.team_code, self.season, self.sport_code)
        coach_matcher = CoachNameMatcher(self.team_code, self.season, self.sport_code, self.game_date)

        activerosters = player_matcher._get_active_players()
        allrosters = player_matcher._get_all_players()
        activecoach = coach_matcher._get_coach()

        tagged_df, coach_tags_s = StanzaTagging.tag_single_word_person(self, ':COACH', tagged_df, activecoach)
        tagged_df, player_ar_s = StanzaTagging.tag_single_word_person(self, ':PLAYER', tagged_df, activerosters)
        tagged_df, player_allr_s = StanzaTagging.tag_single_word_person(self, ':PLAYER', tagged_df, allrosters)
        player_tags_s = player_ar_s + player_allr_s

        tagged_df, coach_tags_m = StanzaTagging.tag_multi_word_person(self, ':COACH', tagged_df, activecoach)
        tagged_df, player_ar_m = StanzaTagging.tag_multi_word_person(self, ':PLAYER', tagged_df, activerosters)
        tagged_df, player_allr_m = StanzaTagging.tag_multi_word_person(self, ':PLAYER', tagged_df, allrosters)
        player_tags_m = player_ar_m + player_allr_m

        coach_tags = coach_tags_s + coach_tags_m
        player_tags = player_tags_s + player_tags_m

        words = tagged_df.shape[0]
        for word in range(words):
            if tagged_df.loc[word, 'Tags'] == ':S-PERSON':
                tagged_df.loc[word, 'Tags'] = ':SP-RESIDUAL'
                self.total_PER_RESIDUALS = self.total_PER_RESIDUALS + 1

            if tagged_df.loc[word, 'Tags'] == ':B-PERSON':
                i = 1
                while i < 10:
                    if (tagged_df.loc[(word + i), 'Tags']) == ':E-PERSON':
                        count = i + 1
                        break;
                    i = i + 1
                tagged_df.loc[word, 'Tags'] = ':BP-RESIDUAL'
                for i in range(count - 1):
                    tagged_df.loc[word + i + 1, 'Tags'] = ':IP-RESIDUAL'
                tagged_df.loc[word + count - 1, 'Tags'] = ':EP-RESIDUAL'
                self.total_PER_RESIDUALS = self.total_PER_RESIDUALS + 1

        return tagged_df, coach_tags, player_tags

    # Change single word ORG tags to CONFERENCE
    def tag_single_word_org_conf(self, tagged_df):
        conf_tags_s = []
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':S-ORG':  # single word conference
                    org = tagged_df.loc[word, 'Words']
                    conference_dict = ConferencesList.get_conference_dict(self)
                    if org in conference_dict:
                        tagged_df.loc[word, 'Tags'] = ":CONFERENCE"
                        conf_tags_s.append(tagged_df.loc[word, 'Words'])
                        self.total_CONF_tags = self.total_CONF_tags + 1
            except:
                pass
        return tagged_df, conf_tags_s

    # Change multi word ORG tags to CONFERENCE
    def tag_multi_word_org_conf(self, tagged_df):
        conf_tags_m = []
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':B-ORG':  # Conference multiple words
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == ':E-ORG':
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        if (tagged_df.loc[(word + i), 'Words']) in ('The', 'the'):
                            continue
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    conference_dict = ConferencesList.get_conference_dict(self)
                    if word_seq in conference_dict:
                        for i in range(count - 1):
                            tagged_df.loc[word + i, 'Tags'] = ''
                        tagged_df.loc[word + count - 1, 'Tags'] = ':CONFERENCE'
                        conf_tags_m.append(word_seq)
                        self.total_CONF_tags = self.total_CONF_tags + 1
            except:
                pass
        return tagged_df, conf_tags_m

    # Change multi word ORG tags to HOME/AWAY/NEUTRAL
    def tag_multi_word_org(self, tagged_df):
        home_org_tags = []
        away_org_tags = []
        neutral_org_tags = []
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':B-ORG':
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == ':E-ORG':
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        if (tagged_df.loc[(word + i), 'Words']) in ('The', 'the'):
                            continue
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    neutral_location, home_team_name, visitor_team_name = StanzaTagging.get_team_games_details(self)
                    if word_seq == self.game_notes_stadium:
                        if home_team_name == self.game_notes_team_name:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':HOME'
                            home_org_tags.append(home_team_name)
                            self.total_HOME_tags = self.total_HOME_tags + 1
                        if visitor_team_name == self.game_notes_team_name:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':AWAY'
                            away_org_tags.append(visitor_team_name)
                            self.total_AWAY_tags = self.total_AWAY_tags + 1
                        if neutral_location == True:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':NEUTRAL'
                            neutral_org_tags.append(word_seq)
                            self.total_NEUTRAL_tags = self.total_NEUTRAL_tags + 1
            except:
                pass
        return tagged_df, home_org_tags, away_org_tags, neutral_org_tags

    # Change single word ORG tags to TEAM/OPPONENT
    def tag_single_word_org_team_oppo(self, tagged_df, tag_type):
        team_org_tags = []
        oppo_org_tags = []
        team_master_provider = TeamMasterProvider(
            shared_mongo_url,
            shared_mongo_db,
            team_master_collection_name,
            logger='NA')
        words = tagged_df.shape[0]
        for word in range(words):
            if tagged_df.loc[word, 'Tags'] == tag_type:
                try:
                    org = tagged_df.loc[word, 'Words']
                    team_details = team_master_provider.get_team_data_from_master(
                        team_name=org,
                        team_code=None,
                        team_id=None)
                    team_name_of_org = team_details['teamName']
                    if team_name_of_org == self.game_notes_team_name:
                        tagged_df.loc[word, 'Tags'] = ":TEAM"
                        team_org_tags.append(tagged_df.loc[word, 'Words'])
                        self.total_TEAM_tags = self.total_TEAM_tags + 1
                    else:
                        tagged_df.loc[word, 'Tags'] = ":OPPONENT"
                        oppo_org_tags.append(tagged_df.loc[word, 'Words'])
                        self.total_OPPO_tags = self.total_OPPO_tags + 1

                except Exception:
                    tagged_df.loc[word, 'Tags'] = ":S-RESIDUAL"
                    self.total_RESIDUALS = self.total_RESIDUALS + 1

        return tagged_df, team_org_tags, oppo_org_tags

    # Change multi word ORG tags to TEAM/OPPONENT
    def tag_multi_word_org_team_oppo(self, tagged_df, b_tag_type, e_tag_type):
        team_org_tags = []
        oppo_org_tags = []
        team_master_provider = TeamMasterProvider(
            shared_mongo_url,
            shared_mongo_db,
            team_master_collection_name,
            logger='NA')
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == b_tag_type:
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == e_tag_type:
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        if (tagged_df.loc[(word + i), 'Words']) in ('The', 'the'):
                            continue
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    try:
                        team_details = team_master_provider.get_team_data_from_master(team_name=word_seq, team_code=None, team_id=None)
                        team_name_of_org = team_details['teamName']
                        if team_name_of_org == self.game_notes_team_name:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':TEAM'
                            team_org_tags.append(team_name_of_org)
                            self.total_TEAM_tags = self.total_TEAM_tags + 1
                        else:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':OPPONENT'
                            oppo_org_tags.append(team_name_of_org)
                            self.total_OPPO_tags = self.total_OPPO_tags + 1

                    except Exception:
                        tagged_df.loc[word, 'Tags'] = ':B-RESIDUAL'
                        for i in range(count - 1):
                            tagged_df.loc[word + i + 1, 'Tags'] = ':I-RESIDUAL'
                        tagged_df.loc[word + count - 1, 'Tags'] = ':E-RESIDUAL'
                        self.total_RESIDUALS = self.total_RESIDUALS + 1
            except:
                pass
        return tagged_df, team_org_tags, oppo_org_tags

    # Change all the ORG tags to CONFERENCE/TEAM/OPPONENT/HOME/AWAY/NEUTRAL/RESIDUALS
    def change_org_tags(self, tagged_df):

        tagged_df, conf_tags_s = StanzaTagging.tag_single_word_org_conf(self, tagged_df)
        tagged_df, conf_tags_m = StanzaTagging.tag_multi_word_org_conf(self, tagged_df)
        conf_tags = conf_tags_s + conf_tags_m

        tagged_df, home_org_tags, away_org_tags, neutral_org_tags = StanzaTagging.tag_multi_word_org(self, tagged_df)

        tagged_df, team_org_tags_o, oppo_org_tags_o = StanzaTagging.tag_single_word_org_team_oppo(self, tagged_df, ':S-ORG')
        tagged_df, team_org_tags_p, oppo_org_tags_p = StanzaTagging.tag_single_word_org_team_oppo(self, tagged_df, ':S-PRODUCT')
        tagged_df, team_org_tags_w, oppo_org_tags_w = StanzaTagging.tag_single_word_org_team_oppo(self, tagged_df, 'S-WORK_OF_ART')
        tagged_df, team_org_tags_pr, oppo_org_tags_pr = StanzaTagging.tag_single_word_org_team_oppo(self, tagged_df, ':SP-RESIDUAL')
        team_org_tags_s = team_org_tags_o + team_org_tags_p + team_org_tags_w + team_org_tags_pr
        oppo_org_tags_s = oppo_org_tags_o + oppo_org_tags_p + oppo_org_tags_w + oppo_org_tags_pr

        tagged_df, team_org_tags_o, oppo_org_tags_o = StanzaTagging.tag_multi_word_org_team_oppo(self, tagged_df, ':B-ORG', ':E-ORG')
        tagged_df, team_org_tags_p, oppo_org_tags_p = StanzaTagging.tag_multi_word_org_team_oppo(self, tagged_df, ':B-PRODUCT', ':E-PRODUCT')
        tagged_df, team_org_tags_w, oppo_org_tags_w = StanzaTagging.tag_multi_word_org_team_oppo(self, tagged_df, 'B-WORK_OF_ART', 'E-WORK_OF_ART')
        tagged_df, team_org_tags_pr, oppo_org_tags_pr = StanzaTagging.tag_multi_word_org_team_oppo(self, tagged_df, ':BP-RESIDUAL', ':EP-RESIDUAL')
        team_org_tags_m = team_org_tags_o + team_org_tags_p + team_org_tags_w + team_org_tags_pr
        oppo_org_tags_m = oppo_org_tags_o + oppo_org_tags_p + oppo_org_tags_w + oppo_org_tags_pr

        team_org_tags = team_org_tags_s + team_org_tags_m
        oppo_org_tags = oppo_org_tags_s + oppo_org_tags_m

        return tagged_df, conf_tags, team_org_tags, oppo_org_tags, home_org_tags, away_org_tags, neutral_org_tags

    # Change single word GPE tags to TEAM/OPPONENT
    def tag_single_word_gpe_team_oppo(self, tagged_df, tag_type):
        team_gpe_tags = []
        oppo_gpe_tags = []
        team_master_provider = TeamMasterProvider(
            shared_mongo_url,
            shared_mongo_db,
            team_master_collection_name,
            logger='NA')

        words = tagged_df.shape[0]
        for word in range(words):
            if tagged_df.loc[word, 'Tags'] == tag_type:
                location = tagged_df.loc[word, 'Words']
                team_loc, team_state = StanzaTagging.get_team_master_details(self)
                team_loc_dict = [team_loc, team_state]
                if location in team_loc_dict:
                    tagged_df.loc[word, 'Tags'] = ":TEAM"
                    team_gpe_tags.append(tagged_df.loc[word, 'Words'])
                    self.total_TEAM_tags = self.total_TEAM_tags + 1
                else:
                    try:
                        team_details = team_master_provider.get_team_data_from_master(
                            team_name=location,
                            team_code=None,
                            team_id=None)
                        if team_details:
                            tagged_df.loc[word, 'Tags'] = ":OPPONENT"
                            oppo_gpe_tags.append(tagged_df.loc[word, 'Words'])
                            self.total_OPPO_tags = self.total_OPPO_tags + 1
                    except Exception:
                        pass

        return tagged_df, team_gpe_tags, oppo_gpe_tags

    # Change multi word GPE tags to TEAM/OPPONENT
    def tag_multi_word_gpe_team_oppo(self, tagged_df, b_tag_type, e_tag_type):
        team_gpe_tags = []
        oppo_gpe_tags = []
        team_master_provider = TeamMasterProvider(
            shared_mongo_url,
            shared_mongo_db,
            team_master_collection_name,
            logger='NA')

        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == b_tag_type:
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == e_tag_type:
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        if (tagged_df.loc[(word + i), 'Words']) in ('The', 'the'):
                            continue
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    team_loc, team_state = StanzaTagging.get_team_master_details(self)
                    team_loc_dict = [team_loc, team_state]
                    if word_seq in team_loc_dict:
                        for i in range(count - 1):
                            tagged_df.loc[word + i, 'Tags'] = ''
                        tagged_df.loc[word + count - 1, 'Tags'] = ":TEAM"
                        team_gpe_tags.append(word_seq)
                        self.total_TEAM_tags = self.total_TEAM_tags + 1
                    else:
                        try:
                            team_details = team_master_provider.get_team_data_from_master(
                                team_name=word_seq,
                                team_code=None,
                                team_id=None)
                            if team_details:
                                for i in range(count - 1):
                                    tagged_df.loc[word + i, 'Tags'] = ''
                                tagged_df.loc[word + count - 1, 'Tags'] = ":OPPONENT"
                                oppo_gpe_tags.append(word_seq)
                                self.total_OPPO_tags = self.total_OPPO_tags + 1
                        except Exception:
                            pass
            except:
                pass

        return tagged_df, team_gpe_tags, oppo_gpe_tags

    # Change all the GPE tags to TEAM/OPPONENT/RESIDUALS
    def change_gpe_tags(self, tagged_df):

        tagged_df, team_gpe_tags_g, oppo_gpe_tags_g = StanzaTagging.tag_single_word_gpe_team_oppo(self, tagged_df, ':S-GPE')
        tagged_df, team_gpe_tags_l, oppo_gpe_tags_l = StanzaTagging.tag_single_word_gpe_team_oppo(self, tagged_df, ':S-LOC')
        tagged_df, team_gpe_tags_n, oppo_gpe_tags_n = StanzaTagging.tag_single_word_gpe_team_oppo(self, tagged_df, ':S-NORP')
        tagged_df, team_gpe_tags_r, oppo_gpe_tags_r = StanzaTagging.tag_single_word_gpe_team_oppo(self, tagged_df, ':S-RESIDUAL')
        team_gpe_tags_s = team_gpe_tags_g + team_gpe_tags_l + team_gpe_tags_n + team_gpe_tags_r
        oppo_gpe_tags_s = oppo_gpe_tags_g + oppo_gpe_tags_l + oppo_gpe_tags_n + oppo_gpe_tags_r

        tagged_df, team_gpe_tags_g, oppo_gpe_tags_g = StanzaTagging.tag_multi_word_gpe_team_oppo(self, tagged_df, ':B-GPE',':E-GPE' )
        tagged_df, team_gpe_tags_l, oppo_gpe_tags_l = StanzaTagging.tag_multi_word_gpe_team_oppo(self, tagged_df, ':B-LOC', ':E-LOC')
        tagged_df, team_gpe_tags_n, oppo_gpe_tags_n = StanzaTagging.tag_multi_word_gpe_team_oppo(self, tagged_df, ':B-NORP', ':E-NORP')
        tagged_df, team_gpe_tags_r, oppo_gpe_tags_r = StanzaTagging.tag_multi_word_gpe_team_oppo(self, tagged_df, ':B-RESIDUAL', ':E-RESIDUAL')
        team_gpe_tags_m = team_gpe_tags_g + team_gpe_tags_l + team_gpe_tags_n + team_gpe_tags_r
        oppo_gpe_tags_m = oppo_gpe_tags_g + oppo_gpe_tags_l + oppo_gpe_tags_n + oppo_gpe_tags_r

        team_gpe_tags = team_gpe_tags_s + team_gpe_tags_m
        oppo_gpe_tags = oppo_gpe_tags_s + oppo_gpe_tags_m

        words = tagged_df.shape[0]
        for word in range(words):

            try:
                if tagged_df.loc[word, 'Tags']  in (':S-GPE', ':S-LOC', 'S-NORP'):
                    tagged_df.loc[word, 'Tags'] = ':S-RESIDUAL'

                if tagged_df.loc[word, 'Tags'] in (':B-GPE', ':B-LOC', 'B-NORP'):
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) in (':E-GPE', ':E-LOC', 'E-NORP'):
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        tagged_df.loc[word, 'Tags'] = ':B-RESIDUAL'
                        for i in range(count - 1):
                            tagged_df.loc[word + i + 1, 'Tags'] = ':I-RESIDUAL'
                        tagged_df.loc[word + count - 1, 'Tags'] = ':E-RESIDUAL'
            except:
                pass

        return tagged_df, team_gpe_tags, oppo_gpe_tags

    # Change multi word fac(facility) tags to HOME/AWAY/NEUTRAL
    def tag_multi_word_fac(self, tagged_df):
        home_fac_tags = []
        away_fac_tags = []
        neutral_fac_tags = []

        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':B-FAC':
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == ':E-FAC':
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        if (tagged_df.loc[(word + i), 'Words']) in ('The', 'the'):
                            continue
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    neutral_location, home_team_name, visitor_team_name = StanzaTagging.get_team_games_details(self)
                    self.game_notes_stadium = self.game_notes_stadium.replace('-', ' ')
                    match = fuzz.WRatio(word_seq, self.game_notes_stadium)
                    if match >= 80:
                        if home_team_name == self.game_notes_team_name:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':HOME'
                            home_fac_tags.append(home_team_name)
                            self.total_HOME_tags = self.total_HOME_tags + 1
                        if visitor_team_name == self.game_notes_team_name:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':AWAY'
                            away_fac_tags.append(visitor_team_name)
                            self.total_AWAY_tags = self.total_AWAY_tags + 1
                        if neutral_location == True:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':NEUTRAL'
                            neutral_fac_tags.append(word_seq)
                            self.total_NEUTRAL_tags = self.total_NEUTRAL_tags + 1
            except:
                pass
        return tagged_df, home_fac_tags, away_fac_tags, neutral_fac_tags

    # Change multi word fac(facility) tags STADIUM
    def tag_multi_word_fac_stadium(self, tagged_df):
        stadium_tags = []
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':B-FAC':
                    word_seq = ''
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == ':E-FAC':
                            count = i + 1
                            break;
                        i = i + 1
                    for i in range(count):
                        if (tagged_df.loc[(word + i), 'Words']) in ('The', 'the'):
                            continue
                        word_seq = word_seq + tagged_df.loc[(word + i), 'Words'] + ' '
                    word_seq = word_seq.strip()
                    for stadium in self.game_notes_all_stadiums:
                        stadium = stadium.replace('-',' ')
                        stadium = stadium.replace('5TAUIUM', 'STADIUM')
                        stadium = stadium.replace('Stadium', '')
                        stadium = stadium.replace('stadium', '')
                        stadium = stadium.replace('STADIUM', '')
                        word_seq = word_seq.replace('Stadium', '')
                        word_seq = word_seq.replace('stadium', '')
                        word_seq = word_seq.replace('STADIUM', '')
                        match = fuzz.WRatio(word_seq, stadium)

                        if match >= 90:
                            for i in range(count - 1):
                                tagged_df.loc[word + i, 'Tags'] = ''
                            tagged_df.loc[word + count - 1, 'Tags'] = ':STADIUM'
                            stadium_tags.append(word_seq)
                            self.total_STADIUM_tags = self.total_STADIUM_tags + 1
            except:
                pass
        return tagged_df, stadium_tags

    # Change all the fac(facility) tags to HOME/AWAY/NEUTRAL/STADIUM/RESIDUALS
    def change_fac_tags(self, tagged_df):
        tagged_df, home_fac_tags, away_fac_tags, neutral_fac_tags = StanzaTagging.tag_multi_word_fac(self, tagged_df)
        tagged_df, stadium_tags = StanzaTagging.tag_multi_word_fac_stadium(self, tagged_df)
        words = tagged_df.shape[0]
        for word in range(words):
            try:
                if tagged_df.loc[word, 'Tags'] == ':B-FAC':
                    i = 1
                    while i < 10:
                        if (tagged_df.loc[(word + i), 'Tags']) == ':E-FAC':
                            count = i + 1
                            break;
                        i = i + 1
                    tagged_df.loc[word, 'Tags'] = ':B-RESIDUAL'
                    for i in range(count - 1):
                        tagged_df.loc[word + i + 1, 'Tags'] = ':I-RESIDUAL'
                    tagged_df.loc[word + count - 1, 'Tags'] = ':E-RESIDUAL'
                    self.total_RESIDUALS = self.total_RESIDUALS + 1
            except:
                pass
        return tagged_df, home_fac_tags, away_fac_tags, neutral_fac_tags, stadium_tags

    # combine NER and POS tagging for all the stories.
    def combine(nlp, tagged_story, nouns):
        doc = nlp(tagged_story)
        output_story = ""
        for sent in doc.sentences:
            for token in sent.words:
                output_story += token.text
                if PosTagger.is_noun(nouns, token.text):
                    output_story += ":NOUN"
                output_story += " "
        return output_story

    # convert all the Stanza tags to Athlyte tags
    def tag_story(self, nlp, short_story):
        tagged_df = pd.DataFrame(columns=['Words', 'Tags'])
        short_story = StanzaTagging.clean_game_note_story(self, short_story)
        doc = nlp(short_story)
        for sent in doc.sentences:
            for token in sent.tokens:
                tagged_df = tagged_df.append({'Words': token.text, 'Tags': ':' + str(token.ner)}, ignore_index=True)
                tagged_df = tagged_df.astype(str)

        tagged_df = StanzaTagging.count_person_tags(self, tagged_df)
        tagged_df = StanzaTagging.clean_unwanted_tags(self, tagged_df)
        tagged_df, coach_tags, player_tags = StanzaTagging.change_person_tags(self, tagged_df)
        tagged_df, conf_tags, team_org_tags, oppo_org_tags, home_org_tags, away_org_tags, neutral_org_tags = StanzaTagging.change_org_tags(self, tagged_df)
        tagged_df, team_gpe_tags, oppo_gpe_tags = StanzaTagging.change_gpe_tags(self, tagged_df)
        tagged_df, home_fac_tags, away_fac_tags, neutral_fac_tags, stadium_tags = StanzaTagging.change_fac_tags(self, tagged_df)

        team_tags = team_org_tags + team_gpe_tags
        oppo_tags = oppo_org_tags + oppo_gpe_tags
        home_tags = home_org_tags + home_fac_tags
        away_tags = away_org_tags + away_fac_tags
        neutral_tags = neutral_org_tags + neutral_fac_tags

        tagged_df["word n tags"] = tagged_df["Words"] + tagged_df["Tags"]
        tagged_story = ' '.join(tagged_df["word n tags"])
        return tagged_story, coach_tags, player_tags, conf_tags, team_tags, oppo_tags, home_tags, away_tags, neutral_tags, stadium_tags

    # Performs the custom tagging for all Athlyte tags
    def stanza_custom_tagging(self):
        nlp = stanza.Pipeline(lang='en', processors='tokenize, pos, ner')
        tagged_stories_df = pd.DataFrame(columns=['Original stories', 'Tagged stories','Coach','Players','Conference','Team','Opponent','Home','Away','Neutral','Stadium'])
        StanzaTagging.get_game_notes_details(self)
        game_stories = StanzaTagging.get_game_note_stories(self)
        stories = len(game_stories)

        for story in range(stories):
            short_story = game_stories[story]['storyContent']
            nouns = PosTagger.noun_tagger(self, nlp, short_story)
            tagged_story, coach_tags, player_tags, conf_tags, team_tags, oppo_tags, home_tags, away_tags, neutral_tags, stadium_tags  = self.tag_story(nlp, short_story)
            tagged_stories = StanzaTagging.combine(nlp, tagged_story, nouns)
            tagged_stories_df = tagged_stories_df.append({'Original stories': short_story,'Tagged stories':tagged_stories,'Coach':coach_tags,'Players':player_tags,
                                                          'Conference': conf_tags,'Team':team_tags,'Opponent':oppo_tags,'Home':home_tags,'Away':away_tags,'Neutral':neutral_tags,
                                                          'Stadium':stadium_tags}, ignore_index=True)

        print("Person", self.total_PERSON_tags)
        print("Player", self.total_PLAYER_tags)
        print("Coach", self.total_COACH_tags)
        print("PER Residuals", self.total_PER_RESIDUALS)
        print("Team", self.total_TEAM_tags)
        print("Conf", self.total_CONF_tags)
        print("Opponent", self.total_OPPO_tags)
        print("Home", self.total_HOME_tags)
        print("Away", self.total_AWAY_tags)
        print("Neutral", self.total_NEUTRAL_tags)
        print("Stadiums", self.total_STADIUM_tags)
        print("Residuals", self.total_RESIDUALS)
        file_name = 'NER_POS_TAGS_teamcode_'+ str(self.team_code)+'_season_'+ str(self.season)+str('.xlsx')
        tagged_stories_df.to_excel(file_name, sheet_name='Stories' )
        return tagged_stories


"""Module for handling automatic unbanishments."""
import asyncio
import datetime
from logging import Logger

from discord import Guild

from mrfreeze.bot import MrFreeze
from mrfreeze.cogs.coginfo import CogInfo
from mrfreeze.lib import default
from mrfreeze.lib.banish import mute_db
from mrfreeze.lib.colors import CYAN
from mrfreeze.lib.colors import CYAN_B
from mrfreeze.lib.colors import MAGENTA
from mrfreeze.lib.colors import RED
from mrfreeze.lib.colors import RESET
from mrfreeze.lib.colors import YELLOW

async def unbanish_loop(server: Guild, coginfo: CogInfo) -> None:
    """Check for people to unbanish every self.banish_interval seconds."""
    if coginfo.bot and coginfo.logger and coginfo.default_mute_interval:
        bot: MrFreeze = coginfo.bot
        logger: Logger = coginfo.logger
        default_mute_interval: int = coginfo.default_mute_interval
    else:
        raise Exception("Failed to create unbanish loop, insufficient cog info.")

    while not bot.is_closed():
        # Wait the time specified by the server (or the default time) before running
        interval = bot.settings.get_mute_interval(server) or default_mute_interval
        logger.debug(f"Running unbanish loop for {server.name} in {interval}.")
        await asyncio.sleep(interval)
        logger.debug(f"Running unbanish loop for {server.name}.")

        # Fetch mute role/channel, which might fail.
        try:
            mute_role = await bot.get_mute_role(server)
            mute_channel = await bot.get_mute_channel(server, silent=True)
        except Exception:
            logger.warning(f"{server.name} Unbanish loop failed to fetch mute role or channel.")
            continue
        unmuted = list()

        # Create a list of mutes which are due for unmuting
        current_time = datetime.datetime.now()
        mutes = mute_db.mdb_fetch(bot, server)
        timed_mutes = [ i for i in mutes if i.until is not None ]
        due_mutes = [ i for i in timed_mutes if i.until < current_time ]
        logger.debug(f"{server.name} mutes: {len(mutes)}/{len(timed_mutes)}/{len(due_mutes)}")

        for mute in due_mutes:
            logger.debug(f"{mute} is due for unbanish!")

            # Refresh member to make sure we have their latest roles.
            try:
                member = await server.fetch_member(mute.member.id)
                logger.debug(f"Refreshed {member}, they have {len(member.roles)} roles.")
            except Exception as e:
                logger.error(f"Failed to refresh muted member: {e}")
                continue  # Will try again next unbanish loop

            # Calculate how late we were in unbanishing
            diff = bot.parse_timedelta(current_time - mute.until)
            if diff == "":
                diff = "now"
            else:
                diff = f"{diff} ago"

            # Remove from database
            mute_db.mdb_del(bot, member, logger)

            if mute_role in member.roles:
                logger.debug(f"{member} has the mute role! Removing it.")
                try:
                    await member.remove_roles(mute_role)
                    logger.debug(f"{member} should no longer have the mute role.")
                    # Members are only considered unmuted if they had the antarctica role
                    unmuted.append(member)

                    log = f"Auto-unmuted {CYAN_B}{member.name}#"
                    log += f"{member.discriminator} @ {server.name}."
                    log += f"{YELLOW} (due {diff}){RESET}"
                    logger.info(log)

                except Exception as e:
                    log = f"Failed to remove mute role of {YELLOW}"
                    log += f"{member.name}#{member.discriminator}"
                    log += f"{CYAN_B} @ {MAGENTA} {server.name}:"
                    log += f"\n{RED}==> {RESET}{e}"
                    logger.error(log)
            else:
                log = f"User {YELLOW}{member.name}#{member.discriminator}"
                log += f"{CYAN_B} @ {MAGENTA} {server.name}{CYAN} "
                log += f"due for unmute but does not have a mute role!{RESET}"
                logger.warning(log)

        # Time for some great regrets
        if len(unmuted) > 0:
            unmuted_str = default.mentions_list(unmuted)

            if len(unmuted_str) == 1:
                msg = "It's with great regret that I must inform you all that "
                msg += f"{unmuted_str}'s exile has come to an end."
            else:
                msg = "It's with great regret that I must inform you all that the exile of "
                msg += f"{unmuted_str} has come to an end."

            await mute_channel.send(msg)


from fredapi import Fred
from api_etl.etl import ETL

class Fredapi(ETL):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.indices = ["PAYEMS", "FEDFUNDS", "UNRATE", "PPIACO", "CPIAUCSL",
                        "PCE", "INDPRO", "UMCSENT", "RSAFS", "HOUST", "CSUSHPINSA"],

    def fetch_macro_data(self):
        fred = Fred(api_key=self.api_keys["Fred"])
        dfs = []

        # Mapping of series IDs to their actual names
        names = {
            "PAYEMS": "NFP", # Non-Farm Payrolls
            "CPIAUCSL": "CPI", # Consumer Price Index
            "FEDFUNDS": "InterestRate", # Effective Federal Funds Rate
            "UNRATE": "UnemploymentRate",
            "PPIACO": "PPI", # Producer Price Index
            "PCE": "PCE", # Personal Consumption Expenditures
            "INDPRO": "IPI", # Industrial Production Index
            "UMCSENT": "ConsumerSentiment",
            "RSAFS": "RetailSales",
            "HOUST": "HousingStarts",
            "CSUSHPINSA": "HPI", # S&P/Case-Shiller U.S. National Home Price Index
        }

        for index in self.indices[0]: # Note: self.indices is a tuple with a single list
            series = fred.get_series(index, self.start_day, self.end_day)
            series = series.resample("D").ffill().reset_index()
            series.columns = ["date", index]
            dfs.append(series)

        # Merge all dataframes on 'date'
        macro_df = dfs[0]
        for df in dfs[1:]:
            macro_df = macro_df.merge(df, on='date', how='outer')

        # Rename the columns
        macro_df.rename(columns=names, inplace=True)

        macro_df = macro_df.round(4)
        macro_df.to_csv('../data/macro.csv', index=False)
        print('macro.csv created')
        return macro_df


'''
데이터 저장
파이썬 - Oracle DB 연동
라이브러리 설치
    - cs-Oracle (pycharm)   cx-oracle(pop install cx_oracle)
    - oracle instant client 다운
'''
import cx_Oracle
import datetime

# 오라클 인스턴트 클라이언트
cx_Oracle.init_oracle_client(lib_dir=r"E:\oracle\instantclient_19_16")
'''
cx_Oracle.connect   : 오라클 커넥션 접속  ('username', 'password', 'url:port/sid')
.cursor             : 쿼리 실행, 결과 데이터 다올 변수 선언
. execute           : SQL 실행, 결과가 cursor에 담김
.fetall             : 쿼리 실행 결과에서 데이터를 한 행씩 fetch 한다.
.description        : 데이터의 컬럼명 추출
'''
conn = cx_Oracle.connect("java06", "java06", "192.168.100.250:1521/orcl")
cursor = conn.cursor()

'''
# 데이터 조회
sql = "select * from test"
cursor.execute(sql) # 쿼리문 실행

for result in cursor:
    # print(result)
    print(result[0])
    print(result[1])
    print(result[2])
    print(result[3])
    print("=" * 50)

# 사용 후 닫기
cursor.close()
conn.close()

# 데이터 저장
import datetime

sql = "insert into test values(:1, :2, :3, :4)"
data = ('py01', '0000', 20, datetime.datetime.now()) # 바인딩시킬 데이터 튜플형태로 만들고
cursor.execute(sql, data)

cursor.close()
conn.commit()
conn.close()

# 데이터 여러개 저장
sql = "insert into test values(:1, :2, :3, :4)"
data = [
    ('py02', '0000', 20, datetime.datetime.now()),
    ('py03', '1111', 30, datetime.datetime.now()),
    ('py04', '2222', 40, datetime.datetime.now()),
    ('py05', '3333', 50, datetime.datetime.now()),
]
cursor.arraysize = len(data)
cursor.executemany(sql, data) # 한번에 여러 레코드 insert

cursor.close()
conn.commit()
conn.close()

# 레코드 개수 수정
sql = "select count(*) from test"
cursor.execute(sql)
count = cursor.fetchone()   # 레코드 한개 추출
print("회원 수 : ", count[0], "명")
cursor.close()
conn.close()

# 데이터 수정
sql = "update test set pw=:1, age=:2 where id=:3"
data = ('9876', 95, 'py01')
cursor.execute(sql, data)

cursor.close()
conn.commit()
conn.close()
'''

# 레코드 삭제
sql = "delete from test where id=:1"
data = ('py01',) # 튜플 형태로 데이터 생성
cursor.execute(sql, data)

cursor.close()
conn.commit()
conn.close()


#!/usr/bin/env python
import soco
from soco.music_services import MusicService
import sys

speaker_name = 'Woonkamer'

def find_speaker(speaker_name):
    speakers = soco.discover()
    for speaker in speakers:
        if speaker.player_name == speaker_name:
            return speaker
    if not speakers:
        sys.exit()

def get_play_url(station_name):
    stations = speaker.get_sonos_favorites()
    for station in stations['favorites']:
        if(str(station_name) in station['title']):
            return str(station['uri'])

if len(sys.argv) <= 1:
    print('No radio Selected')
    print('please add "bbc1" or "radio2" as parameter')
    sys.exit()
else:
    style = sys.argv[1]
    if (str(style) == 'bbc1'):
        station_name = 'BBC Radio 1'
    if (str(style) == 'radio2'):
        station_name = 'NPO Radio 2'

speaker = find_speaker(speaker_name)
play_url = str(get_play_url(station_name))
speaker.play_uri(uri=play_url,title=station_name)


import logging
import argparse
from herobrain import Herobrain
from herobrain import localisation
import sys

# Get the token as input
parser = argparse.ArgumentParser(description="Herobrain, a quiz prediction processor")
parser.add_argument("-t", "--token", dest="token", default="", help="HQTrivia bearer token")
parser.add_argument("-s", "--output", dest="output", default="http://localhost:1029", help="HQhero server")
parser.add_argument("-l", "--locale",
                    dest="locale",
                    default=localisation.ENGLISH_UK,
                    choices=[localisation.ENGLISH_UK,
                             localisation.ENGLISH_US,
                             localisation.GERMANY],
                    help="Configures methods of question analysis")
parser.add_argument("--test", action="store_true", dest="test", help="Run in test mode, doesn't require bearer token")
parser.add_argument("--quiz-api", dest="quiz_api", default="https://api-quiz.hype.space", help="HQTrivia quiz-api")
parser.add_argument("--test-api", dest="test_api", default="http://localhost:8732", help="Simulated quiz-api, requires --test")
parser.add_argument("--log-level", dest="log_level", default="info", choices=["critical", "error", "warning", "info", "debug"])

args = parser.parse_args()

if not args.test and not args.token:
    sys.exit("Token is required when not in test mode")

# Set the localisation
localisation.set_as(args.locale)

# Set up logging
logging.basicConfig(level=args.log_level.upper())
logging.getLogger('websockets').setLevel(logging.ERROR)

service = Herobrain(args.token, args.output, args.test_api if args.test else args.quiz_api)
service.run()


# Sprite class

import pygame as pg

# Custom files
from settings import *

# Function assignments
vec = pg.math.Vector2

class Player(pg.sprite.Sprite):
	def __init__(self):
		pg.sprite.Sprite.__init__(self)
		self.image = pg.Surface((30, 40))
		self.image.fill(BLUE)
		self.rect = self.image.get_rect()
		self.rect.center = (WIDTH/2, HEIGHT/2)

		self.pos = vec(WIDTH/2,HEIGHT/2)
		self.vel = vec(0,0)
		self.acc = vec(0,0)

	def update(self):
		# Initialize acceleration
		self.acc = vec(0,0)

		keys = pg.key.get_pressed()
		if keys[pg.K_LEFT]:
			self.acc.x = -PLAYER_ACC
		if keys[pg.K_RIGHT]:
			self.acc.x = PLAYER_ACC
		if keys[pg.K_UP]:
			self.acc.y = -PLAYER_ACC
		if keys[pg.K_DOWN]:
			self.acc.y = PLAYER_ACC

		self.acc.x += self.vel.x * PLAYER_FRICTION # Friction for x
		self.acc.y += self.vel.y * PLAYER_FRICTION # Friction for y
		self.vel += self.acc # Acceleration adds to velocity
		self.pos += self.vel + 0.5 * self.acc

		self.rect.center = self.pos

		# Collision with wall - Wrap around
		#if self.pos.x > WIDTH:
		#	self.pos.x = 0
		#if self.pos.x < 0:
		#	self.pos.x = WIDTH
		#if self.pos.y > HEIGHT:
		#	self.pos.y = 0
		#if self.pos.y < 0:
		#	self.pos.y = HEIGHT

		# Collision with wall - Stop
		if self.pos.x > WIDTH:
			self.pos.x = WIDTH
		if self.pos.x < 0:
			self.pos.x = 0
		if self.pos.y > HEIGHT:
			self.pos.y = HEIGHT
		if self.pos.y < 0:
			self.pos.y = 0

class Platform(pg.sprite.Sprite):
	def __init__(self, x, y, w, h):
		pg.sprite.Sprite.__init__(self)
		self.image = pg.Surface((w,h))
		self.image.fill(GREEN)
		self.rect = self.image.get_rect()
		self.rect.x = x
		self.rect.y = y


from pprint import pprint

import requests
from django.contrib.auth import login, logout, authenticate
from django.shortcuts import render, redirect

from facebook import settings

def login_fbv(request):
    # print('login_fbv')
    facebook_app_id = settings.config['facebook']['app_id']
    context = {
        'facebook_app_id': facebook_app_id,
    }
    return render(request, 'member/login.html', context)

def logout_fbv(request):
    logout(request)
    return redirect('index')

def login_facebook(request):
    # print(request.GET)
    APP_ID = settings.config['facebook']['app_id']
    SECRET_CODE = settings.config['facebook']['secret_code']
    REDIRECT_URI = 'http://localhost:8000/member/login/facebook/'
    APP_ACCESS_TOKEN = '{app_id}|{secret_code}'.format(
        app_id=APP_ID,
        secret_code=SECRET_CODE,
    )

    # login_fbv에서 페이스북 로그인으로 이동 후,
    # 정상적인 로그인 후 (정상적으로 로그인 시 request.GET에 'code' parameter가 추가됨)
    # redirect_uri를 이용해 다시 login_facebook으로 돌아온 후의 동작
    if request.GET.get('code'):
        # Step 1 : 사용자가 로그인 했다라는 검증
        code = request.GET.get('code')

        # Step 2 : user_access_token(사용자 액세스 토큰) 얻어오기
        # 전달받은 code 값을 이용해서 access_token값을 요청함
        # parameter들을 하기처럼 url에 직접 넣지 말 것.
        # url_request_access_token = 'https://graph.facebook.com/v2.8/oauth/access_token?'\
        #     'client_id={app_id}'\
        #     '&redirect_uri={redirect_uri}'\
        #     '&client_secret={app_secret}'\
        #     '&code={code_parameter}'.format(
        #         app_id=app_id,
        #         app_secret=secret_code,
        #         code_parameter=code,
        #         redirect_uri=redirect_uri,
        # )

        # 하기처럼 params로 넘겨줄 것.
        url_request_access_token = 'https://graph.facebook.com/v2.8/oauth/access_token?'
        params = {
            'client_id': APP_ID,
            'redirect_uri': REDIRECT_URI,
            'client_secret': SECRET_CODE,
            'code': code,
        }
        r = requests.get(url_request_access_token, params=params)
        pprint(r.text)
        dict_access_token = r.json()
        USER_ACCESS_TOKEN = dict_access_token['access_token']
        print('USER_ACCESS_TOKEN : %s' % USER_ACCESS_TOKEN)

        # Step 3: debug_token을 얻어오기
        # 유저 액세스 토큰과 앱 엑세스 토큰을 사용해서 토큰 검증을 거친다
        url_debug_token = 'https://graph.facebook.com/debug_token?'
        params = {
            'input_token': USER_ACCESS_TOKEN,
            'access_token': APP_ACCESS_TOKEN,
        }
        r = requests.get(url_debug_token, params=params)
        dict_debug_token = r.json()
        pprint(dict_debug_token)
        USER_ID = dict_debug_token['data']['user_id']
        print('USER_ID : %s' % USER_ID)

        # 해당 USER_ID로 graph API에 유저정보를 요청
        url_api_user = 'https://graph.facebook.com/{user_id}'.format(
            user_id=USER_ID
        )
        fields = [
            'id',
            'first_name',
            'last_name',
            'gender',
            'picture',
            'email',
        ]
        params = {
            'fields': ','.join(fields),
            'access_token': USER_ACCESS_TOKEN,
        }
        r = requests.get(url_api_user, params)
        dict_user_info = r.json()
        pprint(dict_user_info)

        # 페이스북 유저 ID만으로 인증
        # user = authenticate(facebook_id=USER_ID)
        # 페이스북 유저 ID와 graph API에 요청한 dict_user_info로 인증
        user = authenticate(facebook_id=USER_ID, extra_fields=dict_user_info)
        login(request, user)
        return redirect('index')


from os import read
import random
from typing import Counter

alphabet_s = 'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'
alphabet_d = {alphabet_s[i]: str(i) for i in range(len(alphabet_s))}
alphabet_dr = {str(i): alphabet_s[i] for i in range(len(alphabet_s))}

# sorted_by_freq = ['о','е','а','и','н','т','с','р','в','л',
# 'к','м','д','п','у','я','ы','ь','г','з','б',
# 'ч','й','х','ж','ш','ю','ц','щ','э','ф','ъ','ё']
# sorted_by_freq_cap = [l.upper() for l in sorted_by_freq]
# most_frq_bi = ['СТ', 'НО', 'ЕН', 'ТО', 'НА', 'ОВ', 'НИ', 'РА', 'ВО', 'КО']

def update_map(s, m, k):
    for i in range(len(s) - k + 1):
        if s[i: i + k] in m:
            m[s[i: i + k]] += 1
        else:
            m[s[i: i + k]] = 1

def read_text():
    with open('studies/kmzi/input_4.txt', 'r', encoding='utf-8') as f:
        return f.read()

def get_some_key():
    key = list(alphabet_d.values())
    random.shuffle(key)
    return key

def split_text_by_kgrams(text, k):
    return [text[i: i + k] for i in range(len(text) - k + 1)]

def decrypt(ciphertext, key):
    return ''.join(key[c] for c in ciphertext)

def main():
    text = read_text()
    print(text)
    fr = split_text_by_kgrams(text, 1)
    bigrams = split_text_by_kgrams(text, 2)
    print(bigrams)

    srtfrq = ['Ё', 'Ъ', 'Ф', 'Э', 'Щ', 'Ц', 'Ю', 'Ш', 'Ж', 'Х',
    'Б', 'Ч', 'Й', 'З', 'Г', 'Ы', 'Ь', 'Я', 'У', 'П', 'Л',
    'М', 'К', 'Д', 'Р', 'В', 'С', 'Т', 'Н', 'Е', 'А', 'И', 'О']

    srtfrq_c = sorted(Counter(fr).items(), key=lambda x:x[1])

    print(srtfrq)
    print(srtfrq_c)

    # for t in bigrams:
        # print(text.count(t))

    mappd_b = {bi: text.count(bi) for bi in bigrams}

    print(mappd_b)

    sorted_mappd_b = sorted(mappd_b.items(), key=lambda x:x[1], reverse=True)
    print(sorted_mappd_b)

    with open('studies/kmzi/bigr.txt', 'r') as f:
        a = f.readlines()
        newl = []
        for s in a:
            newl.append(s.replace('\n', '').replace(' ', ''))
        # print(newl)
        r_bd = {d[:2]: int(d[2:]) for d in newl}

    # print(r_bd)

    sorted_b = sorted(r_bd.items(), key=lambda x:x[1])
    print(sorted_b)

    iter_crypt = [x[0].lower() for x in sorted_mappd_b]
    print(iter_crypt)

    print(''.join(sorted_b[iter_crypt.index(b.lower())][0] for b in bigrams))

    # with open('studies\\kmzi\\trigrams', 'rb') as f:
    #     print(pickle.load(f))
    # key = get_some_key()
    # key_s = ''.join(alphabet_dr[c] for c in key)
    # print(key, key_s)
    # for i in range(1, 6):
        # s = split_text_by_kgrams(text, i)
        # print(s, len(s))

if __name__ == '__main__':
    main()