from abc import ABC, abstractmethod
from typing import Any
from pandasai.helpers.env import is_running_in_console
from ..helpers.df_info import polars_imported

from pandasai.exceptions import MethodNotImplementedError

class IResponseParser(ABC):
    @abstractmethod
    def parse(self, result: dict) -> Any:
        """
        Parses result from the chat input
        Args:
            result (dict): result contains type and value
        Raises:
            ValueError: if result is not a dictionary with valid key

        Returns:
            Any: Returns depending on the user input
        """
        raise MethodNotImplementedError

class ResponseParser(IResponseParser):
    _context = None

    def __init__(self, context) -> None:
        """
        Initialize the ResponseParser with Context from SmartDataLake
        Args:
            context (Context): context contains the config, logger and engine
        """
        self._context = context

    def parse(self, result: dict) -> Any:
        """
        Parses result from the chat input
        Args:
            result (dict): result contains type and value
        Raises:
            ValueError: if result is not a dictionary with valid key

        Returns:
            Any: Returns depending on the user input
        """
        if not isinstance(result, dict) or not all(
            key in result for key in ["type", "value"]
        ):
            raise ValueError("Unsupported result format")

        if result["type"] == "dataframe":
            return self.format_dataframe(result)
        elif result["type"] == "plot":
            return self.format_plot(result)
        else:
            return self.format_other(result)

    def format_dataframe(self, result: dict) -> Any:
        """
        Format dataframe generate against a user query
        Args:
            result (dict): result contains type and value
        Returns:
            Any: Returns depending on the user input
        """
        from ..smart_dataframe import SmartDataframe

        df = result["value"]
        if self._context.engine == "polars" and polars_imported:
            import polars as pl

            df = pl.from_pandas(df)

        return SmartDataframe(
            df,
            config=self._context._config.__dict__,
            logger=self._context.logger,
        )

    def format_plot(self, result: dict) -> Any:
        """
        Display matplotlib plot against a user query
        Args:
            result (dict): result contains type and value
        Returns:
            Any: Returns depending on the user input
        """
        import matplotlib.pyplot as plt
        import matplotlib.image as mpimg

        # Load the image file
        try:
            image = mpimg.imread(result["value"])
        except FileNotFoundError:
            raise FileNotFoundError(f"The file {result['value']} does not exist.")
        except OSError:
            raise ValueError(f"The file {result['value']} is not a valid image file.")

        # Display the image
        plt.imshow(image)
        plt.axis("off")
        plt.show(block=is_running_in_console())
        plt.close("all")

    def format_other(self, result) -> Any:
        """
        Returns the result generated against a user query other than dataframes
        and plots
        Args:
            result (dict): result contains type and value
        Returns:
            Any: Returns depending on the user input
        """
        return result["value"]


from typing import List
from aiohttp import web
from src.app import app
from src.db import set_value, get_value, clean_db
from src.schemas import ExchangeRate

routes = web.RouteTableDef()

@routes.get('/')
async def get_handler(request):
    return web.Response(text=f"HELLO")

@routes.get('/convert')
async def get_convert_currency(request):
    currency_from = request.query["from"]
    currency_to = request.query["to"]
    amount = float(request.query["amount"])
    key = f"{currency_from}-{currency_to}"
    value = await get_value(key)
    result = amount / value
    data = {"result": result}
    return web.json_response(data)

async def get_value_of_field(data: str, field: str, integer_type: bool = False):
    index = data.find(field)
    if integer_type:
        start = index + len(field) + 2
    else:
        start = index + len(field) + 3
    end = data.find("\"", start)
    result = data[start:end:]
    return result

@routes.post('/database')
async def post_exchange_rate(request):
    merge = int(request.query["merge"])
    data = await request.post()
    if merge == 0:
        await clean_db()
    elif merge != 1:
        raise web.HTTPException()
    result = []
    for data_item in data:
        currency_from = await get_value_of_field(data[data_item], "currency_from")
        currency_to = await get_value_of_field(data[data_item], "currency_to")
        value = float(await get_value_of_field(data[data_item], "value", True))
        exchange_rate = ExchangeRate(currency_from=currency_from, currency_to=currency_to, value=value)
        key = f"{exchange_rate.currency_from}-{exchange_rate.currency_to}"
        value = exchange_rate.value
        result.append({"key": key, "value": value})
        await set_value(key, value)
    return web.json_response({"result": result})

app.add_routes(routes)


import random
import time

from asyncio import sleep

from p2p.DHTEntry import DHTEntry
from p2p.Message import Message
from p2p.Subscriber import Subscriber
from p2p.Timeline import Timeline
from p2p.constants import BOOTSTRAP_PORT_OFFSET

async def process_message(peer, message: Message):
    peer.logger.print_msg_received(message)
    # TODO:REMOVE FROM ZOMBIE
    if message.header.type == 'PING':
        return await process_ping_message(peer, message)

    if message.header.type == 'SHARE_ENTITY':
        return await process_share_entity_message(peer, message)

    if message.header.type == 'SUBSCRIBE':
        return process_subscribe_message(peer, message)

    if message.header.type == 'UNSUBSCRIBE':
        return process_unsubscribe_message(peer, message)

    if message.header.type == 'SUBSCRIBE_INDIRECT':
        return await process_subscribe_indirect_message(peer, message)

    if message.header.type == 'SUBSCRIBE_INDIRECT_ACK':
        return process_subscribe_indirect_ack_message(peer, message)

    if message.header.type == 'TIMELINE':
        return process_timeline_message(peer, message)

async def process_ping_message(peer, message: Message):
    neighbour = Subscriber(message.body['peer_id'], message.body['username'], message.body['origin_ip_address'],
                           message.body['origin_port'])

    if peer.identifier == neighbour.identifier:
        return

    if neighbour in peer.zombies:
        peer.logger.print_peer_back_online(neighbour)

        if neighbour in peer.followings:
            dht_entry = DHTEntry.from_json(await peer.kademlia_server.get(neighbour.identifier))

            if not dht_entry:
                return

            peer.send_subscribe_message(neighbour, dht_entry)

        peer.zombies.remove(neighbour)

    if neighbour in peer.neighbours:
        return

    if neighbour in peer.followings:
        return

    if neighbour.data_port >= BOOTSTRAP_PORT_OFFSET:
        return

    peer.neighbours.append(neighbour)

async def process_share_entity_message(peer, message):
    neighbour = Subscriber(message.body['peer_id'], message.body['peer_username'], message.body['peer_ip'],
                           message.body['peer_port'])

    if peer.identifier == neighbour.identifier:
        return

    if neighbour in peer.neighbours:
        return

    if neighbour in peer.followings:
        return

    if neighbour.data_port >= BOOTSTRAP_PORT_OFFSET:
        return

    peer.neighbours.append(neighbour)

def process_subscribe_message(peer, message: Message):
    sub = Subscriber(message.body['origin_identifier'], message.body['origin_username'],
                     message.body['origin_ip_address'],
                     message.body['origin_port'])

    if sub in peer.neighbours:
        peer.neighbours.remove(sub)

    if sub not in peer.followers:
        peer.followers.append(sub)
        peer.logger.print_new_follower(sub)
    else:
        print(f"Already Following")

def process_unsubscribe_message(peer, message: Message):
    sub = Subscriber(message.body['origin_identifier'], message.body['origin_username'],
                     message.body['origin_ip_address'],
                     message.body['origin_port'])

    if sub not in peer.neighbours:
        peer.neighbours.append(sub)
    else:
        print(f"Already in Neighbours")
    if sub in peer.followers:
        peer.followers.remove(sub)
        peer.logger.print_new_unfollower(sub)
    else:
        print(f"Already NOT following")

async def process_subscribe_indirect_message(peer, message):
    desired_sub = message.body['desired_identifier']
    desired_username = message.body['desired_username']

    if desired_sub in peer.timelines.keys():
        reply = Message.create_subscribe_indirect_ack_msg(peer.host_ip, peer.data_port,
                                                          peer.identifier, peer.timelines[desired_sub].host_ip,
                                                          peer.timelines[desired_sub].port,
                                                          desired_sub, desired_username,
                                                          peer.timelines[desired_sub].list_tweets)

        return peer.send_message(message.body['origin_ip_address'], message.body['origin_port'], reply)
    else:
        if int(message.body['time_to_live'] == 0):
            print("MSG Expired")
            return

        if len(peer.neighbours) == 0:
            return

        for i in range(3):
            neighbour = random.choice(peer.neighbours)

            if neighbour.identifier == int(message.body['origin_identifier']):
                continue
            if neighbour.identifier == int(message.body['desired_identifier']):
                continue
            try:
                peer.send_message(neighbour.host_ip, neighbour.data_port,
                                  Message.create_subscribe_indirect_msg(message.body['origin_ip_address'],
                                                                        message.body['origin_port'],
                                                                        message.body['origin_identifier'],
                                                                        message.body['desired_identifier'],
                                                                        message.body['desired_username'],
                                                                        int(message.body['time_to_live']) - 1))
            except:
                pass

            await sleep(0.1)

def process_subscribe_indirect_ack_message(peer, message):
    sub = Subscriber(message.body['desired_identifier'],
                     message.body['desired_username'],
                     message.body['desired_ip_address'],
                     message.body['desired_port'],
                     int(message.body['origin_identifier']),
                     message.body['origin_ip_address'],
                     message.body['origin_port'])

    if sub in peer.followings:
        return

    peer.followings.append(sub)

    if sub in peer.neighbours:
        peer.neighbours.remove(sub)

    print("Received Timeline from third party")

    peer.timelines[sub.identifier] = Timeline(
        sub.identifier, sub.host_ip, sub.data_port, message.body['timeline'])

def process_timeline_message(peer, message: Message):
    timeline = message.body['timeline']

    for following in peer.followings:
        if following.identifier == timeline.author_id:
            return process_timeline_from_following(peer, message)

def process_timeline_from_following(peer, message: Message):
    timeline = message.body['timeline']

    # Don't update my timeline with others
    if timeline.author_id != peer.identifier:
        return process_timeline_new_tweets(peer, message)

def process_timeline_new_tweets(peer, message: Message):
    timeline = message.body['timeline']
    rcv_time = time.time()
    for tweet in timeline.list_tweets:

        if tweet not in peer.timelines[timeline.author_id].list_tweets:
            tweet.rcv_time = rcv_time
            peer.timelines[timeline.author_id].list_tweets.append(tweet)


import logging
import argparse
import pandas as pd

from constants import *
from utils.data_tools import solve_postproc

def main(succ_files: list[str], warn_files: list[str], output: str):
    succ_dfs = [pd.read_csv(path, sep=',') for path in succ_files if path != '']
    warn_dfs = [pd.read_csv(path, sep=',') for path in warn_files if path != '']
    total_df = pd.DataFrame(columns=[COL_QUESTION, COL_ANSWER])

    logging.info('Processing successful')
    for df in succ_dfs:
        df.drop(df.columns.difference([COL_QUESTION, COL_ANSWER]), axis=1, inplace=True)
        total_df = pd.concat([total_df, df]).reset_index(drop=True)

    logging.info('Processing warnings')
    for df in warn_dfs:
        new_cols = df.apply(lambda row: solve_postproc(row, force=True), axis=1)
        df = pd.DataFrame(new_cols.tolist(), columns=[COL_QUESTION, COL_ANSWER])
        total_df = pd.concat([total_df, df]).reset_index(drop=True)

    logging.info('Saving results')
    total_df.to_csv(output, sep=',', index=False)

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
    # parse args
    parser = argparse.ArgumentParser(description='This script join the results to one file with fixed schema.')
    parser.add_argument(
        'input',
        type=str,
        help='provide comma separated global paths for input files'
    )
    parser.add_argument(
        'output',
        type=str,
        help='provide the global path for the output file'
    )
    parser.add_argument(
        '--warnings',
        type=str,
        default='',
        help='provide comma separated global paths for input warning files. Warnings may converted incorrect'
    )
    logging.info('Parsing args')
    args = parser.parse_args()

    # validate inputs
    succ_files = args.input.split(',')
    warn_files = args.warnings.split(',')

    main(succ_files, warn_files, args.output)


from django import forms
from django.conf import settings
from django.contrib.auth.models import User
from django.core.validators import ValidationError
from django.urls import reverse
from django.db.models import Q
from django.forms.widgets import Select
from django.utils.encoding import force_str
from django.utils.safestring import mark_safe
from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _, gettext
from django.utils import timezone

from crispy_forms.helper import FormHelper
from crispy_forms.layout import Layout, Field, Div, Submit, HTML
from dal import autocomplete
import logging
import re

from danceschool.core.models import (
    InvoiceItem, StaffMember, EventStaffCategory, Event, PublicEvent, Series
)
from danceschool.core.forms import EventAutocompleteForm

from .models import (
    ExpenseItem, ExpenseCategory, RevenueItem, RepeatedExpenseRule, GenericRepeatedExpense,
    LocationRentalInfo, RoomRentalInfo, StaffDefaultWage, StaffMemberWageInfo, TransactionParty
)
from .autocomplete_light_registry import get_method_list, get_approval_status_list

# Define logger for this file
logger = logging.getLogger(__name__)

PAYBY_CHOICES = (
    (1, _('Hours of Work/Rental (paid at default rate)')),
    (2, _('Flat Payment')),
)

class ExpenseCategoryWidget(Select):
    '''
    Override render_option to permit extra data of default wage to be used by JQuery
    This could be optimized to reduce database calls by overriding the render function.
    '''

    def render_option(self, selected_choices, option_value, option_label):
        if option_value is None:
            option_value = ''
        option_value = force_str(option_value)
        if option_value in selected_choices:
            selected_html = mark_safe(' selected="selected"')
            if not self.allow_multiple_selected:
                # Only allow for a single selection.
                selected_choices.remove(option_value)
        else:
            selected_html = ''

        # Pass the default wage rate as an option
        if option_value:
            defaultRate = ExpenseCategory.objects.filter(id=int(option_value)).first().defaultRate
            extra_value_data = ' data-defaultRate=' + str(defaultRate)
        else:
            extra_value_data = ''

        return format_html('<option value="{}"{}{}>{}</option>',
                           option_value,
                           selected_html,
                           extra_value_data,
                           force_str(option_label))

class ExpenseReportingForm(EventAutocompleteForm, forms.ModelForm):
    payTo = forms.ModelChoiceField(
        queryset=TransactionParty.objects.all(),
        label=_('Pay to'),
        required=True,
        widget=autocomplete.ModelSelect2(
            url='transactionParty-list-autocomplete',
            attrs={
                # This will set the input placeholder attribute:
                'data-placeholder': _('Enter a name or location'),
                # This will set the yourlabs.Autocomplete.minimumCharacters
                # options, the naming conversion is handled by jQuery
                'data-minimum-input-length': 2,
                'data-max-results': 8,
                'class': 'modern-style',
            }
        )
    )

    payBy = forms.ChoiceField(
        widget=forms.RadioSelect, choices=PAYBY_CHOICES,
        label=_('Report this expense as:'), initial=2
    )
    paymentMethod = autocomplete.Select2ListCreateChoiceField(
        choice_list=get_method_list,
        required=False,
        widget=autocomplete.ListSelect2(url='paymentMethod-list-autocomplete'),
        label=_('Payment method'),
    )
    approved = autocomplete.Select2ListCreateChoiceField(
        choice_list=get_approval_status_list,
        required=False,
        widget=autocomplete.ListSelect2(url='approved-list-autocomplete'),
        label=_('Approved'),
    )

    def __init__(self, *args, **kwargs):
        user = kwargs.pop('user', None)
        user_id = getattr(user, 'id', None)

        if user_id:
            kwargs.update(initial={
                'payTo': TransactionParty.objects.get_or_create(
                    user=user, defaults={'name': user.get_full_name()}
                )[0].id,
            })

        super().__init__(*args, **kwargs)

        self.helper = FormHelper()

        # The hidden input of accrual date must be passed as a naive datetime.
        # Django will take care of converting it to local time
        accrualDate_field = Field(
            'accrualDate', type="hidden",
            value=(
                timezone.make_naive(timezone.now()) if
                timezone.is_aware(timezone.now()) else timezone.now()
            )
        )

        if user and user.has_perm('financial.mark_expenses_paid'):
            payment_section = Div(
                Div(
                    HTML(
                        '' +
                        ('%s (%s)' % (
                            _('Mark as Approved/Paid'), _('click to expand')
                        ))
                    ),
                    css_class='card-header'
                ),
                Div(
                    'approved',
                    'paid',
                    Div(
                        Field('paymentDate', css_class='datepicker'),
                        HTML(
                            '
' +
                            '<button id="payment-event-start" ' +
                            'class="btn btn-outline-secondary">' +
                            ('%s</button>
' % _('Event Start Date'))
                        ),
                        css_class='form-row',
                    ),
                    'paymentMethod',
                    HTML(
                        '
' +
                        (
                            '%s %s
' % (
                                _('Note:'),
                                _(
                                    'For accounting purposes, please do not ' +
                                    'mark expenses as paid unless they have ' +
                                    'already been paid to the recipient.'
                                )
                            )
                        ),
                    ),
                    css_class='card-body collapse',
                    id='collapsepayment',
                ),
                css_class='card my-4'
            )
        else:
            payment_section = None

        # Add category button should only appear for users who are allowed to add categories
        if user.has_perm('financial.add_expensecategory'):
            related_url = reverse('admin:financial_expensecategory_add') + '?_to_field=id&_popup=1'
            added_html = [
                (' ',
                '' % (
                    getattr(settings, 'STATIC_URL', '/static/'), _('Add Another')
                )
            ]
            category_field = Div(
                Div('category', css_class='col-sm-11'),
                Div(HTML('\n'.join(added_html)), css_class='col-sm-1', style='margin-top: 25px;'),
                css_class='related-widget-wrapper row'
            )
        else:
            category_field = Div('category')

        self.fields['event'].label = _('Event (optional)')
        self.fields['event'].required = False

        self.helper.layout = Layout(
            Field('submissionUser', type="hidden", value=user_id),
            'payTo',
            'payBy',
            category_field,
            'description',
            'hours',
            'total',
            'event',
            'reimbursement',
            accrualDate_field,
            payment_section,
            'attachment',
            Submit('submit', _('Submit')),
        )

    def clean(self):
        # Custom cleaning ensures that user, hours, and total
        # are not reported where not necessary.
        super().clean()

        payBy = self.cleaned_data.get('payBy')
        hours = self.cleaned_data.get('hours')
        total = self.cleaned_data.get('total')

        paid = self.cleaned_data.get('paid')
        paymentDate = self.cleaned_data.get('paymentDate')

        event = self.cleaned_data.get('event')

        # Automatically marks expenses that are paid
        # upon submission as accruing at the date of payment.
        if paid and paymentDate:
            self.cleaned_data['accrualDate'] = paymentDate
        else:
            self.cleaned_data.pop('paymentDate', None)

        # If an event has been specified, then that takes precedence for setting
        # the accrual date of the expense.
        if event and getattr(event, 'startTime', None):
            self.cleaned_data['accrualDate'] = event.startTime

        if payBy == '1' and total:
            self.cleaned_data.pop('total', None)
        if payBy == '2' and hours:
            self.cleaned_data.pop('hours', None)
        return self.cleaned_data

    class Meta:
        model = ExpenseItem
        fields = [
            'submissionUser', 'payTo', 'category', 'description', 'hours',
            'total', 'reimbursement', 'attachment', 'approved', 'paid',
            'paymentDate', 'paymentMethod', 'accrualDate', 'event'
        ]
        widgets = {
            'category': ExpenseCategoryWidget,
        }

    def media(self):
        ''' Remove jQuery from widget media to avoid duplicate initialization by django-filer '''
        media = super().media

        regex = re.compile(r"^admin/.*jquery(\.min)?\.js$")
        new_js = [x for x in media._js if not regex.search(x)]
        new_css = media._css
        media = forms.Media(js=new_js, css=new_css)

        add_media = forms.Media(
            js = (
                'admin/js/admin/RelatedObjectLookups.js',
                'bootstrap-datepicker/js/bootstrap-datepicker.min.js',
                'js/expense_reporting.js',
            ),
            css = {
                'all': (
                    'bootstrap-datepicker/css/bootstrap-datepicker.standalone.min.css',
                ),
            }
        )
        media += add_media
        return media

class InvoiceItemChoiceField(forms.ModelChoiceField):
    '''
    This exists so that the validators for InvoiceItems (EventRegistrations) are not
    thrown off by the fact that the initial query is blank.
    '''

    def to_python(self, value):
        try:
            value = super().to_python(value)
        except (ValueError, ValidationError):
            key = self.to_field_name or 'pk'
            value = InvoiceItem.objects.filter(**{key: value})
            if not value.exists():
                raise ValidationError(self.error_messages['invalid_choice'], code='invalid_choice')
            else:
                value = value.first()
        return value

class RevenueReportingForm(EventAutocompleteForm, forms.ModelForm):
    '''
    This form is used in the revenue reporting view for quick generation of RevenueItems.
    '''

    receivedFrom = forms.ModelChoiceField(
        queryset=TransactionParty.objects.all(),
        label=_('Received from'),
        required=False,
        widget=autocomplete.ModelSelect2(
            url='transactionParty-list-autocomplete',
            attrs={
                # This will set the input placeholder attribute:
                'data-placeholder': _('Enter a name or location'),
                # This will set the yourlabs.Autocomplete.minimumCharacters
                # options, the naming conversion is handled by jQuery
                'data-minimum-input-length': 2,
                'data-max-results': 8,
                'class': 'modern-style',
            }
        )
    )

    currentlyHeldBy = forms.ModelChoiceField(
        queryset=User.objects.filter(Q(is_staff=True) | Q(staffmember__isnull=False)),
        label=_('Cash currently in possession of'),
        required=False,
        widget=autocomplete.ModelSelect2(
            url='autocompleteUser',
            attrs={
                # This will set the input placeholder attribute:
                'data-placeholder': _('Enter a user name'),
                # This will set the yourlabs.Autocomplete.minimumCharacters
                # options, the naming conversion is handled by jQuery
                'data-minimum-input-length': 2,
                'data-max-results': 4,
                'class': 'modern-style',
            }
        )
    )
    paymentMethod = autocomplete.Select2ListCreateChoiceField(
        choice_list=get_method_list,
        required=False,
        widget=autocomplete.ListSelect2(url='paymentMethod-list-autocomplete'),
        label=_('Payment method'),
    )

    def __init__(self, *args, **kwargs):
        user = kwargs.pop('user', None)

        if hasattr(user, 'id'):
            kwargs.update(initial={
                'submissionUser': user.id
            })
        super().__init__(*args, **kwargs)

        self.helper = FormHelper()

        detail_section = Div(
            Div(
                HTML(
                    '%s (%s)' % (
                        _('Adjustments/Fees'), _('click to expand')
                    )
                ),
                css_class='card-header'
            ),
            Div(
                'total',
                'adjustments',
                'fees',
                css_class='card-body collapse',
                id='collapsedetails',
            ),
            css_class='card my-4')

        event_section = Div(
            Div(
                HTML(
                    '%s (%s)' % (
                        _('Event/Invoice item (optional)'), _('click to expand')
                    )
                ),
                css_class='card-header'
            ),
            Div(
                'event',
                'invoiceItem',
                css_class='card-body collapse',
                id='collapseevent',
            ),
            css_class='card my-4')

        if user and user.has_perm('financial.mark_revenues_received'):
            receipt_section = Div(
                Div(
                    HTML(
                        '%s (%s)' % (
                            _('Mark as Received'), _('click to expand')
                        )
                    ),
                    css_class='card-header'
                ),
                Div(
                    'received',
                    Div(
                        Field('receivedDate', css_class='datepicker'),
                        HTML(
                            '
' +
                            '<button id="received-event-start" ' +
                            'class="btn btn-outline-secondary">' +
                            ('%s</button>
' % _('Event Start Date'))
                        ),
                        css_class='form-row',
                    ),
                    'currentlyHeldBy',
                    # The hidden input of accrual date must be passed as a naive datetime.
                    # Django will take care of converting it to local time
                    HTML('
%s %s
' % (
                        _('Note:'),
                        _(
                            'For accounting purposes, please do not mark revenues ' +
                            'as received until the money is in our possession.'
                        )
                    )),
                    css_class='card-body collapse',
                    id='collapsereceipt',
                ),
                css_class='card my-4')
        else:
            receipt_section = None

        self.fields["invoiceItem"] = InvoiceItemChoiceField(queryset=InvoiceItem.objects.none(), required=False)
        self.fields['event'].required = False

        # Handled by the model's save() method
        self.fields['total'].required = False
        self.fields['fees'].required = False
        self.fields['adjustments'].required = False

        self.helper.layout = Layout(
            Field('submissionUser', type="hidden", value=getattr(user, 'id', None)),
            Field('invoiceNumber', type="hidden"),
            'category',
            'description',
            'receivedFrom',
            'paymentMethod',
            'grossTotal',
            detail_section,
            event_section,
            receipt_section,
            'attachment',
            Submit('submit', _('Submit')),
        )

    def clean_description(self):
        ''' Avoid empty descriptions '''
        return self.cleaned_data['description'] or _('Form Submitted Revenue')

    def clean_invoiceNumber(self):
        ''' Create a unique invoice number '''
        return 'SUBMITTED_%s_%s' % (
            getattr(self.cleaned_data['submissionUser'], 'id', 'None'), timezone.now().strftime('%Y%m%d%H%M%S')
        )

    class Meta:
        model = RevenueItem
        fields = [
            'submissionUser', 'invoiceNumber', 'category', 'description', 'event',
            'invoiceItem', 'receivedFrom', 'paymentMethod', 'currentlyHeldBy',
            'grossTotal', 'total', 'adjustments', 'fees', 'attachment', 'received',
            'receivedDate'
        ]

    def media(self):
        ''' Remove jQuery from widget media to avoid duplicate initialization by django-filer '''
        media = super().media

        regex = re.compile(r"^admin/.*jquery(\.min)?\.js$")
        new_js = [x for x in media._js if not regex.search(x)]
        new_css = media._css
        media = forms.Media(js=new_js, css=new_css)

        add_media = forms.Media(
            js = (
                'js/revenue_reporting.js',
                'bootstrap-datepicker/js/bootstrap-datepicker.min.js',
            ),
            css = {
                'all': (
                    'bootstrap-datepicker/css/bootstrap-datepicker.standalone.min.css',
                ),
            }
        )
        media += add_media
        return media

class CompensationRuleUpdateForm(forms.ModelForm):
    ''' Used for bulk update of StaffMember compensation rules. '''

    def save(self, commit=True):
        ''' Handle the update logic for this in the view, not the form '''
        pass

    class Meta:
        model = StaffMemberWageInfo
        fields = ['category', 'rentalRate', 'applyRateRule', 'dayStarts', 'weekStarts', 'monthStarts']

class CompensationRuleResetForm(forms.Form):
    ''' Used for bulk reset of StaffMember compensation rules to category defaults '''

    def save(self, commit=True):
        ''' Handle the update logic for this in the view, not the form '''
        pass

    def __init__(self, *args, **kwargs):
        staffmembers = kwargs.pop('staffmembers', StaffMember.objects.none())

        # Initialize a default (empty) form to fill
        super().__init__(*args, **kwargs)

        for cat in EventStaffCategory.objects.order_by('name'):
            this_label = cat.name
            this_help_text = ''
            if not getattr(cat, 'defaultwage', None):
                this_help_text += gettext('No default compensation specified. ')
            if staffmembers:
                this_help_text += gettext('{count} selected members with rules specified.').format(
                    count=staffmembers.filter(expenserules__category=cat).count(),
                )

            self.fields['category_%s' % cat.id] = forms.BooleanField(
                required=False, label=this_label, help_text=this_help_text
            )

        self.fields['resetHow'] = forms.ChoiceField(
            label=_('For each selected category:'),
            choices=(
                ('COPY', _('Copy default rules to each staff member')),
                ('DELETE', _('Delete existing custom rules'))
            )
        )

class ExpenseRuleGenerationForm(forms.Form):
    ''' Generate a form with all expense rules '''

    staff = forms.BooleanField(
        required=False, initial=True, label=_('Generate expense items for event staff')
    )
    venues = forms.BooleanField(
        required=False, initial=True, label=_('Generate expense items for venue rental')
    )
    generic = forms.BooleanField(
        required=False, initial=True, label=_('Generate expense items for generic rules')
    )
    registrations = forms.BooleanField(
        required=False, initial=True,
        label=_('Generate revenue items for registrations')
    )

    def __init__(self, *args, **kwargs):

        # Initialize a default form to fill by rule
        super().__init__(*args, **kwargs)

        for rule in RepeatedExpenseRule.objects.filter(disabled=False).order_by('id'):
            prefix = 'genericrule'

            if isinstance(rule, LocationRentalInfo):
                prefix = 'locationrule'
            elif isinstance(rule, RoomRentalInfo):
                prefix = 'roomrule'
            elif isinstance(rule, StaffDefaultWage):
                prefix = 'staffdefaultrule'
            elif isinstance(rule, StaffMemberWageInfo):
                prefix = 'staffmemberule'

            self.fields['%s_%s' % (prefix, rule.id)] = forms.BooleanField(
                required=False, initial=True, label=rule.ruleName
            )


'''
JeanClement GALLARDO
09/07/2020
Comparaison des résultats explicatifs de LIME sur deux modèles différents
'''

import numpy as np
import pandas as pd
import os
import csv

import lime
import lime.lime_tabular
import matplotlib.pyplot as plt
import seaborn as sbn

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

## Function ##

# Import data and encode clinical label
def import_csv(path, encoder):
    ## Import
    script_dir = os.path.dirname("multinomial_logistic_regression")
    data_path = os.path.join(script_dir, path)
    data = pd.read_csv(data_path)
    data.drop(["patient_id"], axis = 1, inplace = True)
    data = data.dropna()
    ## Encode
    categorical_feature_mask = data.dtypes==object
    categorical_cols = data.columns[categorical_feature_mask].tolist()
    data[categorical_cols] = data[categorical_cols].apply(lambda col: encoder.fit_transform(col))
    return data

# Create and fit logistic model
def create_logistic(X_train, Y_train, C_value, tol_value):
    logistic_model = LogisticRegression(
        C = C_value, penalty='l1', solver='saga', tol=tol_value, max_iter = 2000
    )
    logistic_model.fit(X_train, Y_train)
    return logistic_model

def create_svm(X_train, Y_train, C, tol):
    model = SVC(
        C = C, tol = tol, kernel = "linear", probability = True
    )
    model.fit(X_train, Y_train)
    return model

## Create Lime explainer
def lime_explainer(model, training, test, sample, features_names, subtype, sub):
    explainer = lime.lime_tabular.LimeTabularExplainer(training, feature_names = features_names, class_names = subtype)
    exp = explainer.explain_instance(test[sample], model.predict_proba, num_features = 50, labels = [0,1,2,3,4])
    return exp.as_list(label = sub)

if __name__ == "__main__":
    encoder = LabelEncoder()
    data = import_csv("../data/data_pam50.csv", encoder)
    print("Data imported")

    geneTrain, geneTest = train_test_split(data, train_size = 700, random_state = 69780)
    X_train = geneTrain.loc[:, geneTrain.columns != 'subtype']
    Y_train = geneTrain.subtype
    X_test = geneTest.loc[:, geneTest.columns != 'subtype']
    Y_test = geneTest.subtype
    print("Data split")

    logistic = create_logistic(X_train,Y_train, 0.5,0.001)
    svm = create_svm(X_train,Y_train,0.5,0.001)
    print("Model created")

    training = X_train.to_numpy()
    test = X_test.to_numpy()
    features_names = data.columns[1:]
    subtype = encoder.inverse_transform(logistic.classes_)
    sub = Y_test.to_numpy()
    sub_decode = encoder.inverse_transform(sub)
    mat_lrm = {}
    mat_svm = {}

    for sample in range(len(test)):
        tmp_logistic = lime_explainer(logistic, training, test, sample, features_names, subtype, sub[sample])
        tmp_svm = lime_explainer(svm, training, test, sample, features_names, subtype, sub[sample])
        mat_lrm[sample] = {}
        mat_svm[sample] = {}
        for idx in range(len(tmp_logistic)):
            split_row = tmp_logistic[idx][0].split(" ")
            value = tmp_logistic[idx][1]
            j = 0
            while(j < len(split_row)):
                if (split_row[j].isalnum() == False):
                    split_row.pop(j)
                else :
                    j+=1
            gene = str(split_row)
            mat_lrm[sample][gene] = float(value)
        for idx in range(len(tmp_svm)):
            split_row = tmp_svm[idx][0].split(" ")
            value = tmp_svm[idx][1]
            j = 0
            while(j < len(split_row)):
                if (split_row[j].isalnum() == False):
                    split_row.pop(j)
                else :
                    j+=1
            gene = str(split_row)
            mat_svm[sample][gene] = float(value)

    x_label = sorted(mat_lrm[1].keys())

    lrm = []
    svm = []
    maxi = 0
    for sample in mat_lrm.keys():
        gene = mat_lrm[sample].keys()
        gene = sorted(gene)
        tmp_lrm = []
        tmp_svm = []
        for idx in range(len(gene)):
            tmp_lrm.append(mat_lrm[sample][gene[idx]])
            tmp_svm.append(mat_svm[sample][gene[idx]])
            if (maxi < mat_lrm[sample][gene[idx]]):
                maxi = mat_lrm[sample][gene[idx]]
        lrm.append(tmp_lrm)
        svm.append(tmp_svm)

    mini = -maxi

    i = 0
    while(i < len(lrm)):
        if (lrm[i] == []):
            lrm.pop(i)
        else :
            i+=1

    i = 0
    while(i < len(svm)):
        if (svm[i] == []):
            svm.pop(i)
        else :
            i+=1

    mat_final = lrm

    for idx in range(len(svm)):
        mat_final.append(svm[idx])

    print("begin compute heatmap")
    sbn.set(color_codes = True)

    dict_map = {"Her2" : "blue", "Basal" : "red", "Normal" : "green", "LumA" : "yellow", "LumB" : "black"}
    row_colors = []
    for idx in range(len(sub_decode)):
        row_colors.append(dict_map[sub_decode[idx]])

    test_sous_type = row_colors*2

    fg = sbn.clustermap(mat_final, row_colors = test_sous_type, xticklabels = x_label,
        cmap = "twilight_shifted", vmin=mini, vmax=maxi)

    plt.show()


#################################################################
# Wikipedia Search for Historical Firm Founding Dates
# firm-founding-wiki-search.py
# Author: Meredith M. Paker, merpaker@gmail.com
# Last edited: 17 November, 2017
# This script uses python's Wikipedia package search functionality
#     to link a historical firm with its suggested Wikipedia information
#     in order to predict the correct founding date for the firm.
#################################################################

import wikipedia
import re

### IMPORT COMPANY LIST ###

company_list = open('CompanyList.txt', encoding='utf-16')
company_list_text = company_list.read()
company_list.close()
company_set = company_list_text.split('\n')

wiki_output = open('WikipediaFoundingDates.txt', mode='a', encoding='utf-8')

### CYCLE THROUGH EACH COMPANY ###

for company in company_set:

	# Search for company on wikipedia
	isWikiPage = 1
	try:
		search_results = wikipedia.search(company)[0]
	except IndexError as f:
		print("Error: {0}".format(f))
		search_results = ""
		isWikiPage = 0

	# Make new wikipedia object for the company
	if isWikiPage == 1:
		try:
			company_wiki = wikipedia.page(search_results)
		except wikipedia.exceptions.DisambiguationError as e:
			print("Error: {0}".format(e))
			isWikiPage = 0

	# Save the plain text of the wiki page
	if isWikiPage == 1:
		wiki_text = company_wiki.content
	else:
		wiki_text = ""

	# Parse plain text paragraphs into sentences
	from nltk.tokenize import sent_tokenize
	wiki_sentences = sent_tokenize(wiki_text)

	# Store full set of sentences
	wiki_sentences_full = wiki_sentences*1

	# Delete sentences without the company name
	for i in reversed(range(len(wiki_sentences))):
		if company not in wiki_sentences[i]:
			del wiki_sentences[i]

	# Store set of sentences with company name
	wiki_sentences_company = wiki_sentences*1

	# Delete sentences without "founded" or "established" from
	# set of sentences with company name

	for i in reversed(range(len(wiki_sentences))):
		if ("founded" or "established") not in wiki_sentences[i]:
			del wiki_sentences[i]

	# Store set of sentences with company name and founding
	wiki_sentences_founded = wiki_sentences*1

	# Generate output string
	company_output = company
	company_output += "; "
	company_output += search_results
	company_output += "; "

	# Search for years in all three scopes: full page, company sentences, founded/established
	# Put all years in output string
	# Update best-guess as appropriate
	# Update flag if best guess is from full page
	best_guess = 9000
	conf = 0
	# Full wikipedia page search
	for i in range(len(wiki_sentences_full)):
		d = re.findall('\d{4}', wiki_sentences_full[i])
		for j in range(len(d)):
			if int(d[j]) > 1600:
				if int(d[j]) < 1913:
					company_output += d[j]
					company_output += ", "
					if not wiki_sentences_company:
						if int(d[j]) < best_guess:
							best_guess = int(d[j])
						conf = 1
	company_output += "; "

	# Search in sentences with company name
	for i in range(len(wiki_sentences_company)):
		d = re.findall('\d{4}', wiki_sentences_company[i])
		for j in range(len(d)):
			if int(d[j]) > 1600:
				if int(d[j]) < 1913:
					company_output += d[j]
					company_output += ", "
					if not wiki_sentences_founded:
						if int(d[j]) < best_guess:
							best_guess = int(d[j])
						conf = 2
	company_output += "; "

	# Search in sentences with company name & "founded" or "established"
	for i in range(len(wiki_sentences_founded)):
		d = re.findall('\d{4}', wiki_sentences_founded[i])
		for j in range(len(d)):
			if int(d[j]) > 1600:
				if int(d[j]) < 1913:
					company_output += d[j]
					company_output += ", "
					if int(d[j]) < best_guess:
						best_guess = int(d[j])
					conf = 3
	company_output += "; "

	company_output += str(conf)
	company_output += "; "
	company_output += str(best_guess)

	print(company_output)
	wiki_output.write(company_output)

# So final company_output string will be as follows:

# company name, wikipedia page or blank, date(s) from full page,
# dates from sentences with company name, dates from sentences with
# company name and "founded" or "established", confidence in predicted
# founded year, best guess at founded year

wiki_output.close()


import sys
import os

sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
import struct
import socket
import config
import util.simsocket as simsocket
from config import Type
from typing import List, Tuple, Dict

class UDP:
    def __init__(self, sock: simsocket) -> None:
    # def __init__(self, sock: None) -> None:
        self.HEADER_LEN = struct.calcsize(config.headerType)
        self.BUF_SIZE = 1400
        self.MAGIC = 52305
        self.TEAM = config.TEAM
        self.sock = sock
        # self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    def pack(self, type1: int, data: bytes, seq: int, ack: int, sf: int, rwnd: int = 0):
        a = (
                struct.pack(
                    config.headerType,
                    self.MAGIC,
                    self.TEAM,
                    type1,
                    struct.calcsize(config.headerType),
                    self.HEADER_LEN + len(data),
                    seq,
                    ack,
                    sf,
                    rwnd,
                )
                + data
        )
        return a

    def unpack(self, package: bytes):
        return (
            struct.unpack(config.headerType, package[: self.HEADER_LEN]),
            package[self.HEADER_LEN:],
        )

    def send(self, package: bytes, addr: Tuple[str, int]):
        # _, _, type1, _, totalLen, seq, ack, sf, rwnd = struct.unpack(config.headerType, package[: self.HEADER_LEN])
        # print(f"SEND {addr[1]}: dataLen:{totalLen - self.HEADER_LEN} seq:{seq} ack:{ack} sf:{sf} rwnd:{rwnd}")
        self.sock.sendto(package, addr)

    def sendSegment(self, type1: Type, data: bytes, seq: int, ack: int, sf: int, rwnd: int = 0, addr: Tuple[str, int] = None):
        self.send(self.pack(type1.value, data, seq, ack, sf, rwnd), addr)

    def recv(self):
        package, addr = self.sock.recvfrom(self.BUF_SIZE)
        header, data = self.unpack(package)
        return header, data, addr

# UDP = UDP(None)
# # # a = UDP.pack(1, b"123", 1, 1, 1, 1)
# a = b'\xccQ\x01\x00\x00\x15\x00)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00;h\x11\x08G\x94\x1b\x84\xe8\xd0T\x17\xa5\xb2`\x91"\xa5c\x14'
# # # print(a)
# print(UDP.unpack(a))


# -*- coding: utf-8 -*-
"""
Created on Wed Jul 12 16:30:53 2017

@author: luoshichao
"""
import numpy as np
import pandas as pd
import matplotlib.pylab as plt

sample_label=pd.read_csv('E:/geo/bank/sample_label.dst',sep='\t')
sample_bak=pd.read_csv('E:/geo/bank/sample.bak',sep='\t')
sample_bak=sample_bak[sample_bak['mbl_num'].notnull()]
sample_bak['mbl_num']=[str(int(item)) for item in sample_bak['mbl_num']]
sample_label['mbl_num']=[str(int(item)) for item in sample_label['mbl_num']]
data=pd.merge(sample_label,sample_bak,on='mbl_num')
data['target']=0
data['target'][data['overdue_days']>30]=1
data=data[['mbl_num','target','iden_num_x','loan_dt','bankcard_num_x']]

feature_data=pd.read_csv('E:/geo/bank/data/feature_data.csv',low_memory=False)

train=pd.merge(data,feature_data,left_on='bankcard_num_x',right_on='card')

del data,feature_data,sample_bak,sample_label

#删除已知明显的重复字段
train=train.drop(['bankcard_num_x','card','time','年月'],axis=1)
train=train.drop_duplicates()
col_name=list(train.columns)
col_name.remove('target')
#train.to_csv('E:/geo/bank/data/train.csv',index=None)
test_all_data=pd.read_csv('E:/geo/test_all_data.csv',low_memory=False)
test=test_all_data[col_name]
test=test.drop_duplicates()

#缺少值处理
train.fillna(-999.0,inplace=True)
test.fillna(-999.0,inplace=True)
#train.replace(to_replace=-9999999,value=-1, inplace=True)

#分别对列和行做统计
train_feature_miss=(train==-999.0).sum(axis=0).reset_index()
train_feature_miss.columns=['feature','null_num']
train_feature_miss=train_feature_miss.sort_values(by='null_num',ascending=False)
train_feature_miss['null_odds']=train_feature_miss['null_num']/train_feature_miss['null_num'].max()

train_feature_miss=(train==-999.0).sum(axis=0).reset_index()
train_feature_miss.columns=['feature','null_num']
train_feature_miss=train_feature_miss.sort_values(by='null_num',ascending=False)
train_feature_miss['null_odds']=train_feature_miss['null_num']/train_feature_miss['null_num'].max()

'''
test_feature_miss=(test==-999.0).sum(axis=0).reset_index()
test_feature_miss.columns=['feature','null_num']
test_feature_miss=test_feature_miss.sort_values(by='null_num',ascending=False)
test_feature_miss['null_odds']=test_feature_miss['null_num']/test_feature_miss['null_num'].max()
'''

#有时间可以画个图

feature=list(train_feature_miss['feature'][train_feature_miss['null_odds']<0.7])

train['n_null']=(train==-999.0).sum(axis=1)
train_1 = train[['mbl_num','target','n_null']]
train_1=train_1.sort_values(by='n_null')
t_1=train_1['n_null'].values
y_1=train_1.target.cumsum()
x=range(len(t_1))
plt.scatter(x,t_1,c='k')
plt.plot(x,y_1,c='b')
plt.title('train set')
plt.show()

test['n_null']=(test==-999.0).sum(axis=1)
test_1 = test[['mbl_num','n_null']]
test_1=test_1.sort_values(by='n_null')
t_1=test_1['n_null'].values
x=range(len(t_1))
plt.scatter(x,t_1,c='k')
plt.title('test set')
plt.show()

train_1 = train_1[train_1['n_null']<100]
test_1 = test_1[test_1['n_null']<100]
#train_1.to_csv('E:/geo/bank/data/train_1.csv',index=None,encoding='utf-8')

train_1['discret_null'] = train_1.n_null
train_1.discret_null[train_1.discret_null<=20] = 1
train_1.discret_null[(train_1.discret_null>20)&(train_1.discret_null<=40)] = 2
train_1.discret_null[(train_1.discret_null>40)&(train_1.discret_null<=50)] = 3
train_1.discret_null[(train_1.discret_null>50)] = 4
train_1=train_1[['mbl_num','discret_null']]
train_1.to_csv('E:/geo/bank/data/train_n_null.csv',index=None,encoding='utf-8')

test_1['discret_null'] = test_1.n_null
test_1.discret_null[test_1.discret_null<=20] = 1
test_1.discret_null[(test_1.discret_null>20)&(test_1.discret_null<=40)] = 2
test_1.discret_null[(test_1.discret_null>40)&(test_1.discret_null<=50)] = 3
test_1.discret_null[(test_1.discret_null>50)] = 4
test_1=test_1[['mbl_num','discret_null']]
test_1.to_csv('E:/geo/bank/data/test_n_null.csv',index=None,encoding='utf-8')

data_1=train[feature]
data_2=pd.DataFrame(train_1['mbl_num'])
train_data=pd.merge(data_1,data_2,on='mbl_num')
del data_1,data_2
train_data.to_csv('E:/geo/bank/data/train_data.csv',index=None,encoding='utf-8')

feature.remove('target')
data_11=test[feature]
data_22=pd.DataFrame(test_1['mbl_num'])
test_data=pd.merge(data_11,data_22,on='mbl_num')

test_data.to_csv('E:/geo/bank/data/test_data.csv',index=None,encoding='utf-8')


from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Chrome()

driver.get("http://www.learnwebservices.com/locations/server/")

coordinates = driver.find_element(By.XPATH, "/html/body/div/table/tbody[2]/tr[578]/td[3]").text
print(coordinates)
print(type(coordinates))

driver.close()


# -*- coding: utf-8 -*-
import os
import time
from datetime import datetime, timedelta
from configs.path import DOWNLOAD_PATH
from helpers._selenium import ClientSideCrawler
from configs.url import KRX
from helpers._pd import parse_html_to_excel

def kospi_spider():
    browser = ClientSideCrawler(KRX)
    driver = browser.driver

    try:
        # Click the Stock id.
        kospi_btn = driver.find_element_by_css_selector('input.schdate[value=STK]')
        kospi_btn.click()
        print('The radio button is clicked %s' % kospi_btn)

        # if next_date:
        _next_date = datetime.strftime(datetime.now() - timedelta(1), '%Y%m%d')
        date_input = driver.find_element_by_css_selector('.cal-area input.schdate')
        date_input.clear()
        date_input.send_keys(_next_date)

        # Click the search button.
        submit_btn = driver.find_element_by_css_selector('button.btn-board-search')
        submit_btn.click()

        # Download the xml.
        download_btn = driver.find_element_by_css_selector(
            'button.btn-board-download')
        download_btn.click()

        timeout = 5
        seconds = 0
        files = os.listdir(DOWNLOAD_PATH)

        while seconds < timeout:
            time.sleep(1)
            next_files = os.listdir(DOWNLOAD_PATH)

            print('Counting down %s' % seconds)

            if len(next_files) > len(files):
                print('Done. Please read the file in %s' % DOWNLOAD_PATH)
                driver.close()
            seconds += 1

        browser.tear_down()
    except Exception as inst:
        print('Error occurs while running TOPIX')
        print(inst)
        browser.tear_down()

if __name__ == '__main__':
    kospi_spider()


from PyQt5.QtCore import pyqtSignal, QObject, pyqtSlot, QRunnable
from send2trash import send2trash, TrashPermissionError
import shutil
import uuid
import os

ON_SUCCESS = 1
ON_CANCEL = 2
ON_ERROR = 3

class TransferSignals(QObject):
    """
    Defines the signals available from a running worker thread.
    Supported signals are:
    finished
    `str` transfer id
    progress
    `int` indicating % progress
    """
    __slots__ = ()

    finished = pyqtSignal(str)
    progress = pyqtSignal(str, int)
    transferred = pyqtSignal(str, float)
    duplicate = pyqtSignal(str)

class Transfer(QRunnable):
    """
    File transfer runnable
    Runs on a different thread
    inherits:
        QRunnable
    parameters:
        `src`: str file path
        `dst`: str file/dir path
        `size`: float `src` file size in bytes
    """
    __slots__ = (
        "src", "dst", "size",
        "signals", "running", "job_id")

    def __init__(self, src, dst, size):
        super().__init__()
        self.src = src
        self.dst = dst
        # create dst file
        if os.path.isdir(dst):
            self.dst = os.path.join(dst, _basename(src))
        self.size = size
        self.signals = TransferSignals()
        self.running = 0
        # Give this job a unique ID.
        self.job_id = str(uuid.uuid4())

    @pyqtSlot()
    def run(self):
        # run the specified function
        if file_exists(self.src, self.dst):
            # end prematurely and emit dst
            print(f"File already exists '{self.dst}'")
            self.signals.duplicate.emit(self.dst)
            self.signals.progress.emit(self.job_id, 100)
            self.signals.finished.emit(self.job_id)

        else:
            self.copy(self.src, self.dst)

    def _copyfileobj_readinto(self, fsrc, fdst, length=1048576):
        """
        readinto()/memoryview()-based variant of copyfileobj()
        *fsrc* must support readinto() method and both files must be
        open in binary mode.
        """
        print(f"Transferring, method: readinto, buffer: {length}")
        progress = 0
        self.running = 1
        # localize variable access to minimize overhead
        fsrc_readinto = fsrc.readinto
        fdst_write = fdst.write
        with memoryview(bytearray(length)) as mv:
            try:
                while 1:
                    n = fsrc_readinto(mv)
                    if not n:
                        self.signals.finished.emit(self.job_id)
                        self.running = 0
                        print("Successful transfer")
                        return ON_SUCCESS

                    elif n < length:
                        with mv[:n] as smv:
                            fdst.write(smv)
                    else:
                        fdst_write(mv)
                    progress += n
                    percentage = (progress * 100) / self.size
                    self.signals.transferred.emit(self.job_id, progress)
                    self.signals.progress.emit(self.job_id, percentage)
                    # handle cancel
                    if not self.running:
                        print("Cancelled transfer")
                        self.signals.progress.emit(self.job_id, 100)
                        self.signals.finished.emit(self.job_id)
                        return ON_CANCEL
            except Exception as e:
                print(f"Error in transferring: {str(e)}")
                self.running = 0
                self.signals.progress.emit(self.job_id, 100)
                self.signals.finished.emit(self.job_id)
                return ON_ERROR

    def _copyfileobj(self, fsrc, fdst, length=1048576):
        """
        copy data from file-like object fsrc to file-like object fdst
        return success
        """
        print(f"Transferring, method: copy-buffer, buffer: {length}")
        progress = 0
        self.running = 1
        # localize variables to avoid overhead
        fsrc_read = fsrc.read
        fdst_write = fdst.write
        try:
            while 1:
                buff = fsrc_read(length)
                if not buff:
                    self.signals.finished.emit(self.job_id)
                    self.running = 0
                    # break and return success
                    print("Successful transfer")
                    return ON_SUCCESS

                fdst_write(buff)
                progress += len(buff)
                percentage = (progress * 100) / self.size
                self.signals.transferred.emit(self.job_id, progress)
                self.signals.progress.emit(self.job_id, percentage)
                # handle cancel
                if not self.running:
                    print("Cancelled transfer")
                    self.signals.progress.emit(self.job_id, 100)
                    self.signals.finished.emit(self.job_id)
                    return ON_CANCEL

        except Exception as e:
            print(f"Error in transferring: {str(e)}")
            self.running = 0
            self.signals.progress.emit(self.job_id, 100)
            self.signals.finished.emit(self.job_id)
            return ON_ERROR

    def _copyfile(self, src, dst):
        """ check if file exists, if same filesystem, else prepare file objects """

        try:
            # prepare file objects for read/write
            with open(src, 'rb') as fsrc:
                with open(dst, 'wb') as fdst:
                    if self.size > 0:
                        return self._copyfileobj_readinto(fsrc, fdst, length=min(1048576, self.size))
                    # copy files with 0 sizes
                    return self._copyfileobj(fsrc, fdst)
        except PermissionError:
            self.signals.progress.emit(self.job_id, 100)
            self.signals.finished.emit(self.job_id)
            return ON_ERROR

    def copy(self, src, dst):
        """ rename folders and prepare files for copying """

        done = self._copyfile(src, dst)
        if done == ON_SUCCESS:
            # copy file, then copy file stats
            self.copy_stat(src, dst)
        if done == ON_CANCEL:
            # clean up incomplete dst file

            print(f"Deleting incomplete file '{dst}'")
            delete_file(dst)
        return done

    def copy_stat(self, src, dst):
        try:
            shutil.copystat(src, dst)
        except Exception as e:
            print(f"Stats error: {e}")

def _basename(path):
    """ strip trailing slash and return basename """
    # A basename() variant which first strips the trailing slash, if present.
    # Thus we always get the last component of the path, even for directories.
    # borrowed from shutil.py
    sep = os.path.sep + (os.path.altsep or '')
    return os.path.basename(path.rstrip(sep))

def file_exists(src, dst) -> bool:
    """ compare file properties """
    if os.path.isdir(dst):
        dst = os.path.join(dst, _basename(src))
    if os.path.exists(dst):
        # return (os.path.getsize(src) == os.path.getsize(dst))
        return True
    return False

def delete_file(filename, trash=False):
    """
    permanently delete a file if `trash` is False
    """
    if trash:
        try:
            send2trash(filename)
        except TrashPermissionError as e:
            print(f"Cannot trash file: {e}")
    else:
        try:
            os.unlink(filename)
        except Exception as e:
            print(f"Cannot remove file: {e}")

def convert_bytes(num: float) -> str:
    """ format bytes to respective units for presentation (max GB) """
    try:
        if num >= 1073741824:
            return f"{round(num / 1073741824, 2)} GB"
        elif num >= 1048576:
            return f"{round(num / 1048576, 2)} MB"
        elif num >= 1024:
            return f"{round(num / 1024, 2)} KB"
        else:
            return f"{num} Bytes"
    except Exception:
        return "NaN"


#!/usr/bin/env python

from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument("paramFile",help="calculate the FED fragment sizes using the parameters in the given file")
parser.add_argument("relEventSize",type=float,help="scale the event size by this factor")
parser.add_argument("outputFile",help="file name of the output file")
args = vars(parser.parse_args())

eventSize = 0
with open(args['outputFile'],'w') as outputFile:
    outputFile.write("#fedId,size,rms\n")
    with open(args['paramFile']) as paramFile:
        for line in paramFile:
            try:
                (fedIds,a,b,c,rms) = line.split(',',5)
            except ValueError:
                (fedIds,a,b,c,rms,_) = line.split(',',6)
            try:
                for id in fedIds.split('+'):
                    fedSize = float(a)+float(b)*args['relEventSize']+float(c)*args['relEventSize']**2
                    outputFile.write("%d,%d,%d,1\n" % (int(id),int(fedSize)&~0x7,int(fedSize*float(rms))&~0x7))
                    eventSize += int(fedSize)
            except ValueError:
                pass

print("Total event size is %1.3f MB\n" % (float(eventSize)/1000000.))


from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import FunctionTransformer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
import pylab as py
import pickle
from scipy.sparse import hstack
from scipy.sparse import coo_matrix
from tqdm import tqdm
from scipy import sparse
import csv, random, numpy, os, re, scipy, gensim
from sklearn.neural_network import MLPClassifier
from sklearn import tree
from langdetect import detect
from sklearn.ensemble import RandomForestClassifier
from csv import DictReader
import numpy as np
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns
from xgboost import XGBClassifier

# import score

import nltk
import ssl

try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
    pass
else:
    ssl._create_default_https_context = _create_unverified_https_context

nltk.download('punkt')
nltk.download('wordnet')

class DataSet():
    def __init__(self, path=""):
        self.path = path

        print("Reading dataset")
        bodies = "train_bodies.csv"
        stances = "train_stances.csv"

        self.stances = self.read(stances)
        articles = self.read(bodies)
        self.articles = dict()

        #make the body ID an integer value
        for s in self.stances:
            s['Body ID'] = int(s['Body ID'])

        #copy all bodies into a dictionary
        for article in articles:
            self.articles[int(article['Body ID'])] = article['articleBody']

        print("Total stances: " + str(len(self.stances)))
        print("Total bodies: " + str(len(self.articles)))

    def read(self,filename):
        rows = []
        with open(self.path + "" + filename, "r", encoding='utf-8') as table:
            r = DictReader(table)

            for line in r:
                rows.append(line)
        return rows

dataset = DataSet()

def read_ids(file,base):
    ids = []
    with open(base+""+file,"r") as f:
        for line in f:
            ids.append(int(line))
        return ids

def split(dataset, base=""):
    if not (os.path.exists(""+"training_ids.txt")
            and os.path.exists(""+"dev_ids.txt") and os.path.exists(""+"test_ids.txt")):
        raise Exception("There is an error and the dataset reader cannot find the "
                        "{training_ids|test_ids|dev_ids}.txt file. Please make sure your python paths "
                        "are configured correctly")

    training_ids = read_ids("training_ids.txt",base)
    dev_ids = read_ids("dev_ids.txt",base)
    test_ids = read_ids("test_ids.txt",base)

    #return the stances that meet these criteria
    training_stances = []
    dev_stances = []
    test_stances = []

    for stance in dataset.stances:
        if stance['Body ID'] in training_ids:
            training_stances.append(stance)
        elif stance['Body ID'] in dev_ids:
            dev_stances.append(stance)
        elif stance['Body ID'] in test_ids:
            test_stances.append(stance)

    return {"training":training_stances, "dev":dev_stances, "test": test_stances}

print("\n[1] Loading data..")
data_splits = split(dataset)
# in the format: Stance, Headline, BodyID
training_data = data_splits['training']
dev_data = data_splits['dev']
test_data = data_splits['test'] # currently 0 test points

# Change the number of training examples used.
N = int(len(training_data) * 1.0)
training_data = training_data[:N]

print("\t-Training size:\t", len(training_data))
print("\t-Dev size:\t", len(dev_data))
print("\t-Test data:\t", len(test_data))

# Get the bodies of training data points
def get_bodies(data):
    bodies = []
    for i in range(len(data)):
        bodies.append(dataset.articles[data[i]['Body ID']])
    return bodies

# Get the headlines of training data points
def get_headlines(data):
    headlines = []
    for i in range(len(data)):
        headlines.append(data[i]['Headline'])
    return headlines

# Get bodies and headlines for dev and training data
training_bodies = get_bodies(training_data)
training_headlines = get_headlines(training_data)
dev_bodies = get_bodies(dev_data)
dev_headlines = get_headlines(dev_data)
test_bodies = get_bodies(test_data)
test_headlines = get_headlines(test_data)

print(len(training_bodies))
print(len(training_headlines))
print(len(dev_bodies))
print(len(dev_headlines))
print(len(test_bodies))
print(len(test_headlines))

# Function for extracting tf-idf vectors (for both the bodies and the headlines).
# https://www.kaggle.com/adamschroeder/countvectorizer-tfidfvectorizer-predict-comments

def extract_tfidf(training_headlines, training_bodies, dev_headlines="", dev_bodies="", test_headlines="", test_bodies=""):
    # Body vectorisation
    body_vectorizer = TfidfVectorizer(ngram_range=(1, 2), lowercase=True, stop_words='english')#, max_features=1024)
    bodies_tfidf = body_vectorizer.fit_transform(training_bodies)

    # Headline vectorisation
    headline_vectorizer = TfidfVectorizer(ngram_range=(1, 2), lowercase=True, stop_words='english')#, max_features=1024)
    headlines_tfidf = headline_vectorizer.fit_transform(training_headlines)

    with open('body_vectorizer.pk', 'wb') as b:
        pickle.dump(body_vectorizer, b)

    with open('headline_vectorizer.pk', 'wb') as h:
        pickle.dump(headline_vectorizer, h)

    # Tranform dev/test bodies and headlines using the trained vectorizer (trained on training data)
    bodies_tfidf_dev = body_vectorizer.transform(dev_bodies)
    headlines_tfidf_dev = headline_vectorizer.transform(dev_headlines)

    bodies_tfidf_test = body_vectorizer.transform(test_bodies)
    headlines_tfidf_test = headline_vectorizer.transform(test_headlines)

    feature_names = np.array(body_vectorizer.get_feature_names())
    sorted_by_idf = np.argsort(body_vectorizer.idf_)
    print('Features with lowest and highest idf in the body vector:\n')
    # The token which appears maximum times but it is also in all documents, has its idf the lowest
    print("Features with lowest idf:\n{}".format(
    feature_names[sorted_by_idf[:10]]))
    # The tokens can have the most idf weight because they are the only tokens that appear in one document only
    print("\nFeatures with highest idf:\n{}".format(
    feature_names[sorted_by_idf[-10:]]))

    # Combine body_tfdif with headline_tfidf for every data point.
    training_tfidf = scipy.sparse.hstack([bodies_tfidf, headlines_tfidf])
    dev_tfidf = scipy.sparse.hstack([bodies_tfidf_dev, headlines_tfidf_dev])
    test_tfidf = scipy.sparse.hstack([bodies_tfidf_test, headlines_tfidf_test])

    return training_tfidf, dev_tfidf, test_tfidf

lemmatizer = nltk.WordNetLemmatizer()

# Tokenisation, Normalisation, Capitalisation, Non-alphanumeric removal, Stemming-Lemmatization
def preprocess(string):
    # to lowercase, non-alphanumeric removal
    step1 = " ".join(re.findall(r'\w+', string, flags=re.UNICODE)).lower()
    step2 = [lemmatizer.lemmatize(t).lower() for t in nltk.word_tokenize(step1)]

    return step2

# Function for extracting word overlap
def extract_word_overlap(headlines, bodies):
    word_overlap = []
    for i, (headline, body) in tqdm(enumerate(zip(headlines, bodies))):
        preprocess_headline = preprocess(headline)
        preprocess_body = preprocess(body)

        # Lenght of common words b/w body and headline / Length of all the words of body & headline
        features = len(set(preprocess_headline).intersection(preprocess_body)) / float(len(set(preprocess_headline).union(preprocess_body)))
        word_overlap.append(features)

        # Convert the list to a sparse matrix (in order to concatenate the cos sim with other features)
        word_overlap_sparse = scipy.sparse.coo_matrix(numpy.array(word_overlap))
    return word_overlap_sparse

# Function for extracting the cosine similarity between bodies and headlines.
def extract_cosine_similarity(headlines, bodies):
    vectorizer = TfidfVectorizer(ngram_range=(1,2), lowercase=True, stop_words='english')#, max_features=1024)

    cos_sim_features = []
    for i in range(0, len(bodies)):
        body_vs_headline = []
        body_vs_headline.append(bodies[i])
        body_vs_headline.append(headlines[i])
        tfidf = vectorizer.fit_transform(body_vs_headline)

        cosine_similarity = (tfidf * tfidf.T).A
        cos_sim_features.append(cosine_similarity[0][1])

    # Convert the list to a sparse matrix (in order to concatenate the cos sim with other features)
    cos_sim_array = scipy.sparse.coo_matrix(numpy.array(cos_sim_features))

    return cos_sim_array

# Function for combining features of various types (lists, coo_matrix, np.array etc.)
def combine_features(tfidf_vectors, cosine_similarity, word_overlap):
    combined_features =  sparse.bmat([[tfidf_vectors, word_overlap.T, cosine_similarity.T]])
    return combined_features

# Function for extracting features
# Feautres: 1) Word Overlap, 2) TF-IDF vectors, 3) Cosine similarity, 4) Word embeddings
def extract_features(train, dev, test):
# Get bodies and headlines for dev and training data
    training_bodies = get_bodies(training_data)
    training_headlines = get_headlines(training_data)
    dev_bodies = get_bodies(dev_data)
    dev_headlines = get_headlines(dev_data)
    test_bodies = get_bodies(test_data)
    test_headlines = get_headlines(test_data)

    # Extract tfidf vectors
    print("\t-Extracting tfidf vectors..")
    training_tfidf, dev_tfidf, test_tfidf = extract_tfidf(training_headlines, training_bodies, dev_headlines, dev_bodies, test_headlines, test_bodies)
    print("\t-Tfidf vectors extracted..")

    # Extract word overlap
    print("\t-Extracting word overlap..")
    training_overlap = extract_word_overlap(training_headlines, training_bodies)
    dev_overlap = extract_word_overlap(dev_headlines, dev_bodies)
    test_overlap = extract_word_overlap(test_headlines, test_bodies)
    print("\t-Word overlap extracted..")

#     # Extract cosine similarity between bodies and headlines.
    print("\t-Extracting cosine similarity..")
    training_cos = extract_cosine_similarity(training_headlines, training_bodies)
    dev_cos = extract_cosine_similarity(dev_headlines, dev_bodies)
    test_cos = extract_cosine_similarity(test_headlines, test_bodies)
    print("\t-Cosine similarity extracted..")

    # Combine the features
    training_features = combine_features(training_tfidf, training_cos, training_overlap)
    dev_features = combine_features(dev_tfidf, dev_cos, dev_overlap)
    test_features = combine_features(test_tfidf, test_cos, test_overlap)
    print("\t-Combined features returned..")

    return training_features, dev_features, test_features

# Feature extraction
print("[2] Extracting features.. ")
# extract_features(training_data, dev_data, test_data)
training_features, dev_features, test_features = extract_features(training_data, dev_data, test_data)

classes = ['agree', 'disagree', 'discuss', 'unrelated']

def report_score(test,pred,algo):
    #accuracy calculation
    accuracy = accuracy_score(test,pred)
    print('\n Accuracy_score for %s = %s \n'%(algo,accuracy))

    #confusion_matrix
    mat = confusion_matrix(test,pred)
    fig, ax = plt.subplots(figsize=(11,11))
    ax.set_title("Confusion Matrix for %s" %algo)
    sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=True,cmap="icefire",
                xticklabels=classes,yticklabels=classes,linewidths=.3, ax=ax)
    plt.xlabel('True label')
    plt.ylabel('Predicted label');

    #classification report
    cls = classification_report(test,pred, target_names=classes)
    print(cls)

# Creating targets
targets_tr = [a['Stance'] for a in training_data]
targets_dev = [a['Stance'] for a in dev_data]
targets_test = [a['Stance'] for a in test_data]

def change(f):
    if f == 'unrelated':
        return 0
    elif f == 'disagree':
        return 1
    elif f == 'discuss':
        return 2
    elif f == 'agree':
        return 3
    else:
        return 0

y = [change(x) for x in targets_tr]
y_test = [change(x) for x in targets_test]

def predict(self, text):
        predicted = self.logR_pipeline.predict([text])
        predicedProb = self.logR_pipeline.predict_proba([text])[:,1]
        return bool(predicted), float(predicedProb)

# Fitting model
print("[3] Fitting model..")
print("\t-Logistic Regression")
lr = LogisticRegression(C = 1.0, class_weight='balanced', solver="lbfgs", max_iter=150)
y_pred_lr = lr.fit(training_features, targets_tr).predict(test_features)
# Evaluation
print("[4] Evaluating model..")
report_score(targets_test, y_pred_lr,'Logistic Regression')
print("\t-Done with Logistic Regression")

pickle.dump(lr, open('stance.pkl', 'wb'))
print('successfully saved model file')


#Question: Easy
"""
Given numBottles full water bottles, you can exchange numExchange empty water bottles for one full water bottle.

The operation of drinking a full water bottle turns it into an empty bottle.

Return the maximum number of water bottles you can drink.
"""
import logging

class Solution:
    def numWaterBottles(self, numBottles: int, numExchange: int) -> int:
        total = numBottles
        remainder = 0
        while True:
            temp_numBottles = numBottles
            numBottles = (remainder + numBottles) // numExchange
            total += numBottles
            remainder = temp_numBottles % numExchange
            if (numBottles+remainder) < numExchange:
                break
        return total

if __name__ == "__main__":
    format = '%(asctime)s: %(message)s'
    logging.basicConfig(format=format, level=logging.INFO, datefmt='%H:%M:%S')
    logging.getLogger().setLevel(logging.DEBUG)
    numBottles = 9
    numExchange = 3
    numBottles2 = 15
    numExchange2 = 4
    numBottles3 = 5
    numExchange3 =5
    numBottles4 = 2
    numExchange4 = 3
    numBottles5 = 15
    numExchange5 = 8
    result = Solution()
    logging.info(f'Amount of water drink: {result.numWaterBottles(numBottles2, numExchange2)}')


#!/usr/bin/env python3

import json
import os
import pprint
import random
import time

import taskcluster
from natsort import natsorted

from worker_health import tc_helpers

# class goal: see if a worker is working

class Status:
    tc_wm = None
    root_url = "https://firefox-ci-tc.services.mozilla.com"

    def __init__(self, provisioner, worker_type):
        self.provisioner = provisioner
        self.worker_type = worker_type

        with open(os.path.expanduser("~/.tc_token")) as json_file:
            data = json.load(json_file)
        creds = {"clientId": data["clientId"], "accessToken": data["accessToken"]}

        self.tc_wm = taskcluster.WorkerManager({"rootUrl": self.root_url, "credentials": creds})
        self.tc_h = tc_helpers.TCHelper(provisioner=self.provisioner)

    def wait_until_no_jobs_running(self, hosts, sleep_seconds=15, show_indicator=True):
        we_have_waited = False
        while True:
            jrd = self.get_hosts_running_jobs(hosts)
            if len(jrd) == 0:
                if show_indicator and we_have_waited:
                    print("")
                break
            time.sleep(sleep_seconds)
            we_have_waited = True
            if show_indicator:
                print(".", end="")

    # given list of hosts, return one that's idle (once one is available)
    def wait_for_idle_host(self, hosts, sleep_time=15):
        return random.choice(self.wait_for_idle_hosts(hosts, sleep_time))

    # given list of hosts, return those that are idle (once one is available)
    def wait_for_idle_hosts(self, hosts, sleep_time=15, show_indicator=True):
        hosts_set = set(hosts)
        while True:
            if show_indicator:
                print(".", end="", flush=True)
            hosts_with_non_completed_or_failed_jobs_set = set(self.get_hosts_running_jobs(hosts))
            hosts_idle = hosts_set - hosts_with_non_completed_or_failed_jobs_set
            if hosts_idle:
                if show_indicator:
                    print("")
                return list(hosts_idle)
            if show_indicator:
                print("z", end="", flush=True)
            time.sleep(sleep_time)

    def get_hosts_running_jobs(self, hosts, verbose=False):
        worker_groups = self.tc_h.get_worker_groups(self.worker_type)
        worker_group = worker_groups[0]

        hosts_with_non_completed_or_failed_jobs = []
        hosts_checked = []
        for host in hosts:
            hosts_checked.append(host)
            results = self.tc_h.get_worker_jobs(self.worker_type, worker_group, host)
            # pprint.pprint(results)
            if "recentTasks" not in results:
                # if no recent tasks, host is not running jobs
                if verbose:
                    print(f"get_hosts_running_jobs: {host}: no recentTasks")
                continue

            for result in results["recentTasks"]:
                task_id = result["taskId"]
                # pprint.pprint(task_id)
                _tid, status_blob, _exc = self.tc_h.get_task_status(task_id)
                # pprint.pprint(status_blob)
                try:
                    t_status = status_blob["status"]["state"]
                except KeyError:
                    # TODO: do additional checking if problems occur with this...
                    #   example response that we are catching here:
                    #
                    #      {'code': 'ResourceNotFound',
                    #       'message': '`NpY_UNDIRAW8Jaix-kc-SA` does not correspond to a task that '
                    #                  'exists.\n'
                    #                  'Are you sure this task exists?\n'
                    #                  '\n'
                    #                  '---\n'
                    #                  '\n'
                    #                  '* method:     status\n'
                    #                  '* errorCode:  ResourceNotFound\n'
                    #                  '* statusCode: 404\n'
                    #                  '* time:       2023-06-28T19:39:09.337Z',
                    #       'requestInfo': {'method': 'status',
                    #                       'params': {'taskId': 'NpY_UNDIRAW8Jaix-kc-SA'},
                    #                       'payload': {},
                    #                       'time': '2023-06-28T19:39:09.337Z'}}

                    # print(f"{task_id}: no task[status][state] assuming task is not running.")
                    continue
                if t_status != "completed" and t_status != "failed" and t_status != "exception":
                    # TODO: show task id and state if in verbose?
                    # pprint.pprint(status_blob["status"]["state"])
                    hosts_with_non_completed_or_failed_jobs.append(host)
        return hosts_with_non_completed_or_failed_jobs

    # TOOD: rename/decompose (fetch and present) this function
    def show_jobs_running_report(self, hosts):
        hosts_with_non_completed_or_failed_jobs = self.get_hosts_running_jobs(hosts)
        hosts_checked = hosts

        # TODO: move this print into get_hosts_running_jobs()
        #   - less useful now that it's just a len call vs the internal value from above?
        print(f"hosts checked ({len(hosts_checked)}): {hosts_checked}")
        print(
            f"hosts_with_non_completed_or_failed_jobs ({len(hosts_with_non_completed_or_failed_jobs)}): "
            f"{hosts_with_non_completed_or_failed_jobs}",
        )

        # return hosts not idle
        return hosts_with_non_completed_or_failed_jobs

    def list_workers_human(self):
        results = self.tc_wm.listWorkers(self.provisioner, self.worker_type)
        return_str = ""
        # TODO: fix natsorting, copy how _py does it?
        for result in natsorted(results["workers"]):
            return_str += f"{result['workerPoolId']} {result['workerGroup']} {result['workerId']}\n"
        print(return_str)
        return return_str

    def list_workers_csv(self):
        results = self.tc_wm.listWorkers(self.provisioner, self.worker_type)
        return_str = ""
        # TODO: fix natsorting, copy how _py does it?
        for result in natsorted(results["workers"]):
            return_str += f"{result['workerId']},"
        # trim trailing comma
        print(return_str[:-1])

    def list_workers_py(self):
        results = self.tc_wm.listWorkers(self.provisioner, self.worker_type)
        return_arr = []
        for result in results["workers"]:
            return_arr.append(result["workerId"])
        pprint.pprint(natsorted(return_arr))

if __name__ == "__main__":
    si = Status("releng-hardware", "gecko-t-osx-1015-r8")
    # example usage
    # si.show_jobs_running_report(
    #     [
    #         "macmini-r8-1",
    #         # "macmini-r8-2",
    #         # "macmini-r8-3",
    #         # "macmini-r8-4",
    #         # "macmini-r8-5",
    #         # "macmini-r8-7",
    #         # "macmini-r8-8",
    #         # "macmini-r8-9",
    #         # "macmini-r8-10",
    #     ],
    # )
    # for futher debugging
    # import ipdb; ipdb.set_trace()

    # TODO: pull out into binary `list_workers`
    # si.list_workers_human()
    # si.list_workers_csv()
    # si.list_workers_py()


import sys
sys.path.append('./')
sys.path.append('./../')
import openai, os, time, torch, sys, importlib, json, copy
import matplotlib.pyplot as plt
import numpy as np
from functools import partial
import numpy as np
from models import lora_gptj
from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score

from utils.helper import log

def get_accuracy(y_pred_val, y_val):
    acc_val = (np.array(y_pred_val) == np.array(y_val)).mean()
    acc_val = round(acc_val * 100, 2)
    return acc_val

class GPTJFineTuner(object):
    def __init__(self,config:dict,train_jsonl,valid_jsonl,cuda_idx = 0):
        self.config = config
        self.train_jsonl=train_jsonl
        self.valid_jsonl=valid_jsonl

        self.device = torch.device('cuda:%d' % cuda_idx) if torch.cuda.is_available() else 'cpu'
        torch.cuda.set_device(cuda_idx)

    def init_model(self):
        print('=====Initialize a new GPTJ Model=====')
        self.ft_model = lora_gptj.LoRaQGPTJ(adapter=True, device=self.device)

    def fine_tune(self):
        self.init_model()

    def generate(self, gpt, text_lst, max_token=10, batch_size=2):
        gpt.model.eval()
        outputs = []
        for i in np.arange(0, len(text_lst), batch_size):
            texts = text_lst[i:min(i + batch_size, len(text_lst))]
            prompt = gpt.tokenizer(texts, truncation=True, padding = True, max_length=1024, return_tensors='pt')
            prompt = {key: value.to(gpt.device) for key, value in prompt.items()}
            outs = gpt.model.generate(**prompt, max_new_tokens=max_token, pad_token_id=gpt.tokenizer.eos_token_id, do_sample=True, early_stopping = True)
            outs = gpt.tokenizer.batch_decode(outs, skip_special_tokens=True)
            outputs += outs
        return outputs

    def prompt2value(self, x):
        # print("Output:",x)
        c = x.strip().split('@@@')[0]
        return float(c)

    def query(self, gpt, prompts, bs=10):
        outputs = self.generate(gpt, prompts, batch_size=bs)
        ans = []
        for txt in outputs:
            try:
                output = self.prompt2value(txt.split('@@@')[-1].split('###')[0])
            except:
                output = None
            ans.append(output)
        return ans

    def eval(self,valid_prompts,valid_df,test_prompts,test_df,logf,y_name='y',train_df = None,imbalance=False,flip=False):
        """
            number of valid samples
            L2 error on the valid samples
        """
        y_valid_outputs_, y_test_outputs_, len_valid_valid_y_, val_acc_list, test_acc_list = [], [], [], [], []
        best_idx = 0
        for model_idx in range(len(self.config['epochs'])):
            config = copy.deepcopy(self.config)
            epochs_ran = 0 if model_idx == 0 else self.config['epochs'][model_idx-1]
            config['epochs'] = self.config['epochs'][model_idx] - epochs_ran
            print('==== Epoch %.4f ====' % self.config['epochs'][model_idx])
            self.ft_model.finetune(self.train_jsonl,
                self.valid_jsonl,
                config,
                saving_checkpoint = False)

            # validation
            y_valid_outputs = self.query(self.ft_model, valid_prompts, bs = 15)
            y_valid_outputs_.append(y_valid_outputs)

            valid_valid_y = [valid_df[y_name][i] for i in range(len(y_valid_outputs)) if y_valid_outputs[i] != None]
            valid_valid_y_outputs = [y_valid_outputs[i] for i in range(len(y_valid_outputs)) if y_valid_outputs[i] != None]

            len_valid_valid_y = len(valid_valid_y)
            print("| Valid Val #outputs/Total #outputs:%d/%d" % (len_valid_valid_y,len(y_valid_outputs)))
            len_valid_valid_y_.append(len_valid_valid_y)

            print(type(valid_valid_y_outputs), type(valid_valid_y))
            # from IPython import embed; embed()

            val_acc = get_accuracy(valid_valid_y_outputs, valid_valid_y)
            val_acc_list.append(val_acc)

            print('| Val Acc     : %.2f' % val_acc)
            if (val_acc < val_acc_list[best_idx]) or (np.isnan(val_acc_list[best_idx])):
                best_idx = model_idx

            # Testing
            y_test_outputs = self.query(self.ft_model, test_prompts, bs = 10)
            y_test_outputs_.append(y_test_outputs)

            valid_test_y = [test_df[y_name][i] for i in range(len(y_test_outputs)) if y_test_outputs[i] != None]
            valid_test_y_outputs = [y_test_outputs[i] for i in range(len(y_test_outputs)) if y_test_outputs[i] != None]
            print("Valid Test #outputs/Total #outputs:%d/%d" % (len(valid_test_y),len(y_test_outputs)))
            test_acc = get_accuracy(valid_test_y_outputs, valid_test_y)
            test_acc_list.append(test_acc)
            print('| Test Acc     : %.2f' % test_acc)

            if imbalance:
                if flip:
                    valid_valid_y = (-1*(valid_valid_y-1)).astype("int")
                    valid_valid_y_outputs = (-1*(valid_valid_y_outputs-1)).astype("int")
                    valid_test_y = (-1*(valid_test_y-1)).astype("int")
                    valid_test_y_outputs = (-1*(valid_test_y_outputs-1)).astype("int")

                precision_val = round(precision_score(valid_valid_y, valid_valid_y_outputs) * 100, 2)
                recall_val = round(recall_score(valid_valid_y, valid_valid_y_outputs) * 100, 2)
                f1_val = round(f1_score(valid_valid_y, valid_valid_y_outputs) * 100, 2)

                precision = round(precision_score(valid_test_y, valid_test_y_outputs) * 100, 2)
                recall = round(recall_score(valid_test_y, valid_test_y_outputs) * 100, 2)
                f1 = round(f1_score(valid_test_y, valid_test_y_outputs) * 100, 2)
                log(logf, f"val {self.config['epochs'][model_idx]} {val_acc} {f1_val} {precision_val} {recall_val}")
                log(logf, f"test {self.config['epochs'][model_idx]} {test_acc} {f1} {precision} {recall}")
            else:
                log(logf, f"{self.config['epochs'][model_idx]} {val_acc} {test_acc}")

        print('Selected epoch: %.4f' % self.config['epochs'][best_idx])
        self.best_idx = best_idx

        return y_test_outputs_[best_idx], len_valid_valid_y_,val_acc_list, test_acc_list


# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.

import random
import itertools
import sys

class Automaton:
    def __init__(self, atype=None, states=None, alphabet=None, instructions=None, initial_states=None):
        self.atype = type
        self.__states = set()
        self.__alphabet = set()
        if not instructions:
            self.__instructions = {}
        else:
            self.__instructions = instructions
        self.__initial_states = set()
        self.__curr_state = None
        self._tape = ''
        self._position = -1

    @property
    def alphabet(self):
        return self.__alphabet

    @alphabet.setter
    def alphabet(self, abet):
        self.__alphabet = set(abet)

    @property
    def initial_states(self):
        return self.__initial_states

    def set_initial_state(self, state):
        if state not in self.states:
            self.__states.add(state)
        self.__initial_states = {state}

    @initial_states.setter
    def initial_states(self, states):
        # states should be a set
        if not states <= self.__states:
            self.__states = self.__states | states
        self.__initial_states = states

    @property
    def states(self) -> set:
        return self.__states

    @states.setter
    def states(self, states):
        self.__states = set(states)

    def add_instruction(self, left, right):
        self.__instructions[left] = right

    @property
    def instructions(self):
        return self.__instructions

    @instructions.setter
    def instructions(self, instructions):
        self.__instructions = instructions

    @property
    def state(self):
        return self.__curr_state

    @state.setter
    def state(self, state):
        self.__curr_state = state

    def step(self):
        pass

    @property
    def configuration(self):
        return self.__curr_state, self._tape, self._position

    @configuration.setter
    def configuration(self, conf):
        if conf[0] not in self.__states:
            raise f"Wrong configuration {conf} - unknown state"
        self.__curr_state = conf[0]
        self._tape = conf[1]
        self._position = conf[2]

class nrNFAwtl(Automaton):
    def __init__(self, atype="nrNFAwtl", states=None, alphabet=None, instructions=None, initial_states=None):
        super().__init__(type, states, alphabet, instructions, initial_states)
        if instructions is None:
            instructions = {}
        if alphabet is None:
            alphabet = set()
        self._endmarker = '$'
        self.__transparency = {}
        if states:
            for st in states:
                self.__transparency[st] = set()

    @property
    def transparency(self) -> dict:
        return self.__transparency

    @transparency.setter
    def transparency(self, states_transparency):
        self.__transparency = states_transparency

    def step(self):
        while self._tape[self._position] in self.transparency[self.state]:
            self._position += 1
        symbol = self._tape[self._position]
        if symbol not in self.alphabet and symbol != self._endmarker:
            raise Exception(f"Symbol {symbol} is not in the automaton's alphabet")
        if (self.state, symbol) not in self.instructions:
            return "Reject"
        self.state = one_of(self.instructions[(self.state, symbol)])
        if self._tape[self._position] == self._endmarker:
            self._position = 0
        else:
            self._tape = self._tape[:self._position] + self._tape[self._position + 1:]
        return self.state

    def next_configs(self, state, tape, position):
        while tape[position] in self.transparency[state]:
            position += 1
        symbol = tape[position]
        if symbol not in self.alphabet and symbol != self._endmarker:
            raise Exception(f"Symbol {symbol} is not in the automaton's alphabet")
        if (state, symbol) not in self.instructions:
            return []
        next_states = self.instructions[(state, symbol)]
        if tape[position] == self._endmarker:
            next_position = 0
            next_tape = tape
        else:
            next_position = position
            next_tape = tape[:position] + tape[position + 1:]

        return [(st, next_tape, next_position) for st in next_states]

    def accepts_from_configuration(self, state, tape, position):
        if state == 'Accept':
            return True
        next_configs = self.next_configs(state, tape, position)
        for c in next_configs:
            if self.accepts_from_configuration(c[0], c[1], c[2]):
                return True
        return False

    def accepts(self, word):
        orig_recursion_limit = sys.getrecursionlimit()
        tape = word + self._endmarker
        if len(tape) > orig_recursion_limit + 20:
            sys.setrecursionlimit(len(tape) + 20)
        for state in self.initial_states:
            if self.accepts_from_configuration(state, tape, 0):
                if sys.getrecursionlimit() > orig_recursion_limit:
                    sys.setrecursionlimit(orig_recursion_limit)
                return True
        if sys.getrecursionlimit() > orig_recursion_limit:
            sys.setrecursionlimit(orig_recursion_limit)
        return False

def one_of(state_set):
    if not state_set:
        return None
    else:
        return random.choice(list(state_set))

def accepted_words(aut, max_len):
    res = []
    for l in range(max_len + 1):
        for w in itertools.product(aut.alphabet, repeat=l):
            word = ''.join(w)
            if aut.accepts(word):
                res.append(word)
    return res

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    aut = nrNFAwtl('nrNFAwtl')
    aut.alphabet = ['a', 'b', 'c']
    aut.states = ['q0', 'q1', 'q2', 'q3', 'q4', 'q5', 'q6', 'q7', 'q8', 'q9', 'q10', 'q11', 'qI', 'qI\'']
    aut.transparency = {'q0': {'a', 'c'}, 'q1': set(), 'q2': {'a', 'b'},
                        'q3': set(), 'q4': {'b', 'c'}, 'q5': set(),
                        'q6': {'c'}, 'q7': {'a'}, 'q8': {'b'},
                        'q9': {'c'}, 'q10': set(), 'q11': set(), 'qI': {'a'}, 'qI\'': set()}
    aut.set_initial_state('qI')
    print("Alphabet:", aut.alphabet)
    print("Set of states:", aut.states)
    print("Initial states:", aut.initial_states)

    aut.state = 'qI'
    print("Current state:", aut.state)

    aut.add_instruction(('qI', 'b'), {'q1'})
    aut.add_instruction(('qI', '$'), {'qI\''})
    aut.add_instruction(('qI\'', '$'), {'Accept'})
    aut.add_instruction(('q0', 'b'), {'q1'})
    aut.add_instruction(('q0', '$'), {'q6'})
    aut.add_instruction(('q1', 'c'), {'q2'})
    aut.add_instruction(('q2', 'c'), {'q3'})
    aut.add_instruction(('q2', '$'), {'q10'})
    aut.add_instruction(('q3', 'a'), {'q4'})
    aut.add_instruction(('q4', 'a'), {'q5'})
    aut.add_instruction(('q5', 'b'), {'q0'})
    aut.add_instruction(('q6', 'a'), {'q7'})
    aut.add_instruction(('q6', '$'), {'Accept'})
    aut.add_instruction(('q7', 'b'), {'q8'})
    aut.add_instruction(('q8', 'c'), {'q9'})
    aut.add_instruction(('q9', 'a'), {'q7'})
    aut.add_instruction(('q9', '$'), {'q0'})
    aut.add_instruction(('q10', 'a'), {'q11'})
    aut.add_instruction(('q11', '$'), {'Accept'})

    aut.configuration = ('qI', 'abc$', 0)

    state = aut.configuration[0]
    print(aut.configuration)
    while state != 'Reject' and state != 'Accept':
        state = aut.step()
        print(aut.configuration)

    print("End of computation:", state)

    aut.configuration = ('qI', 'abcabcabcabc$', 0)

    state = aut.configuration[0]
    print(aut.configuration)
    while state != 'Reject' and state != 'Accept':
        state = aut.step()
        print(aut.configuration)

    print('End of computation:', state)

    print('-------------------')

    print(aut.accepts('abc'))

    print(accepted_words(aut, 15))

    print('-------------------')
    print('Accepted numbers of repeats of "abc"')

    for k in range(1100):
        w = 'abc' * k
        if aut.accepts(w):
            print(k)

    print('-------------------------------')
    print('-- another example automaton --')
    print('-------------------------------')

# See PyCharm help at https://www.jetbrains.com/help/pycharm/