import requests
import ratelimit
from ratelimit import sleep_and_retry
import csv
import json
import enlighten
import os

@sleep_and_retry
@ratelimit.limits(calls=3, period=1)
def fetch_user_details(user):
    user_profile = "https://www.chiefdelphi.com/u/{user}.json"
    url = user_profile.format(user=user)
    try:
        profile = requests.get(url).json()
        with open('mf_users/' + user + '.json', 'w') as f:
            json.dump(profile, f)
    except:
        print("Problem:", url)

def fetch_user_files(users_csv='mf_users.csv'):
    mf_users = []
    with open(users_csv, newline='') as infile:
        for row in csv.reader(infile):
            mf_users.append(row[0])

    user_counter = enlighten.Counter(total=len(mf_users), desc="Users", unit="Users", color="blue", leave=False)
    for user in mf_users:
        fetch_user_details(user)
        user_counter.update()
    user_counter.close()

def process_mf_files(dir='mf_users/'):
    mf_data = []
    file_cnt = enlighten.Counter(total=len(os.listdir(dir)), desc="Users", unit="Users", color="blue", leave=False)
    for file in os.listdir(dir):
        try:
            user = json.load(open(dir + file))['user']
            if 'profile_hidden' in user.keys():
                mf_data.append({
                    'username': user['username'],
                    'name': user['name'],
                    'hidden': True
                })
            else:
                mf_data.append({
                    'username': user['username'],
                    'name': user['name'],
                    'hidden': False,
                    'team': user['user_fields']['1'],
                    'location': user['location'] if 'location' in user.keys() else '',
                    'rookie_year': user['user_fields']['2'],
                    'created': user['created_at'],
                    'seen': user['last_seen_at'],
                    'posted': user['last_posted_at'],
                    'level': user['trust_level'],
                    'reading_time': user['time_read'],
                    'profile_views': user['profile_view_count']
                })
        except:
            print("Problem:", file)
        file_cnt.update()

    with open('mf_data.csv', 'w', newline='') as outfile:
        dict_writer = csv.DictWriter(outfile, mf_data[0].keys())
        dict_writer.writeheader()
        dict_writer.writerows(mf_data)

if __name__ == '__main__':
    process_mf_files()


import os
from flask import Flask,render_template,request,flash,redirect,url_for

from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate

import pyshorteners
from urllib.parse import urlparse

from validators import url as validate_url

app=Flask(__name__)

#######################SQLALCHEMY CONFIGURATION#############################

basedir = os.path.abspath(os.path.dirname(__file__))
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'data.sqlite')
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

db = SQLAlchemy(app)  #passing application to sqlalchemy this creates object to sqlalchemy
Migrate(app, db)        #migrate flask object and sqlalchemy object

############################################################################
###############################creating tabkle/model#########################

class Url_shortner(db.Model):
    __tablename__='URL' #defining table name as grocery

    #definning columns of table
    id=db.Column(db.Integer, primary_key = True)
    url=db.Column(db.Text,unique=True)
    short_url=db.Column(db.Text,unique=True)

    #creating a constructor
    def __init__(self, url, short_url):
        self.url=url
        self.short_url=short_url

    def __repr__(self):
        return "url name is {} and short url is {}".format(self.url,self.short_url)

@app.route('/')
def index():
    return render_template('Home.html')

@app.route('/add',methods=['GET','POST'])
def add_input():
    url=''
    short_url=''
    if request.method=='POST':
        url=request.form.get('url_L')

        # check if input is a valid url
        if not validate_url(url):
            return "Invalid Url!Go back to our User Input page anbd enter a valid url"

        else:
    #check whether url already exist in databse
            found_url =Url_shortner.query.filter_by(url=url).first()
            if found_url:
                return render_template('input.html',short=found_url.short_url)

            else:
                short_url = pyshorteners.Shortener().tinyurl.short(url)

        # appending input values to database created
                new_url = Url_shortner(url, short_url)
                db.session.add(new_url)
                db.session.commit()
    return render_template('input.html',short=short_url)

@app.route('/searchitem',methods=['GET','POST'])
def find():
    if request.method=='POST':
        url = request.form.get('item')

        if not validate_url(url):
            return "Invalid Url!Go back and enter the correct url"
        else:
            item = Url_shortner.query.filter_by(url=url).first()
            if item:
                return render_template('search.html', item=item.short_url)
            else:

                return "Url not found!Use our User input page to shorten you url"

    return render_template('search.html')

@app.route('/display',methods=['Get','POST'])
def display_items():
    items=Url_shortner.query.all()

    if request.method=='POST':
        db.session.query(Url_shortner).delete()
        db.session.commit()
        return redirect(url_for('display_items'))

    return render_template('displays.html',items=items)

if __name__=='__main__':
    app.run(debug=True)


import numpy as np

def savgol_filter(y, window_size, degree): #Savitzky-Golay Filter
    y_smooth = []
    for x in range(len(y)):
        a = int(x - window_size / 2 if x >= window_size / 2 else 0)
        b = int(x + window_size / 2 if x + window_size < len(y) else len(y))
        p_vec = np.polyfit(range(a, b), y[a:b], degree)
        p = lambda x: np.dot([x ** d for d in range(degree + 1)], np.flip(p_vec))
        y_smooth.append(p(x))

    return y_smooth

if __name__ == "__main__":
    import matplotlib.pyplot as plt

    window_size = 50
    degree = 3

    x = np.linspace(0,2*np.pi,100)
    y = np.sin(x) + np.random.random(100) * 0.2
    yhat = savgol_filter(y, window_size, degree)

    plt.plot(x,y)
    plt.plot(x,yhat, color='red')
    plt.show()


from django.conf import settings
from django.template.loader import render_to_string

def analytics(request):
    """
    1. Returns analytics code for Google Analytics
    2. Also returns flashapp id for macromeme generator
    3. Returns standard js files used
    """
    if not settings.DEBUG:
        # production
        return { 'analytics_code': render_to_string("processors/analytics.html", { 'GOOGLE_ANALYTICS_KEY': settings.GOOGLE_ANALYTICS_KEY}), 'flashappid': '320238601417173','standard_js': render_to_string("processors/standard_js.html", { 'STATIC_URL': settings.STATIC_URL})}
    else:
        return { 'analytics_code': "", 'flashappid': '479174175436255', 'standard_js': render_to_string("processors/standard_js.html", { 'STATIC_URL': settings.STATIC_URL})}


from rest_framework import serializers
from .models import Order

from django.forms.models import model_to_dict

from rest_framework.utils import html, model_meta, representation

class OrderSerializer(serializers.ModelSerializer):

    # def _user(self, obj):
    #     request = self.context.get('request', None)
    #     if request:
    #         return model_to_dict(request.user)

    # owner = serializers.SerializerMethodField('_user')
    # owner = serializers.SlugRelatedField(
    #     many=True,
    #     read_only=True,
    #     slug_field='username'
    # )

    owner = serializers.StringRelatedField(many=False)
    id = serializers.IntegerField(read_only=True)

    class Meta:

        model = Order
        # fields = '__all__'
        fields = ['id', 'resquest_date', 'product', 'quantity', 'owner']
        read_only_fields = (('resquest_date', True))
        depth = 1

    def create(self, validated_data):

        ModelClass = self.Meta.model
        info = model_meta.get_field_info(ModelClass)

        for field_name, relation_info in info.relations.items():
            if relation_info.to_many and (field_name in validated_data):
                print(field_name, " - ", validated_data.pop(field_name))
                # many_to_many[field_name] = validated_data.pop(field_name)

        aux = super().create(validated_data)

        # set owner to current user (the one that made the request)
        aux.owner = self.context['request'].user
        aux.save()

        return aux

    # def save(self, **kwargs):
    #     aux = super().save(**kwargs)

    #     print("\n\nolha o owner", aux.owner)

    #     return aux


from django.core import signals
from django.core.exceptions import (
    PermissionDenied, SuspiciousOperation,
)
from django.http.multipartparser import MultiPartParserError
from django.urls import get_resolver, get_urlconf

import contextlib
import importlib
import json
import logging
import time
from typing import List, Union

import channels
import otree.channels.utils as channel_utils
from asgiref.sync import async_to_sync
from channels.layers import get_channel_layer

import redis_lock
import vanilla
from django.conf import settings
from django.urls import resolve
from django.db.models import Max, Min
from django.http import HttpResponseRedirect, Http404, HttpResponse
from django.shortcuts import get_object_or_404
from django.template.response import TemplateResponse
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext as _, ugettext_lazy
from django.views.decorators.cache import never_cache, cache_control
import idmap
import otree.common_internal
import otree.constants_internal as constants
import otree.db.idmap
import otree.forms
import otree.models
import otree.timeout.tasks
from otree.bots.bot import bot_prettify_post_data, ExpectError
import otree.bots.browser as browser_bots
from otree.common_internal import (
    get_app_label_from_import_path, get_dotted_name, get_admin_secret_code,
    DebugTable, BotError, wait_page_thread_lock, ResponseForException
)
from otree.models import (
    Participant, Session, BasePlayer, BaseGroup, BaseSubsession)
from otree.models_concrete import (
    PageCompletion, CompletedSubsessionWaitPage,
    CompletedGroupWaitPage, PageTimeout, UndefinedFormModel,
    ParticipantLockModel, ParticipantToPlayerLookup
)
from django.core.handlers.exception import handle_uncaught_exception

logger = logging.getLogger(__name__)

UNHANDLED_EXCEPTIONS = (
    Http404, PermissionDenied, MultiPartParserError,
    SuspiciousOperation, SystemExit
)

# make the technical 500 page auto-reload when the server restarts
# when the websocket reconnects, that means the server must have restarted.
# hardcode path to reconnecting-websocket because
# can't use Django template tags because template is already rendered
TECHNICAL_500_AUTORELOAD_JS = b'''

Lost server connection...

'''

def response_for_exception(request, exc):
    '''simplified from Django 1.11 source.
    The difference is that we use the exception that was passed in,
    rather than referencing sys.exc_info(), which gives us the ResponseForException
    the original exception was wrapped in, which we don't want to show to users.
        '''
    if isinstance(exc, UNHANDLED_EXCEPTIONS):
        '''copied from Django source, but i don't think these
        exceptions will actually occur.'''
        raise exc
    signals.got_request_exception.send(sender=None, request=request)
    exc_info = (type(exc), exc, exc.__traceback__)
    response = handle_uncaught_exception(
        request, get_resolver(get_urlconf()), exc_info)
    if settings.DEBUG:
        response_content = response.content.split(b'
')[0]
        response_content += TECHNICAL_500_AUTORELOAD_JS
        response.content = response_content

    # Force a TemplateResponse to be rendered.
    if not getattr(response, 'is_rendered', True) and callable(getattr(response, 'render', None)):
        response = response.render()

    return response

NO_PARTICIPANTS_LEFT_MSG = (
    "The maximum number of participants for this session has been exceeded.")

ADMIN_SECRET_CODE = get_admin_secret_code()

def get_view_from_url(url):
    view_func = resolve(url).func
    module = importlib.import_module(view_func.__module__)
    Page = getattr(module, view_func.__name__)
    return Page

@contextlib.contextmanager
def participant_scoped_db_lock(participant_code):
    '''
    prevent the same participant from executing the page twice
    use this instead of a transaction because it's more lightweight.
    transactions make it harder to reason about wait pages
    '''
    TIMEOUT = 10
    start_time = time.time()
    while time.time() - start_time < TIMEOUT:
        updated_locks = ParticipantLockModel.objects.filter(
            participant_code=participant_code,
            locked=False
        ).update(locked=True)
        if not updated_locks:
            time.sleep(0.2)
        else:
            try:
                yield
            finally:
                ParticipantLockModel.objects.filter(
                    participant_code=participant_code,
                ).update(locked=False)
            return
    exists = ParticipantLockModel.objects.filter(
        participant_code=participant_code
    ).exists()
    if not exists:
        raise Http404((
            "This user ({}) does not exist in the database. "
            "Maybe the database was recreated."
        ).format(participant_code))

    # could happen if the request that has the lock is paused somehow,
    # e.g. in a debugger
    raise Exception(
        'Another HTTP request has the lock for participant {}.'.format(
            participant_code))

def get_redis_lock(*, name='global'):
    if otree.common_internal.USE_REDIS:
        return redis_lock.Lock(
            redis_client=otree.common_internal.get_redis_conn(),
            name='OTREE_LOCK_{}'.format(name),
            expire=10,
            auto_renewal=True
        )

BOT_COMPLETE_HTML_MESSAGE = '''

        <title>Bot completed</title>

    Bot completed

'''

class FormPageOrInGameWaitPage(vanilla.View):
    """
    View that manages its position in the group sequence.
    for both players and experimenters
    """

    template_name = None

    is_debug = settings.DEBUG

    def inner_dispatch(self):
        '''inner dispatch function'''
        raise NotImplementedError()

    def get_template_names(self):
        raise NotImplementedError()

    @classmethod
    def url_pattern(cls, name_in_url):
        p = r'^p/(?P<participant_code>\w+)/{}/{}/(?P<page_index>\d+)/$'.format(
            name_in_url,
            cls.__name__,
        )
        return p

    @classmethod
    def get_url(cls, participant_code, name_in_url, page_index):
        '''need this because reverse() is too slow in create_session'''
        return r'/p/{pcode}/{name_in_url}/{ClassName}/{page_index}/'.format(
            pcode=participant_code, name_in_url=name_in_url,
            ClassName=cls.__name__, page_index=page_index
        )

    @classmethod
    def url_name(cls):
        '''using dots seems not to work'''
        return get_dotted_name(cls).replace('.', '-')

    def _redirect_to_page_the_user_should_be_on(self):
        return HttpResponseRedirect(self.participant._url_i_should_be_on())

    @method_decorator(never_cache)
    @method_decorator(cache_control(must_revalidate=True, max_age=0,
                                    no_cache=True, no_store=True))
    def dispatch(self, request, participant_code, **kwargs):

        if otree.common_internal.USE_REDIS:
            lock = redis_lock.Lock(
                otree.common_internal.get_redis_conn(),
                participant_code,
                expire=60,
                auto_renewal=True
            )
        else:
            lock = participant_scoped_db_lock(participant_code)

        with lock, otree.db.idmap.use_cache():
            try:
                participant = Participant.objects.get(
                    code=participant_code)
            except Participant.DoesNotExist:
                msg = (
                    "This user ({}) does not exist in the database. "
                    "Maybe the database was recreated."
                ).format(participant_code)
                raise Http404(msg)

            # if the player tried to skip past a part of the subsession
            # (e.g. by typing in a future URL)
            # or if they hit the back button to a previous subsession
            # in the sequence.
            url_should_be_on = participant._url_i_should_be_on()
            if not self.request.path == url_should_be_on:
                return HttpResponseRedirect(url_should_be_on)

            self.set_attributes(participant)

            try:
                response = self.inner_dispatch()
                # need to render the response before saving objects,
                # because the template might call a method that modifies
                # player/group/etc.
                if hasattr(response, 'render'):
                    response.render()
            except (ResponseForException, ExpectError) as exc:
                response = response_for_exception(
                    self.request, exc.__cause__ or exc.__context__
                )
            except Exception as exc:
                # this is still necessary, e.g. if an attribute on the page
                # is invalid, like form_fields, form_model, etc.
                response = response_for_exception(self.request, exc)

            otree.db.idmap.save_objects()
            if self.participant.is_browser_bot:
                html = response.content.decode('utf-8')
                # 2018-04-25: not sure why i didn't use an HTTP header.
                # the if statement doesn't even seem to make a difference.
                # shouldn't it always submit, if we're in a Page class?
                # or why not just set an attribute directly on the response object?
                # OTOH, this is pretty guaranteed to work. whereas i'm not sure
                # that we can isolate the exact set of cases when we have to
                # add the auto-submit flag (only GET requests, not wait pages,
                # ...etc?)
                if 'browser-bot-auto-submit' in html:
                    # needs to happen in GET, so that we can set the .html
                    # attribute on the bot.
                    browser_bots.set_attributes(
                        participant_code=self.participant.code,
                        request_path=self.request.path,
                        html=html,
                    )
            return response

    def get_context_data(self, **context):

        context.update(
            view=self,
            object=getattr(self, 'object', None),
            player=self.player,
            group=self.group,
            subsession=self.subsession,
            session=self.session,
            participant=self.participant,
            Constants=self._Constants,
            timer_text=getattr(self, 'timer_text', None)
        )

        views_module = otree.common_internal.get_pages_module(
            self.subsession._meta.app_config.name)
        if hasattr(views_module, 'vars_for_all_templates'):
            vars_for_template = views_module.vars_for_all_templates(self)
        else:
            vars_for_template = {}

        try:
            user_vars = self.vars_for_template()
            context['javascript_vars'] = self.javascript_vars().items()
        except:
            raise ResponseForException

        vars_for_template.update(user_vars or {})

        context.update(vars_for_template)

        if settings.DEBUG:
            self.debug_tables = self._get_debug_tables(vars_for_template)
        return context

    def render_to_response(self, context):
        """
        Given a context dictionary, returns an HTTP response.
        """
        return TemplateResponse(
            request=self.request,
            template=self.get_template_names(),
            context=context
        )

    def vars_for_template(self):
        return {}

    def javascript_vars(self):
        return {}

    def _get_debug_tables(self, vars_for_template):
        try:
            group_id = self.group.id_in_subsession
        except:
            group_id = ''

        tables = []
        if vars_for_template:
            # use repr() so that we can distinguish strings from numbers
            # and can see currency types, etc.
            items = [(k, repr(v)) for (k, v) in vars_for_template.items()]
            rows = sorted(items)
            tables.append(DebugTable(title='Vars for template', rows=rows))

        basic_info_table = DebugTable(
            title='Basic info',
            rows=[
                ('ID in group', self.player.id_in_group),
                ('Group', group_id),
                ('Round number', self.subsession.round_number),
                ('Participant', self.player.participant._id_in_session()),
                ('Participant label', self.player.participant.label or ''),
                ('Session code', self.session.code)
            ]
        )

        tables.append(basic_info_table)

        return tables

    def _load_all_models(self):
        '''Load all model instances into idmap cache'''
        self.PlayerClass.objects.select_related(
            'group', 'subsession', 'session'
        ).get(pk=self._player_pk)

    def _is_displayed(self):
        try:
            is_displayed = self.is_displayed()
        except:
            raise ResponseForException
        if is_displayed is None:
            msg = (
                '{}: is_displayed() did not return anything. '
                'You should fix this. To show the page, you should return True; '
                'to skip the page, return False.'
            ).format(self.__class__.__name__)
            logger.warning(msg)
        return is_displayed

    @property
    def player(self) -> BasePlayer:
        # NOTE:
        # these properties look in the idmap cache, so they don't touch
        # the database if they are already loaded
        return self.PlayerClass.objects.get(pk=self._player_pk)

    @property
    def group(self) -> BaseGroup:
        '''can't cache self._group_pk because group can change'''
        return self.player.group

    @property
    def subsession(self) -> BaseSubsession:
        return self.SubsessionClass.objects.get(pk=self._subsession_pk)

    @property
    def participant(self) -> Participant:
        return Participant.objects.get(pk=self._participant_pk)

    @property
    def session(self) -> Session:
        return Session.objects.get(pk=self._session_pk)

    _round_number = None
    @property
    def round_number(self):
        if self._round_number is None:
            self._round_number = self.subsession.round_number
        return self._round_number

    def set_attributes(self, participant, lazy=False):

        player_lookup = participant.player_lookup()

        app_name = player_lookup['app_name']

        models_module = otree.common_internal.get_models_module(app_name)
        self._Constants = models_module.Constants
        self.PlayerClass = getattr(models_module, 'Player')
        self.GroupClass = getattr(models_module, 'Group')
        self.SubsessionClass = getattr(models_module, 'Subsession')
        self._player_pk = player_lookup['player_pk']
        self._subsession_pk = player_lookup['subsession_pk']
        self._session_pk = player_lookup['session_pk']
        self._participant_pk = participant.pk

        # it's already validated that participant is on right page
        self._index_in_pages = participant._index_in_pages

        # for the participant changelist
        participant._current_app_name = app_name
        participant._current_page_name = self.__class__.__name__
        participant._last_request_timestamp = time.time()

        if not lazy:
            self._load_all_models()
            self.participant._round_number = self.player.round_number

        self._is_frozen = True

    # python 3.5 type hint
    def set_attributes_waitpage_clone(self, *, original_view: 'WaitPage'):
        '''put it here so it can be compared with set_attributes...
        but this is really just a method on wait pages'''

        self._Constants = original_view._Constants
        self.PlayerClass = original_view.PlayerClass
        self.GroupClass = original_view.GroupClass
        self.SubsessionClass = original_view.SubsessionClass
        self._subsession_pk = original_view._subsession_pk
        self._session_pk = original_view._session_pk
        self._participant_pk = original_view._participant_pk

        # is this needed?
        self._index_in_pages = original_view._index_in_pages

    def _increment_index_in_pages(self):
        # when is this not the case?
        assert self._index_in_pages == self.participant._index_in_pages

        self._record_page_completion_time()
        # we should allow a user to move beyond the last page if it's mturk
        # also in general maybe we should show the 'out of sequence' page

        # we skip any page that is a sequence page where is_displayed
        # evaluates to False to eliminate unnecessary redirection

        page_index_to_skip_to = self._get_next_page_index_if_skipping_apps()
        is_skipping_apps = bool(page_index_to_skip_to)

        for page_index in range(
                # go to max_page_index+2 because range() skips the last index
                # and it's possible to go to max_page_index + 1 (OutOfRange)
                self._index_in_pages+1, self.participant._max_page_index+2):
            self.participant._index_in_pages = page_index
            if page_index == self.participant._max_page_index+1:
                # break and go to OutOfRangeNotification
                break
            if is_skipping_apps and page_index == page_index_to_skip_to:
                break

            url = self.participant._url_i_should_be_on()

            Page = get_view_from_url(url)
            page = Page()

            page.set_attributes(self.participant, lazy=True)
            if (not is_skipping_apps) and page._is_displayed():
                break

            # if it's a wait page, record that they visited
            # but don't run after_all_players_arrive
            if isinstance(page, WaitPage):

                if page.group_by_arrival_time:
                    # keep looping
                    # if 1 participant can skip the page,
                    # then all other participants should skip it also,
                    # as described in the docs
                    # so there is no need to mark as complete.
                    continue

                # save the participant, because tally_unvisited
                # queries index_in_pages directly from the DB
                # this fixes a bug reported on 2016-11-04 on the mailing list
                self.participant.save()
                # you could just return page.dispatch(),
                # but that could cause deep recursion

                unvisited = page._get_unvisited_ids()
                if not unvisited:
                    if page.wait_for_all_groups:
                        group = None
                    else:
                        group = self.group
                    page._mark_completed_and_notify(group=group)
                    # we don't run after_all_players_arrive()

    def is_displayed(self):
        return True

    def app_after_this_page(self, upcoming_apps):
        pass

    def _get_next_page_index_if_skipping_apps(self):
        # don't run it if the page is not displayed, because:
        # (1) it's consistent with other functions like before_next_page, vars_for_template
        # (2) then when we do
        # a lookahead skipping pages, we would need to check each page if it
        # has app_after_this_page defined, then set attributes and run it.
        # what if we are already skipping to a future app, then another page
        # has app_after_this_page? does it override the first one?
        if not self._is_displayed():
            return

        current_app = self.participant._current_app_name
        app_sequence = self.session.config['app_sequence']
        current_app_index = app_sequence.index(current_app)
        upcoming_apps = app_sequence[current_app_index+1:]

        app_to_skip_to = self.app_after_this_page(upcoming_apps)
        if app_to_skip_to:
            if app_to_skip_to not in upcoming_apps:
                raise InvalidAppError(
                    f'"{app_to_skip_to}" is not in the upcoming_apps list'
                )
            return (
                ParticipantToPlayerLookup.objects
                    .filter(participant=self.participant, app_name=app_to_skip_to)
                    .aggregate(Min('page_index'))
            )['page_index__min']

    def _record_page_completion_time(self):

        now = int(time.time())

        last_page_timestamp = self.participant._last_page_timestamp
        if last_page_timestamp is None:
            logger.warning(
                'Participant {}: _last_page_timestamp is None'.format(
                    self.participant.code))
            last_page_timestamp = now

        seconds_on_page = now - last_page_timestamp

        self.participant._last_page_timestamp = now
        page_name = self.__class__.__name__

        timeout_happened = bool(getattr(self, 'timeout_happened', False))

        PageCompletion.objects.create(
            app_name=self.subsession._meta.app_config.name,
            page_index=self._index_in_pages,
            page_name=page_name, time_stamp=now,
            seconds_on_page=seconds_on_page,
            subsession_pk=self.subsession.pk,
            participant=self.participant,
            session=self.session,
            auto_submitted=timeout_happened)
        self.participant.save()

    _is_frozen = False

    _setattr_whitelist = {
        '_is_frozen',
        'object',
        'form',
        'timeout_happened',
        # i should send some of these through context
        '_remaining_timeout_seconds',
        'first_field_with_errors',
        'other_fields_with_errors',
        'debug_tables',
        '_round_number',
        'request' # this is just used in a test case mock.
    }

    def __setattr__(self, attr: str, value):
        if self._is_frozen and not attr in self._setattr_whitelist:
            msg = (
                'You set the attribute "{}" on the page {}. '
                'Setting attributes on page instances is not permitted. '
            ).format(attr, self.__class__.__name__)
            raise AttributeError(msg)
        else:
            # super() is a bit slower but only gets run during __init__
            super().__setattr__(attr, value)

class Page(FormPageOrInGameWaitPage):

    # if a model is not specified, use empty "StubModel"
    form_model = UndefinedFormModel
    form_fields = []

    def inner_dispatch(self):
        if self.request.method == 'POST':
            return self.post()
        return self.get()

    def get(self):
        if not self._is_displayed():
            self._increment_index_in_pages()
            return self._redirect_to_page_the_user_should_be_on()

        # this needs to be set AFTER scheduling submit_expired_url,
        # to prevent race conditions.
        # see that function for an explanation.
        self.participant._current_form_page_url = self.request.path
        self.object = self.get_object()
        form = self.get_form(instance=self.object)
        context = self.get_context_data(form=form)
        return self.render_to_response(context)

    def get_template_names(self):
        if self.template_name is not None:
            return [self.template_name]
        return ['{}/{}.html'.format(
            get_app_label_from_import_path(self.__module__),
            self.__class__.__name__)]

    def get_form_fields(self):
        return self.form_fields

    def _get_form_model(self):
        form_model = self.form_model
        if isinstance(form_model, str):
            if form_model == 'player':
                return self.PlayerClass
            if form_model == 'group':
                return self.GroupClass
            raise ValueError(
                "'{}' is an invalid value for form_model. "
                "Try 'player' or 'group' instead.".format(form_model)
            )
        return form_model

    def get_form_class(self):
        try:
            fields = self.get_form_fields()
        except:
            raise ResponseForException
        form_model = self._get_form_model()
        if form_model is UndefinedFormModel and fields:
            raise Exception(
                'Page "{}" defined form_fields but not form_model'.format(
                    self.__class__.__name__
                )
            )
        return otree.forms.modelform_factory(
            form_model, fields=fields,
            form=otree.forms.ModelForm,
            formfield_callback=otree.forms.formfield_callback)

    def before_next_page(self):
        pass

    def get_form(self, data=None, files=None, **kwargs):
        """Given `data` and `files` QueryDicts, and optionally other named
        arguments, and returns a form.
        """

        cls = self.get_form_class()
        return cls(data=data, files=files, view=self, **kwargs)

    def form_invalid(self, form):
        context = self.get_context_data(form=form)

        fields_with_errors = [
            fname for fname in form.errors
            if fname != '__all__']

        # i think this should be before we call render_to_response
        # because the view (self) is passed to the template and rendered
        if fields_with_errors:
            self.first_field_with_errors = fields_with_errors[0]
            self.other_fields_with_errors = fields_with_errors[1:]

        response = self.render_to_response(context)
        response[constants.redisplay_with_errors_http_header] = (
            constants.get_param_truth_value)

        return response

    def post(self):
        request = self.request

        self.object = self.get_object()

        if self.participant.is_browser_bot:
            submission = browser_bots.get_next_post_data(
                participant_code=self.participant.code)
            if submission is None:
                browser_bots.send_completion_message(
                    session_code=self.session.code,
                    participant_code=self.participant.code
                )
                return HttpResponse(BOT_COMPLETE_HTML_MESSAGE)
            else:
                # convert MultiValueKeyDict to regular dict
                # so that we can add entries to it in a simple way
                # before, we used dict(request.POST), but that caused
                # errors with BooleanFields with blank=True that were
                # submitted empty...it said [''] is not a valid value
                post_data = request.POST.dict()
                post_data.update(submission)
        else:
            post_data = request.POST

        form = self.get_form(
                data=post_data, files=request.FILES, instance=self.object)
        self.form = form

        auto_submitted = request.POST.get(constants.timeout_happened)

        # if the page doesn't have a timeout_seconds, only the timeoutworker
        # should be able to auto-submit it.
        # otherwise users could append timeout_happened to the URL to skip pages
        has_secret_code = (
            request.POST.get(constants.admin_secret_code) == ADMIN_SECRET_CODE)

        # todo: make sure users can't change the result by removing 'timeout_happened'
        # from URL
        if auto_submitted and (has_secret_code or self.has_timeout_()):
            self.timeout_happened = True  # for public API
            self._process_auto_submitted_form(form)
        else:
            self.timeout_happened = False
            is_bot = self.participant._is_bot
            if form.is_valid():
                if is_bot and post_data.get('must_fail'):
                    raise BotError(
                        'Page "{}": Bot tried to submit intentionally invalid '
                        'data with '
                        'SubmissionMustFail, but it passed validation anyway:'
                        ' {}.'.format(
                            self.__class__.__name__,
                            bot_prettify_post_data(post_data)))
                # assigning to self.object is not really necessary
                self.object = form.save()
            else:
                response = self.form_invalid(form)
                if is_bot:
                    PageName = self.__class__.__name__
                    if not post_data.get('must_fail'):
                        errors = [
                            "{}: {}".format(k, repr(v))
                            for k, v in form.errors.items()]
                        raise BotError(
                            'Page "{}": Bot submission failed form validation: {} '
                            'Check your bot code, '
                            'then create a new session. '
                            'Data submitted was: {}'.format(
                                PageName,
                                errors,
                                bot_prettify_post_data(post_data),
                            ))
                    if post_data.get('error_fields'):
                        # need to convert to dict because MultiValueKeyDict
                        # doesn't properly retrieve values that are lists
                        post_data_dict = dict(post_data)
                        expected_error_fields = set(post_data_dict['error_fields'])
                        actual_error_fields = set(form.errors.keys())
                        if not expected_error_fields == actual_error_fields:
                            raise BotError(
                                'Page {}, SubmissionMustFail: '
                                'Expected error_fields were {}, but actual '
                                'error_fields are {}'.format(
                                    PageName,
                                    expected_error_fields,
                                    actual_error_fields,
                                )
                            )
                return response
        try:
            self.before_next_page()
        except Exception as exc:
            # why not raise ResponseForException?
            return response_for_exception(self.request, exc)

        if self.participant.is_browser_bot:
            if self._index_in_pages == self.participant._max_page_index:
                # fixme: is it right to set html=''?
                # could this break any asserts?
                browser_bots.set_attributes(
                    participant_code=self.participant.code,
                    request_path=self.request.path,
                    html='',
                )
                submission = browser_bots.get_next_post_data(
                        participant_code=self.participant.code)
                if submission is None:
                    browser_bots.send_completion_message(
                        session_code=self.session.code,
                        participant_code=self.participant.code
                    )
                    return HttpResponse(BOT_COMPLETE_HTML_MESSAGE)
                else:
                    raise BotError(
                        'Finished the last page, '
                        'but the bot is still trying '
                        'to submit more data ({}).'.format(submission)
                    )
        self._increment_index_in_pages()
        return self._redirect_to_page_the_user_should_be_on()

    def get_object(self):
        Cls = self._get_form_model()
        if Cls == self.GroupClass:
            return self.group
        if Cls == self.PlayerClass:
            return self.player
        if Cls == UndefinedFormModel:
            return UndefinedFormModel.objects.all()[0]

    def socket_url(self):
        '''called from template. can't start with underscore because used
        in template
        '''
        return channel_utils.auto_advance_path(
            participant_code=self.participant.code,
            page_index=self._index_in_pages
        )

    def redirect_url(self):
        '''called from template'''
        # need full path because we use query string
        return self.request.get_full_path()

    def _get_auto_submit_values(self):
        # TODO: auto_submit_values deprecated on 2015-05-28
        auto_submit_values = getattr(self, 'auto_submit_values', {})
        timeout_submission = self.timeout_submission or auto_submit_values
        for field_name in self.get_form_fields():
            if field_name not in timeout_submission:
                # get default value for datatype if the user didn't specify

                ModelClass = self._get_form_model()
                ModelField = ModelClass._meta.get_field(field_name)
                # TODO: should we warn if the attribute doesn't exist?
                value = getattr(ModelField, 'auto_submit_default', None)
                timeout_submission[field_name] = value
        return timeout_submission

    def _process_auto_submitted_form(self, form):
        '''
        # an empty submitted form looks like this:
        # {'f_currency': None, 'f_bool': None, 'f_int': None, 'f_char': ''}
        '''
        auto_submit_values = self._get_auto_submit_values()

        # force the form to be cleaned
        form.is_valid()

        has_non_field_error = form.errors.pop('__all__', False)

        # In a non-timeout form, error_message is only run if there are no
        # field errors (because the error_message function assumes all fields exist)
        # however, if there is a timeout, we accept the form even if there are some field errors,
        # so we have to make sure we don't skip calling error_message()
        if form.errors and not has_non_field_error:
            if hasattr(self, 'error_message'):
                try:
                    has_non_field_error = bool(self.error_message(form.cleaned_data))
                except:
                    has_non_field_error = True

        if has_non_field_error:
            # non-field errors exist.
            # ignore form, use timeout_submission entirely
            auto_submit_values_to_use = auto_submit_values
        elif form.errors:
            auto_submit_values_to_use = {}
            for field_name in form.errors:
                auto_submit_values_to_use[field_name] = auto_submit_values[field_name]
            form.errors.clear()
            form.save()
        else:
            auto_submit_values_to_use = {}
            form.save()
        for field_name in auto_submit_values_to_use:
            setattr(self.object, field_name, auto_submit_values_to_use[field_name])

    def has_timeout_(self):
        return PageTimeout.objects.filter(
            participant=self.participant,
            page_index=self.participant._index_in_pages).exists()

    _remaining_timeout_seconds = 'unset'
    def remaining_timeout_seconds(self):

        if self._remaining_timeout_seconds is not 'unset':
            return self._remaining_timeout_seconds

        try:
            timeout_seconds = self.get_timeout_seconds()
        except:
            raise ResponseForException

        if timeout_seconds is None:
            # don't hit the DB at all
            pass
        else:
            current_time = time.time()
            expiration_time = current_time + timeout_seconds

            timeout_object, created = PageTimeout.objects.get_or_create(
                participant=self.participant,
                page_index=self.participant._index_in_pages,
                defaults={'expiration_time': expiration_time})

            timeout_seconds = timeout_object.expiration_time - current_time
            if created and otree.common_internal.USE_REDIS:
                # if using browser bots, don't schedule the timeout,
                # because if it's a short timeout, it could happen before
                # the browser bot submits the page. Because the timeout
                # doesn't query the botworker (it is distinguished from bot
                # submits by the timeout_happened flag), it will "skip ahead"
                # and therefore confuse the bot system.
                if not self.participant.is_browser_bot:
                    otree.timeout.tasks.submit_expired_url.schedule(
                        (
                            self.participant.code,
                            self.request.path,
                        ),
                        # add some seconds to account for latency of request + response
                        # this will (almost) ensure
                        # (1) that the page will be submitted by JS before the
                        # timeoutworker, which ensures that self.request.POST
                        # actually contains a value.
                        # (2) that the timeoutworker doesn't accumulate a lead
                        # ahead of the real page, which could result in being >1
                        # page ahead. that means that entire pages could be skipped

                        # task queue can't schedule tasks in the past
                        # at least 1 second from now
                        delay=max(1, timeout_seconds+8))
        self._remaining_timeout_seconds = timeout_seconds
        return timeout_seconds

    def get_timeout_seconds(self):
        return self.timeout_seconds

    timeout_seconds = None
    timeout_submission = None
    timer_text = ugettext_lazy("Time left to complete this page:")

_MSG_Undefined_GetPlayersForGroup = (
    'You cannot reference self.player, self.group, or self.participant '
    'inside get_players_for_group.'
)

_MSG_Undefined_AfterAllPlayersArrive_Player = (
    'self.player and self.participant do not exist in after_all_players_arrive. '
    'You should use self.group.get_players() instead.'
)

_MSG_Undefined_AfterAllPlayersArrive_Group = (
    'self.group does not exist in after_all_players_arrive '
    'if wait_for_all_groups=True. '
    'You should use self.subsession.get_groups() instead.'
)

class Undefined_AfterAllPlayersArrive_Player:
    def __getattribute__(self, item):
        raise AttributeError(_MSG_Undefined_AfterAllPlayersArrive_Player)

    def __setattr__(self, item, value):
        raise AttributeError(_MSG_Undefined_AfterAllPlayersArrive_Player)

class Undefined_AfterAllPlayersArrive_Group:
    def __getattribute__(self, item):
        raise AttributeError(_MSG_Undefined_AfterAllPlayersArrive_Group)

    def __setattr__(self, item, value):
        raise AttributeError(_MSG_Undefined_AfterAllPlayersArrive_Group)

class Undefined_GetPlayersForGroup:

    def __getattribute__(self, item):
        raise AttributeError(_MSG_Undefined_GetPlayersForGroup)

    def __setattr__(self, item, value):
        raise AttributeError(_MSG_Undefined_GetPlayersForGroup)

class GenericWaitPageMixin:
    """used for in-game wait pages, as well as other wait-type pages oTree has
    (like waiting for session to be created, or waiting for players to be
    assigned to matches

    """
    request = None

    def redirect_url(self):
        '''called from template'''
        # need get_full_path because we use query string here
        return self.request.get_full_path()

    def get_template_names(self):
        '''built-in wait pages should not be overridable'''
        return ['otree/WaitPage.html']

    def _get_wait_page(self):
        response = TemplateResponse(
            self.request, self.get_template_names(), self.get_context_data())
        response[constants.wait_page_http_header] = (
            constants.get_param_truth_value)
        return response

    # Translators: the default title of a wait page
    title_text = ugettext_lazy('Please wait')
    body_text = None

    def _get_default_body_text(self):
        '''
        needs to be a method because it could say
        "waiting for the other player", "waiting for the other players"...
        '''
        return ''

    def get_context_data(self):
        title_text = self.title_text
        body_text = self.body_text

        # could evaluate to false like 0
        if body_text is None:
            body_text = self._get_default_body_text()

        # default title/body text can be overridden
        # if user specifies it in vars_for_template
        return {
            'view': self,
            'title_text': title_text,
            'body_text': body_text,
        }

class WaitPage(FormPageOrInGameWaitPage, GenericWaitPageMixin):
    """
    Wait pages during game play (i.e. checkpoints),
    where users wait for others to complete
    """
    wait_for_all_groups = False
    group_by_arrival_time = False

    def get_context_data(self):
        context = GenericWaitPageMixin.get_context_data(self)
        return FormPageOrInGameWaitPage.get_context_data(self, **context)

    def get_template_names(self):
        """fallback to otree/WaitPage.html, which is guaranteed to exist.
        the reason for the 'if' statement, rather than returning a list,
        is that if the user explicitly defined template_name, and that template
        does not exist, then we should not fail silently.
        (for example, the user forgot to add it to git)
        """
        if self.template_name:
            return [self.template_name]
        return ['global/WaitPage.html', 'otree/WaitPage.html']

    def inner_dispatch(self, *args, **kwargs):
        with get_redis_lock(name='otree_waitpage') or wait_page_thread_lock:
            otree.db.idmap.save_objects()
            idmap.flush()
            if self.wait_for_all_groups == True:
                resp = self.inner_dispatch_subsession()
            elif self.group_by_arrival_time:
                resp = self.inner_dispatch_gbat()
            else:
                resp = self.inner_dispatch_group()
            return resp

    def inner_dispatch_group(self):
        ## EARLY EXITS
        if CompletedGroupWaitPage.objects.filter(
            page_index=self._index_in_pages,
            id_in_subsession=self.group.id_in_subsession,
            session=self.session,
        ).exists():
            return self._response_when_ready()
        if self._is_displayed():
            if self._get_unvisited_ids():
                self.participant.is_on_wait_page = True
                return self._get_wait_page()
            # make a clean copy for AAPA
            # self.player and self.participant etc are undefined
            # and no objects are cached inside it
            # and it doesn't affect the current instance

            wp = type(self)() # type: WaitPage
            wp.set_attributes_waitpage_clone(original_view=self)
            # if any player can skip the wait page,
            # then we shouldn't run after_all_players_arrive
            # because if some players are able to proceed to the next page
            # before after_all_players_arrive is run,
            # then after_all_players_arrive is probably not essential.
            # often, there are some wait pages that all players skip,
            # because they should only be shown in certain rounds.
            # maybe the fields that after_all_players_arrive depends on
            # are null
            # something to think about: ideally, should we check if
            # all players skipped, or any player skipped?
            # as a shortcut, we just check if is_displayed is true
            # for the last player.

            wp._player_access_forbidden = Undefined_AfterAllPlayersArrive_Player()
            wp._participant_access_forbidden = Undefined_AfterAllPlayersArrive_Player()
            wp._group_access_forbidden = None
            wp._group_for_aapa = self.group
            try:
                wp.after_all_players_arrive()
            except:
                raise ResponseForException
            # need to save to the results of after_all_players_arrive
            # to the DB, before sending the completion message to other players
            # this was causing a race condition on 2016-11-04
            otree.db.idmap.save_objects()

        # even if this player skips the page and after_all_players_arrive
        # is not run, we need to indicate that the waiting players can advance
        self._mark_completed_and_notify(group=self.group)
        return self._response_when_ready()

    def inner_dispatch_subsession(self):

        if CompletedSubsessionWaitPage.objects.filter(
                page_index=self._index_in_pages,
                session=self.session,
        ).exists():
            return self._save_and_flush_and_response_when_ready()

        if self._is_displayed():
            if self._get_unvisited_ids():
                self.participant.is_on_wait_page = True
                return self._get_wait_page()

            # make a clean copy for AAPA
            # self.player and self.participant etc are undefined
            # and no objects are cached inside it
            # and it doesn't affect the current instance
            wp = type(self)() # type: WaitPage
            wp.set_attributes_waitpage_clone(original_view=self)

            # if any player can skip the wait page,
            # then we shouldn't run after_all_players_arrive
            # because if some players are able to proceed to the next page
            # before after_all_players_arrive is run,
            # then after_all_players_arrive is probably not essential.
            # often, there are some wait pages that all players skip,
            # because they should only be shown in certain rounds.
            # maybe the fields that after_all_players_arrive depends on
            # are null
            # something to think about: ideally, should we check if
            # all players skipped, or any player skipped?
            # as a shortcut, we just check if is_displayed is true
            # for the last player.

            wp._player_access_forbidden = Undefined_AfterAllPlayersArrive_Player()
            wp._participant_access_forbidden = Undefined_AfterAllPlayersArrive_Player()
            wp._group_access_forbidden = Undefined_AfterAllPlayersArrive_Group()
            try:
                wp.after_all_players_arrive()
            except:
                raise ResponseForException
            # need to save to the results of after_all_players_arrive
            # to the DB, before sending the completion message to other players
            # this was causing a race condition on 2016-11-04
            otree.db.idmap.save_objects()
        # even if this player skips the page and after_all_players_arrive
        # is not run, we need to indicate that the waiting players can advance
        self._mark_completed_and_notify(group=None)
        return self._response_when_ready()

    def inner_dispatch_gbat(self):
        if CompletedGroupWaitPage.objects.filter(
            page_index=self._index_in_pages,
            # no race condition, this will be up to date
            # because after taking the lock we flushed the IDmap cache
            id_in_subsession=self.group.id_in_subsession,
            session=self.session,
        ).exists():
            return self._response_when_ready()

        if not self._is_displayed():
            # in GBAT, either all players should skip a page, or none should.
            # we don't support some players skipping and others not.
            return self._response_when_ready()

        self.player._gbat_arrived = True
        # _last_request_timestamp is already set in set_attributes,
        # but set it here just so we can guarantee
        self.participant._last_request_timestamp = time.time()
        # need to save it inside the lock (check-then-act)
        # also because it needs to be up to date for get_players_for_group
        # which gets this info from the DB
        self.player.save()
        self.participant.save()

        # make a clean copy for GBAT and AAPA
        # self.player and self.participant etc are undefined
        # and no objects are cached inside it
        # and it doesn't affect the current instance
        wp = type(self)() # type: WaitPage
        wp.set_attributes_waitpage_clone(original_view=self)

        wp._player_access_forbidden = Undefined_GetPlayersForGroup()
        wp._participant_access_forbidden = Undefined_GetPlayersForGroup()
        wp._group_access_forbidden = Undefined_GetPlayersForGroup()

        # DELETE THIS after a few months
        if self.player._gbat_grouped:
            raise ValueError(
                'Internal oTree error: player was grouped '
                'but no completion exists (should not happen if lock is working) '
            )

        gbat_new_group = wp._gbat_try_to_make_new_group()

        if gbat_new_group:
            wp._player_access_forbidden = Undefined_AfterAllPlayersArrive_Player()
            wp._participant_access_forbidden = Undefined_AfterAllPlayersArrive_Player()
            wp._group_access_forbidden = None
            wp._group_for_aapa = gbat_new_group
            try:
                wp.after_all_players_arrive()
            except:
                raise ResponseForException

            # need to save before sending completion message
            otree.db.idmap.save_objects()

            self._mark_completed_and_notify(gbat_new_group)
            # gbat_new_group may not include the current player!
            # maybe this will not work if i change the implementation
            # so that the player is cached,
            # but that's OK because it will be obvious it doesn't work.
            if self.player._gbat_grouped:
                return self._save_and_flush_and_response_when_ready()

        self.participant.is_on_wait_page = True
        return self._get_wait_page()

    def _gbat_try_to_make_new_group(self) -> Union[BaseGroup, None]:
        '''Returns the group ID of the participants who were regrouped'''

        # if someone arrives within this many seconds of the last heartbeat of
        # a player who drops out, they will be stuck.
        # that sounds risky, but remember that
        # if a player drops out at any point after that,
        # the other players in the group will also be stuck.
        # so the purpose of this is not to prevent dropouts that happen
        # for random reasons, but specifically on a wait page,
        # which is usually because someone gets stuck waiting for a long time.
        # we don't want to make it too short, because that means the page
        # would have to refresh very quickly, which could be disruptive.
        STALE_THRESHOLD_SECONDS = 20

        # count how many are re-grouped
        waiting_players = list(self.subsession.player_set.filter(
            _gbat_arrived=True,
            _gbat_grouped=False,
            participant___last_request_timestamp__gte=time.time()-STALE_THRESHOLD_SECONDS
        ))

        try:
            players_for_group = self.get_players_for_group(waiting_players)
        except:
            raise ResponseForException

        if not players_for_group:
            return None
        participant_ids = [p.participant.id for p in players_for_group]

        group_id_in_subsession = self._gbat_next_group_id_in_subsession()

        Constants = self._Constants

        this_round_new_group = None
        with otree.common_internal.transaction_except_for_sqlite():
            for round_number in range(self.round_number, Constants.num_rounds+1):
                subsession = self.subsession.in_round(round_number)

                unordered_players = subsession.player_set.filter(
                    participant_id__in=participant_ids)

                participant_ids_to_players = {
                    player.participant.id: player for player in unordered_players}

                ordered_players_for_group = [
                    participant_ids_to_players[participant_id]
                    for participant_id in participant_ids]

                if round_number == self.round_number:
                    for player in ordered_players_for_group:
                        player._gbat_grouped = True
                        player.save()

                group = self.GroupClass.objects.create(
                    subsession=subsession, id_in_subsession=group_id_in_subsession,
                    session=self.session, round_number=round_number)
                group.set_players(ordered_players_for_group)

                if round_number == self.round_number:
                    this_round_new_group = group

                # prune groups without players
                # apparently player__isnull=True works, didn't know you could
                # use this in a reverse direction.
                subsession.group_set.filter(player__isnull=True).delete()
        return this_round_new_group

    def get_players_for_group(self, waiting_players):
        Constants = self._Constants

        if Constants.players_per_group is None:
            raise AssertionError(
                'Page "{}": if using group_by_arrival_time, you must either set '
                'Constants.players_per_group to a value other than None, '
                'or define get_players_for_group() on the page.'.format(
                    self.__class__.__name__
                )
            )

        if len(waiting_players) >= Constants.players_per_group:
            return waiting_players[:Constants.players_per_group]

    def _gbat_next_group_id_in_subsession(self):
        # 2017-05-05: seems like this can result in id_in_subsession that
        # doesn't start from 1.
        # especially if you do group_by_arrival_time in every round
        # is that a problem?
        res = self.GroupClass.objects.filter(
            session=self.session).aggregate(Max('id_in_subsession'))
        return res['id_in_subsession__max'] + 1

    _player_access_forbidden = None
    @property
    def player(self):
        return self._player_access_forbidden or super().player

    _group_access_forbidden = None
    _group_for_aapa = None
    @property
    def group(self):
        return self._group_access_forbidden or self._group_for_aapa or super().group

    _participant_access_forbidden = None
    @property
    def participant(self):
        return self._participant_access_forbidden or super().participant

    @property
    def _group_or_subsession(self):
        return self.subsession if self.wait_for_all_groups else self.group

    def _save_and_flush_and_response_when_ready(self):
        # need to deactivate cache, in case after_all_players_arrive
        # finished running after the moment set_attributes
        # was called in this request.

        # because in response_when_ready we will call
        # increment_index_in_pages, which does a look-ahead and calls
        # is_displayed() on the following pages. is_displayed() might
        # depend on a field that is set in after_all_players_arrive
        # so, need to clear the cache to ensure
        # that we get fresh data.

        # Note: i was never able to reproduce this myself -- just heard
        # from Anthony N.
        # and it shouldn't happen, because only the last player to visit
        # can set is_ready(). if there is a request coming after that,
        # then it must be someone refreshing the page manually.
        # i guess we should protect against that.

        # is_displayed() could also depend on a field on participant
        # that was set on the wait page, so need to refresh participant,
        # because it is passed as an arg to set_attributes().

        otree.db.idmap.save_objects()
        idmap.flush()
        return self._response_when_ready()

    def _mark_completed_and_notify(self, group: BaseGroup):
        # if group is not passed, then it's the whole subsession
        # could be 2 people creating the record at the same time
        # in _increment_index_in_pages, so could end up creating 2 records
        # but it's not a problem.

        base_kwargs = dict(
            page_index=self._index_in_pages,
            session_id=self._session_pk,
        )

        if self.wait_for_all_groups:
            CompletedSubsessionWaitPage.objects.create(**base_kwargs)
            obj = self.subsession
        else:
            CompletedGroupWaitPage.objects.create(
                **base_kwargs,
                id_in_subsession=group.id_in_subsession,
            )
            obj = group

        if otree.common_internal.USE_REDIS:
            participant_pks = obj.player_set.values_list('participant__pk', flat=True)
            # 2016-11-15: we used to only ensure the next page is visited
            # if the next page has a timeout, or if it's a wait page
            # but this is not reliable because next page might be skipped anyway,
            # and we don't know what page will actually be shown next to the user.
            otree.timeout.tasks.ensure_pages_visited.schedule(
                kwargs={
                    'participant_pks': participant_pks
                },
                delay=10)

        if self.group_by_arrival_time:
            channel_utils.sync_group_send_wrapper(
                type='gbat_ready',
                group=channel_utils.gbat_group_name(**base_kwargs),
                event={}
            )
        else:
            if self.wait_for_all_groups:
                channels_group_name = channel_utils.wait_page_group_name(**base_kwargs)
            else:
                channels_group_name = channel_utils.wait_page_group_name(
                    **base_kwargs,
                    group_id_in_subsession=group.id_in_subsession,
                )

            channel_utils.sync_group_send_wrapper(
                type='wait_page_ready',
                group=channels_group_name,
                event={}
            )

    def socket_url(self):
        if self.group_by_arrival_time:
            return channel_utils.gbat_path(
                self._session_pk, self._index_in_pages,
                self.player._meta.app_config.name, self.player.id
            )
        elif self.wait_for_all_groups:
            return channel_utils.wait_page_path(
                self._session_pk,
                self._index_in_pages
            )
        else:
            return channel_utils.wait_page_path(
                self._session_pk,
                self._index_in_pages,
                self.group.id_in_subsession
            )

    def _get_unvisited_ids(self):
        """
        Don't need a lock
        """

        participant_ids = set(
            self._group_or_subsession.player_set.values_list(
                'participant_id', flat=True))

        # essential query whose results can change from moment to moment
        participant_data = Participant.objects.filter(
            id__in=participant_ids
        ).values('id', 'id_in_session', '_index_in_pages')

        visited = []
        unvisited = []
        for p in participant_data:
            if p['_index_in_pages'] < self._index_in_pages:
                unvisited.append(p)
            else:
                visited.append(p)

        # this is not essential to functionality.
        # just for the display in the Monitor tab.
        # so, we don't need a lock, even though there could be a race condition.
        if 1 <= len(unvisited) <= 3:

            unvisited_description = ', '.join(
                'P{}'.format(p['id_in_session']) for p in unvisited)

            visited_ids = [p['id'] for p in visited]
            Participant.objects.filter(
                id__in=visited_ids
            ).update(_waiting_for_ids=unvisited_description)

        return {p['id'] for p in unvisited}

    def is_displayed(self):
        return True

    def _response_when_ready(self):
        '''
        Before calling this function, the following must be satisfied:
        - The completion object exists
        OR
        - The player skips this page
        '''
        self.participant.is_on_wait_page = False
        self.participant._waiting_for_ids = None
        self._increment_index_in_pages()
        return self._redirect_to_page_the_user_should_be_on()

    def after_all_players_arrive(self):
        pass

    def _get_default_body_text(self):
        num_other_players = self._group_or_subsession.player_set.count() - 1
        if num_other_players > 1:
            return _('Waiting for the other participants.')
        if num_other_players == 1:
            return _('Waiting for the other participant.')
        return ''

class AdminSessionPageMixin:

    @classmethod
    def url_pattern(cls):
        return r"^{}/(?P[a-z0-9]+)/$".format(cls.__name__)

    def get_context_data(self, **kwargs):
        context = super().get_context_data(
            session=self.session,
            is_debug=settings.DEBUG,
            request=self.request,
            **kwargs)
        # vars_for_template has highest priority
        context.update(self.vars_for_template())
        return context

    def vars_for_template(self):
        '''
        simpler to use vars_for_template, but need to use get_context_data when:
        -   you need access to the context produced by the parent class,
            such as the form
        '''
        return {}

    def get_template_names(self):
        return ['otree/admin/{}.html'.format(self.__class__.__name__)]

    def dispatch(self, request, code, **kwargs):
        self.session = get_object_or_404(
            otree.models.Session, code=code)
        return super().dispatch(request, **kwargs)

    def get_form_class(self):
        """A drop-in replacement for
        ``vanilla.model_views.GenericModelView.get_form_class``. The only
        difference is that we use oTree's modelform_factory in order to always
        get a floppyfied form back which supports richer widgets.
        """
        if self.form_class is not None:
            return self.form_class

        return otree.forms.modelform_factory(
            self.model,
            fields=self.fields,
            formfield_callback=otree.forms.formfield_callback)

class InvalidAppError(Exception):
    pass


from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail, Attachment, FileContent, FileName, FileType, Disposition
from utils.email_html_template import get_email_html_template
from utils.encrypt import encrypt_email
from utils.convert_url_firebase_to_base64 import convert_url_firebase_to_base64
from utils.get_file_type import get_file_type
from dotenv import load_dotenv
import os

load_dotenv()

class EmailSender:
    def __init__(self):
        self.sg = SendGridAPIClient(os.getenv('SENDGRID_API_KEY'))

    def send_email(self, recipient_email, subject, body, article_file_path, newsletter_id):
        try:
            encrypted_recipient = encrypt_email(recipient_email)
            unsubscribe_url = 'http://localhost:9999/newsletters/' + \
                newsletter_id + '/' + encrypted_recipient + '/unsubscribe'

            mail = Mail(
                from_email='ayelenmaite2000@gmail.com',
                to_emails=recipient_email,
                subject=subject,
                plain_text_content=body,
                html_content=get_email_html_template(unsubscribe_url)
                )

            file_type = get_file_type(article_file_path)
            image_content_base64 = convert_url_firebase_to_base64(
                article_file_path)

            attachment = Attachment()
            attachment.file_content = image_content_base64
            attachment.file_name = FileName('article')
            attachment.file_type = FileType(file_type)
            attachment.content_id = "image_cid"
            attachment.disposition = Disposition('attachment')
            mail.attachment = attachment

            response = self.sg.send(mail)

            if response.status_code == 202:
                return True
            else:
                return False
        except Exception as e:
            raise Exception(str(e))


'''
Script for plotting the solutions for n=10,100,1000 coputed with the general,
specialized and LU decomposition algorithm.
'''
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns

#analytic solution
def u(x):
    f = 1 - (1 - np.exp(-10))*x - np.exp(-10*x)
    return f

#values for n
datalist = [10, 100, 1000]

#setting plot style using seaborn
sns.set()
sns.set_style("whitegrid")
sns.set_palette("husl")

#figure with three subplots
fig, ax = plt.subplots(3, 1, sharex='col', sharey='row')

#list of file names and algorithm names
filename = ["general_matrix","special_matrix","LU_decomp"]
algorithm = ["general algorithm", "special algorithm", "LU decomposition"]

#plotting
for j in range(3):
    legends = []
    for i in datalist:
        data = np.loadtxt("../data/" + filename[j] + "{}.dat".format(i))
        ax[j].plot(data[:,0], data[:,1],label='n = {}'.format(i))
    #ax[j].plot(data[:,0], u(data[:,0]),label='analytic')
    ax[j].text(0.3, 0.15, algorithm[j], horizontalalignment='center', \
    verticalalignment='center', color="grey")

#adding legends above the first plot
ax[0].legend(loc='upper center', bbox_to_anchor=(0.5, 1.3),
          ncol=4,fontsize=12)

#adding one common x and y label
fig.text(0.5, 0.04, 'x', fontsize=14, ha='center')
fig.text(0.03, 0.5, 'u(x)', fontsize=14, va='center', rotation='vertical')

plt.savefig("../figurer/all_matrix_compare.pdf")

plt.show()


#! /usr/local/bin/python
import os

import zulip

import rsvp
import zulip_users

class Bot():
    """ bot takes a zulip username and api key, a word or phrase to respond to, a search string for giphy,
        an optional caption or list of captions, and a list of the zulip streams it should be active in.
        it then posts a caption and a randomly selected gif in response to zulip messages.
     """
    def __init__(self, zulip_username, zulip_api_key, key_word, subscribed_streams=None, zulip_site=None):
        self.key_word = key_word.lower()
        self.subscribed_streams = subscribed_streams or []
        self.client = zulip.Client(zulip_username, zulip_api_key, site=zulip_site)
        self.client._register('get_users', method='GET', url='users')
        self.subscriptions = self.subscribe_to_streams()
        self.rsvp = rsvp.RSVP(key_word)

    @property
    def streams(self):
        """Standardizes a list of streams in the form [{'name': stream}]."""
        if not self.subscribed_streams:
            streams = [{'name': stream['name']} for stream in self.get_all_zulip_streams()]
            return streams
        else:
            streams = [{'name': stream} for stream in self.subscribed_streams]
            return streams

    def get_all_zulip_streams(self):
        """Call Zulip API to get a list of all streams."""
        response = self.client.get_streams()
        if response['result'] == 'success':
            return response['streams']
        else:
            raise RuntimeError('check yo auth')

    def subscribe_to_streams(self):
        """Subscribes to zulip streams."""
        self.client.add_subscriptions(self.streams)

    def process(self, event):
        if event['type'] == 'realm_user':
            zulip_users.update_zulip_user_dict(event['person'], self.client)
        elif event['type'] == 'message':
            self.respond(event['message'])

    def respond(self, message):
        """Now we have an event dict, we should analyze it completely."""

        replies = self.rsvp.process_message(message)

        for reply in replies:
            if reply:
                self.send_message(reply)

    def send_message(self, msg):
        """Sends a message to zulip stream or user."""
        msg_to = msg['display_recipient']
        if msg['type'] == 'private':
            msg_to = msg.get('sender_email') or msg_to

        self.client.send_message({
            "type": msg['type'],
            "subject": msg["subject"],
            "to": msg_to,
            "content": msg['body']
        })

    def main(self):
        """Blocking call that runs forever. Calls self.respond() on every event received."""
        self.client.call_on_each_event(self.process, ['message', 'realm_user'])

""" The Customization Part!

    Create a zulip bot under "settings" on zulip.
    Zulip will give you a username and API key
    key_word is the text in Zulip you would like the bot to respond to. This may be a
        single word or a phrase.
    search_string is what you want the bot to search giphy for.
    caption may be one of: [] OR 'a single string' OR ['or a list', 'of strings']
    subscribed_streams is a list of the streams the bot should be active on. An empty
        list defaults to ALL zulip streams

"""
if __name__ == "__main__":
    ZULIP_USERNAME = os.environ['ZULIP_RSVP_EMAIL']
    ZULIP_API_KEY = os.environ['ZULIP_RSVP_KEY']
    ZULIP_SITE = os.getenv('ZULIP_RSVP_SITE', 'https://zulip.com')
    KEY_WORD = os.getenv('ZULIP_KEY_WORD', 'rsvp')
    SANDBOX_STREAM = os.getenv('ZULIP_RSVP_SANDBOX_STREAM', None)
    SUBSCRIBED_STREAMS = []
    new_bot = Bot(
        ZULIP_USERNAME,
        ZULIP_API_KEY,
        KEY_WORD,
        SUBSCRIBED_STREAMS,
        ZULIP_SITE,
    )
    new_bot.main()


# Adapted from code in https://github.com/airsplay/py-bottom-up-attention
# Please see https://github.com/airsplay/py-bottom-up-attention/blob/master/LICENSE for the Apache 2.0 License of that code

import os
import sys
import io
import json
from tqdm import tqdm

import detectron2

# import some common detectron2 utilities
from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.utils.visualizer import Visualizer
from detectron2.data import MetadataCatalog

# import some common libraries
import numpy as np
import cv2
import torch

NUM_OBJECTS = 36

from torch import nn

from detectron2.modeling.postprocessing import detector_postprocess
from detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers, FastRCNNOutputs, fast_rcnn_inference_single_image
from detectron2.structures.boxes import Boxes
from detectron2.structures.instances import Instances

# Load VG Classes
data_path = 'data/genome/1600-400-20'

vg_classes = []
with open(os.path.join(data_path, 'objects_vocab.txt')) as f:
    for object in f.readlines():
        vg_classes.append(object.split(',')[0].lower().strip())

vg_attrs = []
with open(os.path.join(data_path, 'attributes_vocab.txt')) as f:
    for object in f.readlines():
        vg_attrs.append(object.split(',')[0].lower().strip())

MetadataCatalog.get("vg").thing_classes = vg_classes
MetadataCatalog.get("vg").attr_classes = vg_attrs

cfg = get_cfg()
cfg.merge_from_file("configs/VG-Detection/faster_rcnn_R_101_C4_caffe.yaml")
cfg.MODEL.RPN.POST_NMS_TOPK_TEST = 300
cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.6
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.2
# VG Weight
# cfg.MODEL.WEIGHTS = "http://nlp.cs.unc.edu/models/faster_rcnn_from_caffe.pkl"
cfg.MODEL.WEIGHTS = "http://nlp.cs.unc.edu/models/faster_rcnn_from_caffe_attr_original.pkl"
predictor = DefaultPredictor(cfg)

def doit(raw_image, raw_boxes):
    # Process Boxes
    raw_boxes = Boxes(torch.from_numpy(raw_boxes).cuda())
    with torch.no_grad():
        raw_height, raw_width = raw_image.shape[:2]
        print("Original image size: ", (raw_height, raw_width))
        # Preprocessing
        image = predictor.transform_gen.get_transform(raw_image).apply_image(raw_image)
        print("Transformed image size: ", image.shape[:2])
        # Scale the box
        new_height, new_width = image.shape[:2]
        scale_x = 1. * new_width / raw_width
        scale_y = 1. * new_height / raw_height
        #print(scale_x, scale_y)
        boxes = raw_boxes.clone()
        boxes.scale(scale_x=scale_x, scale_y=scale_y)
        image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
        inputs = [{"image": image, "height": raw_height, "width": raw_width}]
        images = predictor.model.preprocess_image(inputs)
        # Run Backbone Res1-Res4
        features = predictor.model.backbone(images.tensor)
        # Run RoI head for each proposal (RoI Pooling + Res5)
        proposal_boxes = [boxes]
        features = [features[f] for f in predictor.model.roi_heads.in_features]
        box_features = predictor.model.roi_heads._shared_roi_transform(
            features, proposal_boxes
        )
        feature_pooled = box_features.mean(dim=[2, 3])  # pooled to 1x1
        print('Pooled features size:', feature_pooled.shape)
        # Predict classes and boxes for each proposal.
        pred_class_logits, pred_proposal_deltas = predictor.model.roi_heads.box_predictor(feature_pooled)
        print(pred_class_logits.shape)
        pred_class_prob = nn.functional.softmax(pred_class_logits, -1)
        pred_scores, pred_classes = pred_class_prob[..., :-1].max(-1)
        # Detectron2 Formatting (for visualization only)
        roi_features = feature_pooled
        instances = Instances(
            image_size=(raw_height, raw_width),
            pred_boxes=raw_boxes,
            scores=pred_scores,
            pred_classes=pred_classes
        )
        return instances, roi_features

# Image root
img_root = sys.argv[1]
# jsonl input file
f = open(sys.argv[2])
lines = f.readlines()
data = [json.loads(line) for line in lines]
if len(sys.argv) > 4:
    # Predicted boxes JSON file
    f = open(sys.argv[3])
    predicted_boxes = json.load(f)
boxes_dict = {}
for datum in tqdm(data):
    if 'coco' in datum['file_name'].lower():
        datum['file_name'] = '_'.join(datum['file_name'].split('_')[:-1])+'.jpg'
    im = cv2.imread(img_root+datum['file_name'])
    if len(sys.argv) == 4:
        boxes = np.array(
            [[ann['bbox'][0], ann['bbox'][1], ann['bbox'][0]+ann['bbox'][2], ann['bbox'][1]+ann['bbox'][3]] for ann in datum['anns']]
        )
        conf = np.array([1 for _ in datum["anns"]])
    else:
        assert len(sys.argv) > 4
        conf = np.array(predicted_boxes[str(datum["image_id"])]["scores"])
        print(conf.shape)
        if len(conf) > 0:
            boxes = np.array(predicted_boxes[str(datum["image_id"])]["boxes"])[(-conf).argsort(),:]
            conf = conf[(-conf).argsort()]
        else:
            boxes = np.array([[0, 0, im.shape[1], im.shape[0]]])
            conf = np.array([1.])
    _, feats = doit(im, boxes)
    boxes_dict[datum['image_id']] = {"boxes": torch.from_numpy(boxes), "features": feats, "width": im.shape[1], "height": im.shape[0], "conf": torch.from_numpy(conf)}
# Output .pt file
torch.save(boxes_dict, sys.argv[4])


from ..logging import logging # Ensure use of custom logger class
from . import s3c
from .s3c import get_S3Error
from .common import NoSuchObject, retry
from ..common import QuietError, BUFSIZE
from ..inherit_docstrings import copy_ancestor_docstring
from xml.sax.saxutils import escape as xml_escape
import re

log = logging.getLogger(__name__)

# Maximum number of keys that can be deleted at once
MAX_KEYS = 1000

# Pylint goes berserk with false positives
#pylint: disable=E1002,E1101

class Backend(s3c.Backend):
    """A backend to store data in Amazon S3

    This class uses standard HTTP connections to connect to S3.

    The backend guarantees get after create consistency, i.e. a newly created
    object will be immediately retrievable. Additional consistency guarantees
    may or may not be available and can be queried for with instance methods.
    """

    def __init__(self, storage_url, login, password, ssl_context=None,
                 proxy=None):
        super().__init__(storage_url, login, password, proxy=proxy,
                         ssl_context=ssl_context)

    @staticmethod
    def _parse_storage_url(storage_url, ssl_context):
        hit = re.match(r'^s3s?://([^/]+)(?:/(.*))?$', storage_url)
        if not hit:
            raise QuietError('Invalid storage URL')

        bucket_name = hit.group(1)

        # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/dev/BucketRestrictions.html
        if not re.match('^[a-z0-9][a-z0-9.-]{1,60}[a-z0-9]$', bucket_name):
            raise QuietError('Invalid bucket name.')

        # Dots in the bucket cause problems with SSL certificate validation,
        # because server certificate is for *.s3.amazonaws.com (which does not
        # match e.g. a.b.s3.amazonaws.com). However, when using path based
        # bucket selection, AWS issues a redirect to the former
        # location. Therefore, S3 buckets with dots in their names cannot be
        # accessed securely. This can only be fixed by Amazon by allowing
        # path-based bucket selection, or providing a valid SSL certificate for
        # endpoints that also matches if the bucket name contains dots.
        if '.' in bucket_name and ssl_context:
            raise QuietError('Buckets with dots in the name cannot be accessed over SSL.\n'
                             'This is purely Amazon\'s fault, see '
                             'https://forums.aws.amazon.com/thread.jspa?threadID=130560')
        hostname = '%s.s3.amazonaws.com' % bucket_name

        prefix = hit.group(2) or ''
        port = 443 if ssl_context else 80
        return (hostname, port, bucket_name, prefix)

    def __str__(self):
        return 'Amazon S3 bucket %s, prefix %s' % (self.bucket_name, self.prefix)

    @copy_ancestor_docstring
    def delete_multi(self, keys, force=False):
        log.debug('delete_multi(%s)', keys)

        while len(keys) > 0:
            tmp = keys[:MAX_KEYS]
            try:
                self._delete_multi(tmp, force=force)
            finally:
                keys[:MAX_KEYS] = tmp

    @retry
    def _delete_multi(self, keys, force=False):

        body = [ '<Delete>' ]
        esc_prefix = xml_escape(self.prefix)
        for key in keys:
            body.append('<Object><Key>%s%s</Key></Object>' % (esc_prefix, xml_escape(key)))
        body.append('</Delete>')
        body = '\n'.join(body).encode('utf-8')
        headers = { 'content-type': 'text/xml; charset=utf-8' }

        resp = self._do_request('POST', '/', subres='delete', body=body, headers=headers)
        try:
            root = self._parse_xml_response(resp)
            ns_p = self.xml_ns_prefix

            error_tags = root.findall(ns_p + 'Error')
            if not error_tags:
                # No errors occured, everything has been deleted
                del keys[:]
                return

            # Some errors occured, so we need to determine what has
            # been deleted and what hasn't
            offset = len(self.prefix)
            for tag in root.findall(ns_p + 'Deleted'):
                fullkey = tag.find(ns_p + 'Key').text
                assert fullkey.startswith(self.prefix)
                keys.remove(fullkey[offset:])

            if log.isEnabledFor(logging.DEBUG):
                for errtag in error_tags:
                    log.debug('Delete %s failed with %s',
                              errtag.findtext(ns_p + 'Key')[offset:],
                              errtag.findtext(ns_p + 'Code'))

            # If *force*, just modify the passed list and return without
            # raising an exception, otherwise raise exception for the first error
            if force:
                return

            errcode = error_tags[0].findtext(ns_p + 'Code')
            errmsg = error_tags[0].findtext(ns_p + 'Message')
            errkey = error_tags[0].findtext(ns_p + 'Key')[offset:]

            if errcode == 'NoSuchKeyError':
                raise NoSuchObject(errkey)
            else:
                raise get_S3Error(errcode, 'Error deleting %s: %s' % (errkey, errmsg))

        finally:
            # Need to read rest of response
            while True:
                buf = resp.read(BUFSIZE)
                if buf == b'':
                    break


import requests
import pandas as pd
from bs4 import BeautifulSoup
from datetime import datetime

# Определяем URL сайта
url = 'https://www.cbr.ru/currency_base/daily/'

# Получаем HTML-страницу с помощью requests
response = requests.get(url)
html = response.text

# Используем BeautifulSoup для парсинга HTML
soup = BeautifulSoup(html, 'html.parser')

# Находим таблицу с курсами валют
table = soup.find('table', {'class': 'data'})

# Создаем пустой список для данных о валютах
currencies = []

# Проходим по каждой строке таблицы
for tr in table.find_all('tr')[1:]:
    # Получаем название валюты и ее код
    name, code = [td.text.strip() for td in tr.find_all('td')[:2]]
    # Получаем курс валюты к рублю
    rate = float(tr.find_all('td')[4].text.replace(',', '.'))
    # Добавляем данные о валюте в список
    currencies.append({'name': name, 'code': code, 'rate': rate})

# Создаем DataFrame из списка данных о валютах
df = pd.DataFrame(currencies)

# Добавляем столбец с датой
now = datetime.now()
df['date'] = now.strftime('%Y-%m-%d')

# Сохраняем DataFrame в Excel-файл
filename = 'currency_rates_{}.xlsx'.format(now.strftime('%Y%m%d_%H%M%S'))
df.to_excel(filename, index=False)

print('Данные о курсах валют сохранены в файл', filename)


from asyncio import subprocess

import warnings
from datetime import timedelta, datetime
from logging import DEBUG, INFO, basicConfig, captureWarnings, getLogger
from time import sleep
import subprocess
import click
from timeloop import Timeloop

from livedivulgador.bots.livedivulgador import LiveDivulgador
from livedivulgador.handlers.create_streamer import CreateStreamerFromUsername
from livedivulgador.plugins.plugin import Plugin
from livedivulgador.plugins.twitter import TwitterPlugin
from livedivulgador.service.streamers_service import StreamersService

from dotenv import load_dotenv
from os import getenv

load_dotenv()

logger = getLogger(__name__)

@click.group(context_settings=dict(help_option_names=["-h", "--help"]))
@click.option("--debug", "-d", help="Enable debug logging", is_flag=True, default=False)
def main(debug):
    basicConfig(
        format="%(asctime)s [%(levelname)8s] [%(threadName)20s] %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
        level=DEBUG if debug else INFO,
    )

    def custom_formatwarning(msg, *args, **kwargs):
        return str(msg)

    warnings.formatwarning = custom_formatwarning
    warnings.simplefilter("default")
    captureWarnings(True)

    logger.warning("Starting Live Divulgador bot")

def create_bots(_class, plugins: list[Plugin]):
    bots = [create_bot(_class, plugin) for plugin in plugins]

    return bots

def create_bot(_class, plugin: Plugin):
    bot = _class()
    bot.add_plugin(plugin)

    return bot

@main.command("run")
def run():
    tl = Timeloop()
    tl.logger = logger
    execute = True
    plugins = [TwitterPlugin]
    bots = create_bots(LiveDivulgador, plugins)

    def exec_bots(bots: list[LiveDivulgador]) -> None:
        [create_bot_routine(bot) for bot in bots]

    def create_bot_routine(bot: LiveDivulgador) -> None:
        @tl.job(interval=timedelta(seconds=300))
        def start_loop():
            bot.run()

    exec_bots(bots)

    tl.start()

    while execute:
        try:
            sleep(1)
        except KeyboardInterrupt:
            logger.warning("Stopping Live Divulgador bot")
            tl.stop()
            execute = False

@main.command()
@click.option("twitch_username", "-u", help="Twitch username")
@click.option("twitter_username", "-t", help="Twitter username", default=None)
def add(twitch_username, twitter_username):
    logger.info(f"Trying to add `{twitch_username}`")
    CreateStreamerFromUsername.handle(twitch_username, twitter_username)

@main.command()
@click.option("twitch_username", "-u", help="Twitch username")
def rm(twitch_username):
    logger.info(f"Deleting streamer `{twitch_username}`")
    StreamersService.delete_streamer_by_name(twitch_username)

@main.command("backup")
@click.option("backup", "-b", help="Backup file")
def backup(backup):
    utc_now = datetime.utcnow().strftime("%Y-%m-%d_%H-%M-%S")
    with open(f"./backups/{utc_now}.sql", "w") as f:
        logger.info("Creating a backup .sql file")
        command = [
            "docker",
            "exec",
            "-it",
            "livedivulgador_db",
            "mysqldump",
            f'-u{getenv("DATABASE_USER")}',
            f'-p{getenv("DATABASE_PASSWORD")}',
            "--all-databases",
        ]

        out = subprocess.run(command, capture_output=True).stdout

        f.write(out.decode("utf-8"))

if __name__ == "__main__":
    main()


import os
import pandas as pd
import torch
import argparse
import random
from segment_data import construct_segment_dict
from segment_data import convert_to_dataframe
from strategy_model import strategy_evaluate
from detection_model import RobertaClassifier
from detection_model import article_train
from detection_model import article_evaluate
from detection_model import calculate_article_weights
from article_data import build_complete_dataset
from configs import get_config
from utils import set_seed
import numpy as np
class Config:
    dropout = 0.5
    max_length = 512
    num_labels = 2
    freeze_layers = 181
    longformer = False
    classifier_second_layer = None
    batch_size = 10
    epochs = 50
    learning_rate = 5e-5
    seed = random.randint(0,99999999)
    # seed = 9877

parser = argparse.ArgumentParser()
parser.add_argument('--mode', type=str,default="train")
parser.add_argument('--context', type=str,choices="none,low,high".split(","),default="none")
parser.add_argument('--source', type=str,choices=["claim","article","gt_strategy", "claim_article","pred_strategy","claim_gt","claim_article_gt" ,"claim_pred","claim_article_pred"])
parser.add_argument('--norm', type=int,choices=[0,1])
args = parser.parse_args()
context = args.context

# Declare the learning rate and batch size for detection training
article_config = Config()
set_seed(article_config.seed)

test_data = pd.read_csv(f"data/test_article_data_{context}.csv")
corr_labels = {True: "true", False: "false"}
test_data["label"] = test_data["label"].apply(lambda x: corr_labels[x])
test_data = test_data[test_data["label"] != "none"]

# Define the layer to train and test on.
# "article" for base article text
# "target_combined" for article and ground truth labels
# "pred_combined" for article and predicted labels

models_address = "./models"
column = args.source
macro_f1s = []
micro_f1s = []
min_loss = 10e10
print(column,context)
import re
for model_address in filter(lambda x:re.match(f"{column}_{context}_[0-9]+", x),os.listdir(path=models_address)):
    path = os.path.join(models_address, model_address)
    print(path)
    strategy_model = RobertaClassifier(article_config)
    loaded = torch.load(path)
    best_model_weights = loaded["best_model_weights"]
    strategy_model.load_state_dict(best_model_weights)
    vaL_loss, val_micro_f1, val_macro_f1 = article_evaluate(model=strategy_model, test_data=test_data, batch_size=article_config.batch_size, column=column, verbose=False)
    micro_f1s.append(val_micro_f1)
    macro_f1s.append(val_macro_f1)
    if vaL_loss < min_loss:
        min_loss =  vaL_loss
        print("saved")
        torch.save(loaded, os.path.join(models_address, f"{column}_{context}_best.pt"))
micro_f1s = np.array(micro_f1s)
macro_f1s = np.array(macro_f1s)
print("############","Average RESULT", "micro", f"{micro_f1s.mean():.3f}", "macro",f"{ macro_f1s.mean():.3f}")