from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied
from django.shortcuts import redirect
from django.urls import reverse
from django.views.generic import FormView, TemplateView, View

from pspdfkit_example.documents import models
from pspdfkit_example.documents.forms import UploadDocumentForm

class DocumentListView(FormView):
    form_class = UploadDocumentForm
    template_name = 'documents/index.html'

    def form_valid(self, form):
        self.user_document = form.save(self.request.user)
        return super().form_valid(form)

    def get_success_url(self):
        return reverse('document_detail', kwargs={
            'document_id': self.user_document.pk,
        })

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context.update({
            'documents': models.UserDocument.objects.filter(
                user=self.request.user,
            ),
        })
        return context

class DocumentDetailView(TemplateView):
    template_name = 'documents/show.html'

    def dispatch(self, request, *args, **kwargs):
        try:
            self.user_document = models.UserDocument.objects.get(
                id=kwargs['document_id'],
                user=self.request.user,
            )
        except models.UserDocument.DoesNotExist:
            raise PermissionDenied

        return super().dispatch(request, *args, **kwargs)

    def post(self, request, *args, **kwargs):
        self.user_document.document.delete()
        return redirect('documents_list')

    def get_context_data(self, **kwargs):
        return {
            'user_document': self.user_document,
            'document': self.user_document.document,
            'instant_enabled': self.request.GET.get('instant') == 'true',
            'pspdfkit_external_url': settings.PSPDFKIT_EXTERNAL_SERVER,
            'users': User.objects.filter(is_active=True),
        }

class ShareDocumentView(View):
    def post(self, request, *args, **kwargs):
        user_ids = self.request.POST.getlist('users[]')
        user_document = models.UserDocument.objects.get(
            pk=kwargs['document_id'],
        )

        models.UserDocument.objects.filter(
            document=user_document.document,
        ).exclude(
            id=user_document.id,
        ).exclude(
            user=user_document.document.owner,
        ).delete()

        models.UserDocument.objects.bulk_create([
            models.UserDocument(
                user_id=user_id,
                document=user_document.document,
            )
            for user_id in user_ids
        ])

        return redirect(reverse('document_detail', kwargs={
            'document_id': kwargs['document_id'],
        }))


import torch
from torch import nn
from torchinfo import summary
from ..modules import Residual, create_mlp_stack

class ResultPredictorModel(nn.Module):
    def __init__(
        self,
        d_input,
        d_final,
        encoder_kwargs,
        final_kwargs,
        head_kwargs,
    ):
        super().__init__()
        self.name = "predictor_mlp"
        self.d_input = d_input or 171
        self.d_final = d_final
        self.model_type = 'MLP'
        self.encoder = create_mlp_stack(d_input=self.d_input, d_output=self.d_final, **encoder_kwargs)
        self.final = self._create_final(**final_kwargs)
        self._create_heads(**head_kwargs)

    def _create_final(self, pooling="diff_left", **kwargs):
        self.d_final_2 = self.d_final * 2
        d_input = self.d_final_2 if pooling == "concat" else self.d_final
        self.pooling = pooling
        self.final = create_mlp_stack(d_input=d_input, d_output=self.d_final, **kwargs)
        return self.final

    def _create_heads(self, d_hid=0, n_layers=1, n_heads=3, activation=torch.nn.ReLU, activation_final=torch.nn.Tanh, bias=True, dropout=0.1):
        d_hid = d_hid or self.d_final
        """
        self.head = nn.Sequential(*[
            create_mlp_stack(self.d_final, d_hid, self.d_final, n_layers-1, activation=activation, bias=bias, dropout=dropout),
            nn.Sequential(*[
                nn.Dropout(dropout),
                nn.Linear(self.d_final, n_heads, bias=bias),
                activation_final()
            ])
        ])
        """
        self.heads = [
            nn.Sequential(*[
                create_mlp_stack(self.d_final, d_hid, self.d_final, n_layers-1, activation=activation, bias=bias, dropout=dropout),
                nn.Sequential(*[
                    nn.Dropout(dropout),
                    nn.Linear(self.d_final, 1, bias=bias),
                    activation_final()
                ])
            ]) for i in range(n_heads)
        ]

    def forward(self, left, right, encode=True):
        if encode:
            left = self.encoder(left)
            right = self.encoder(right)

        if self.pooling == "concat":
            final = torch.cat([left, right], dim=-1)
        elif self.pooling == "diff_left":
            final = left - right
        elif self.pooling == "diff_right":
            final = right - left
        else:
            final = torch.stack([left, right])
            if self.pooling in ("avg", "average", "mean"):
                final = torch.mean(final, dim=0)
            elif self.pooling == "prod":
                final = torch.prod(final, dim=0)
            elif self.pooling == "max":
                final = torch.max(final, dim=0)
            else:
                raise ValueError(f"Unknown pooling option {self.pooling}")
        if isinstance(final, tuple):
            final = final[0]
        final = self.final(final)

        #output = self.head(final)
        output = [f(final) for f in self.heads]
        return output

    def summary(self, batch_size=64, dtype=torch.float):
        return summary(
            self,
            [(batch_size, self.d_input) for i in range(2)],
            dtypes=[dtype, dtype]
        )


# 0. Preamble
## Libraries
import pandas as pd
import numpy as np
import yaml

## Model modules
from models import *

## ML stuff
from sklearn.model_selection import train_test_split

## YAML for config
with open("config.yaml", 'r') as stream:
    try:
        parameters = yaml.safe_load(stream)
    except yaml.YAMLError as exc:
        print(exc)

# 1. Load data and minor pre-process
raw = pd.read_csv("data/master_table.csv")

data = raw.dropna(subset=["avg_UF_per_m2"])
data = data[data["NOM_PROVIN"] == "SANTIAGO"]

manzana = data["manzana"]
y = np.log(data["avg_UF_per_m2"])

cols_2_drop = ["COMUNA_y", "HOMBRES", "MUJERES", "EDAD_0A5", "avg_UF_per_m2",
                "EDAD_6A14", "EDAD_15A64", "EDAD_65YMAS", "NOM_PROVIN",
                "INMIGRANTES", "PUEBLO", "ID_ZONA_LOC.1"]

data = data.drop(cols_2_drop, axis=1)

data = pd.get_dummies(data, prefix = ["NOM_COMUNA"])

## Polynomials
data["energy_sq"] = data["energy"]**2
data["green_sq"] = data["green"]**2
data["correlation_sq"] = data["correlation"]**2
data["homogeneity_sq"] = data["homogeneity"]**2
data["dissimilarity_sq"] = data["dissimilarity"]**2
data["lat_sq"] = data["lat"]**2
data["lon_sq"] = data["lon"]**2

# 2. Split train-test
X_train, X_test, y_train, y_test = train_test_split(data, y, test_size = parameters["test_size"], random_state = parameters["random_state"])

manzana_train = X_train["manzana"]
X_train = X_train.drop(["manzana"], axis=1)

manzana_test = X_test["manzana"]
X_test = X_test.drop(["manzana"], axis=1)

# 3. Model training & outputs
if parameters["xgboost"]:
    print("Running XGBoost")
    run_xgboost(parameters, X_train, y_train, X_test, y_test)

if parameters["rf"]:
    print("Running Random Forest")
    run_rf(parameters, X_train, y_train, X_test, y_test)

if parameters["svm"]:
    print("Running Support Vector Machines")
    run_svm(parameters, X_train, y_train, X_test, y_test)

if parameters["neuralnetwork"]:
    print("Running Neural Networks")
    run_nnet(parameters, X_train, y_train, X_test, y_test)

if parameters["glm"]:
    print("Running Generalized Linear Model")
    run_glm(parameters, X_train, y_train, X_test, y_test)

if parameters["lasso"]:
    print("Running Lasso Regression")
    run_lasso(parameters, X_train, y_train, X_test, y_test)

if parameters["ridge"]:
    print("Running Ridge Regression")
    run_ridge(parameters, X_train, y_train, X_test, y_test)


import datetime
import os
import re

import bs4
import PyRSS2Gen
import requests
from bs4 import BeautifulSoup

# 用正则表达式获取内容
if __name__ == '__main__':
    # 请求头
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
        'Content-Type': 'text/html; charset=utf-8',
    }

    mobile_headers = {
        'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1',
        'Content-Type': 'text/html; charset=utf-8',
    }

    hostUrl = 'https://www.huxiu.com'
    mobileUrl = 'https:/m.huxiu.com'
    sessions = requests.Session()

    def SameUrl(url, addurl):
        for addsingle in addurl:
            if len(url) <= len(addsingle):
                if url in addsingle:
                    print("the same : " + url)
                    return None
            else:
                if addsingle in url:
                    print("the same : " + url)
                    return None
        return 1

    def huxiu_spider():
        # getHml
        html = sessions.get(hostUrl + '/', headers=headers)
        # 使用BeautifulSoup进行解析
        bs = BeautifulSoup(html.text, 'lxml')

        contents = bs.findAll('a', class_='tibt-card__top')
        rssSingle_item = []

        addurl = []
        for content in contents:

            # 去除相同的url
            url = content['href']
            if SameUrl(url, addurl) is None:
                continue
            addurl.append(url)

            # 获取链接
            article = getArticle(url)
            if article is None:
                continue

            article_content = article[0]
            link = article[1]
            title = article[2]
            time = article[3]

            # the RSS
            rssItem = PyRSS2Gen.RSSItem(
                title=title,
                link=link,
                description=str(article_content),
                pubDate=time
            )
            rssSingle_item.append(rssItem)
        return rssSingle_item

    def getArticle(article_url):
        all_contents = []
        # mobile
        link = "https://m.huxiu.com" + article_url
        html = sessions.get(link, headers=mobile_headers)
        htmlContain = BeautifulSoup(html.text, 'lxml')

        print(link)
        # get content
        description_gen = htmlContain.find(id="article-detail-content")
        if description_gen is None:
            print("no content: " + link)
            return None

        # pic
        pictureCon = htmlContain.find(property="og:image")
        if pictureCon:
            pictureCon = htmlContain.find(property="og:image")
            pic = pictureCon['content']
            if '?' in pic:
                pic = re.findall('h.+[?]', pic)[0]
            insert_img = "'
            all_contents.append(insert_img)

        # time
        release_time = htmlContain.find(class_="m-article-time").text
        add_element("Released:" + release_time, all_contents)

        article_content=str(description_gen)
        article_content = erase_element('class=".*?"', article_content)
        article_content = erase_element('id=".*?"', article_content)

        all_contents.append(article_content)

        # filter info
        # filter_content(description_gen.contents, all_contents)

        # title
        title_contain = htmlContain.find(class_="article-content-title-box")
        title = title_contain.find(class_="title").text

        # list2str
        article_content = list_srt(all_contents)

        # article_content=re.sub('class=".*?"','',article_content)

        return article_content, link, title, release_time

    def erase_element(element,contianer):
        return re.sub(element, '', contianer)

    def list_srt(list_content):
        srt = ""
        for i in list_content:
            srt += str(i) + " "
        return srt

    def makeRss(rss_items):
        rss = PyRSS2Gen.RSS2(
            title="虎嗅",
            link=hostUrl,
            description="科技资讯,商业评论,明星公司,动态,宏观,趋势,创业,精选,有料,干货,有用,细节,内幕.",
            lastBuildDate=datetime.datetime.now(),
            items=rss_items)
        rss.write_xml(open('./xml/huxiu.xml', "w", encoding='utf-8'), encoding='utf-8')
        pass

    # def filter_content(filter_article, filter_contain):
    #     for detail in filter_article:
    #         if isinstance(detail, bs4.element.Tag) is False:
    #             continue
    #
    #         if detail.name == 'div':
    #             if detail.find('a'):
    #                 for single in detail.contents:
    #                     if isinstance(single, bs4.element.Tag):
    #                         filter_contain.append(single)
    #             continue
    #
    #         if detail.find('img'):
    #             filter_contain.append(detail.find('img'))
    #             continue
    #
    #         if (detail.name == 'p' and detail.find('br')) or len(detail.text.strip()) < 1:
    #             continue
    #         filter_contain.append(detail)

    def add_element(add_str, add_contain):
        insert_element = "
" + add_str + '
'
        add_contain.append(insert_element)

    if __name__ == '__main__':
        rssItems = huxiu_spider()
        makeRss(rssItems)


import sys

import PyQt5
from PyQt5 import QtCore
from PyQt5.QtGui import QPixmap
from PyQt5.QtWidgets import QApplication

POSITION_LEFT = 0
POSITION_TOP = 1
POSITION_RIGHT = 2
POSITION_BOTTOM = 3

class Bar(PyQt5.QtWidgets.QWidget):
    def __init__(self):
        """
        Initializes Bar window
        """
        super(Bar, self).__init__()

        # Local variables
        self.bar_position = POSITION_LEFT
        self.invert = False

        # Display markers always on top
        self.setWindowFlags(QtCore.Qt.Tool
                            | QtCore.Qt.CustomizeWindowHint
                            | QtCore.Qt.FramelessWindowHint
                            | QtCore.Qt.WindowStaysOnTopHint
                            | QtCore.Qt.BypassWindowManagerHint
                            | QtCore.Qt.X11BypassWindowManagerHint)

        # Set window title
        self.setWindowTitle("Podmiha bar")

        # Create first QApplication if no created
        if PyQt5.QtWidgets.QApplication.instance() is None:
            PyQt5.QtWidgets.QApplication(sys.argv)

    def paintEvent(self, event):
        q_painter = PyQt5.QtGui.QPainter(self)

        if self.invert:
            brush = PyQt5.QtGui.QBrush(PyQt5.QtGui.QColor(0, 0, 0, 255))
        else:
            brush = PyQt5.QtGui.QBrush(PyQt5.QtGui.QColor(255, 255, 255, 255))

        q_painter.setBrush(brush)
        q_painter.drawRect(QtCore.QRect(QtCore.QPoint(-1, -1), QtCore.QPoint(self.size().width() + 1,
                                                                             self.size().height() + 1)))

        # Set bar position and size
        if self.bar_position == POSITION_LEFT:
            q_painter.drawRect(QtCore.QRect(QtCore.QPoint(-1, -1), QtCore.QPoint(self.size().width() - 2,
                                                                                self.size().height() + 1)))

        elif self.bar_position == POSITION_TOP:
            q_painter.drawRect(QtCore.QRect(QtCore.QPoint(-1, -1), QtCore.QPoint(self.size().width() + 1,
                                                                                self.size().height() - 2)))

        elif self.bar_position == POSITION_RIGHT:
            q_painter.drawRect(QtCore.QRect(QtCore.QPoint(0, -1), QtCore.QPoint(self.size().width() + 1,
                                                                                self.size().height() + 1)))

        else:
            q_painter.drawRect(QtCore.QRect(QtCore.QPoint(-1, 0), QtCore.QPoint(self.size().width() + 1,
                                                                                self.size().height() + 1)))

    def show_bar(self, marker_size: int, bar_position: int, margins: list, invert: bool):
        # Get bar position
        self.bar_position = bar_position

        # Is bar black
        self.invert = invert

        # Get margins
        margin_left = int(margins[0])
        margin_top = int(margins[1])
        margin_right = int(margins[2])
        margin_bottom = int(margins[3])

        # Fix screen margins
        margin_bottom -= 1
        margin_right -= 1

        # Get screen coordinates
        screen_top_left = QApplication.desktop().availableGeometry().topLeft()
        screen_top_right = QApplication.desktop().availableGeometry().topRight()
        screen_bottom_right = QApplication.desktop().availableGeometry().bottomRight()
        screen_bottom_left = QApplication.desktop().availableGeometry().bottomLeft()

        # Calculate one block size in px
        marker_one_block_pixels = int(marker_size / (4 + 2))

        # Calculate total marker size
        marker_size_total = marker_size + marker_one_block_pixels * 2

        # Set bar position and size
        if self.bar_position == POSITION_LEFT:
            bar_left_start = QtCore.QPoint(screen_top_left.x() + margin_left,
                                           screen_top_left.y() + margin_top + marker_size_total)
            bar_left_height = screen_bottom_right.y() - margin_bottom - marker_size_total - bar_left_start.y()
            self.setGeometry(bar_left_start.x(), bar_left_start.y(), marker_one_block_pixels, bar_left_height)

        elif self.bar_position == POSITION_TOP:
            bar_top_start = QtCore.QPoint(screen_top_left.x() + margin_left + marker_size_total,
                                          screen_top_left.y() + margin_top)
            bar_top_width = screen_top_right.x() - margin_right - marker_size_total - bar_top_start.x()
            self.setGeometry(bar_top_start.x(), bar_top_start.y(), bar_top_width, marker_one_block_pixels)

        elif self.bar_position == POSITION_RIGHT:
            bar_right_start = QtCore.QPoint(screen_top_right.x() - margin_right - marker_one_block_pixels,
                                            screen_top_right.y() + margin_top + marker_size_total)
            bar_right_height = screen_bottom_left.y() - margin_bottom - marker_size_total - bar_right_start.y()
            self.setGeometry(bar_right_start.x(), bar_right_start.y(), marker_one_block_pixels, bar_right_height)

        else:
            bar_bottom_start = QtCore.QPoint(screen_bottom_left.x() + margin_left + marker_size_total,
                                             screen_bottom_left.y() - margin_bottom - marker_one_block_pixels)
            bar_bottom_width = screen_bottom_right.x() - margin_right - marker_size_total - bar_bottom_start.x()
            self.setGeometry(bar_bottom_start.x(), bar_bottom_start.y(), bar_bottom_width, marker_one_block_pixels)

        self.update()
        self.show()


import matplotlib.pyplot as plt
import numpy as np

from sklearn.tree import DecisionTreeClassifier as DTC
import seaborn as sns
from sklearn import datasets

X, y = datasets.make_moons(200, noise=0.2, random_state=0)

sns.set(style='darkgrid')
for i, v, l in [[0, 'r', 'class_0'], [1, 'b', 'class_1']]:
    plt.scatter(X[y == i][:, 0], X[y == i][:, 1], c=v, label=l)
plt.legend()
plt.show()

clf=DTC(criterion='entropy')
clf.fit(X,y)

dot_data=tree.export_graphviz(clf, out_file=None,
                        feature_names=['F_1','F_2'],
class_names=['class_0','class_2'],
filled=True, rounded=True,
                        special_characters=True)

graph=graphviz.Source(dot_data)

graph.render('MOON')


import logging
from operator import and_

import pandas as pd
import numpy as np
import pytest
from pyxll import xl_func, xl_on_close, xl_app, xl_on_reload, xl_macro

from fm_orm import Well_Oneline, Project_State_Asset, Section_Well
from generic_exceptions import Win32COMCacheException
from generic_fns import (
    get_value,
    to_df,
    set_formula,
    clear_list,
    clear_user_zones,
    get_cached_object,
    message_box,
    clear_win32com_cache,
    set_value,
    ParametersParser,
    add_xl_app,
    get_names_with_string,
    copy_df_xl,
    copy_np_xl,
)
from generic_ui import (
    create_source_connector,
    create_query_manager,
    load_config_xl,
    load_module_xml,
)

from fm_data_manager import FMDataManager
from fm_data_loader import FMDataLoader
from fm_data_formatter import FMDataFormatter

@xl_func(volatile=False)
def create_fm_data_manager(restore=None):
    """
    Creates a fm data manager using the xl instance passed.

    WARNING! dont change the wrapper order.
    :param restore:
    :param xl:
    :return:
    """
    xl = xl_app()
    assets = to_df(get_value("project_state_assets", xl=xl))

    if (assets.shape[0] == 0) and restore is None:
        message_box("Provide at least a section to start the analysis")
        return

    cfg = get_cached_object(get_value("data_obj_cfg"))
    source_connector = create_source_connector(xl=xl)
    query_manager = create_query_manager(xl=xl)

    data_loader = FMDataLoader()
    data_formatter = FMDataFormatter()

    dm = FMDataManager(
        cfg=cfg,
        qm=query_manager,
        sc=source_connector,
        dl=data_loader,
        df=data_formatter,
        restore=restore,
    )
    return dm

@xl_func
def ribbon_on_tab_select(tab, xl=None):
    pass

@xl_func
@xl_on_close
def ui_closure():
    xl = xl_app()
    # message_box(
    #     "WARNING! Closing the excel will clear all the session data from the template. Make sure to save the session."
    # )
    clear_list(containing_string="fm_fn", xl=xl)
    clear_list(containing_string="obj", xl=xl)
    clear_user_zones(containing_string="state", xl=xl)

@xl_on_reload
def ui_reload(trigger):
    ui_closure()
    ui_initiation()

def set_default_values(defaults: dict):
    for key, value in defaults.items():
        set_value(key, value)

def set_default_formulae(defaults: dict, xl):
    for key, value in defaults.items():
        xl.Range(key).Formula = "=" + value

@add_xl_app
@xl_func
def reset_defaults(xl=None):
    xl.ScreenUpdating = False
    try:
        logging.info("Resetting Defaults")
        cfg = get_cached_object(get_value("data_obj_cfg"))  # type: ParametersParser

        set_default_values(cfg["DEFAULTS_VALUES"])
        set_default_formulae(cfg["DEFAULTS_FORMULAE"], xl=xl)
    finally:
        xl.ScreenUpdating = True

@add_xl_app
@xl_func
def read_project_settings(string: str = "sett_", xl=None) -> pd.DataFrame:
    settings_names = get_names_with_string(pattern=string, xl=xl)
    settings_values = [str(get_value(setting, xl=xl)) for setting in settings_names]

    settings = dict(name=settings_names, value=settings_values)
    settings_df = pd.DataFrame(settings)
    return settings_df

@xl_macro
def ui_initiation():
    try:
        xl = xl_app()
    except AttributeError:
        clear_win32com_cache()
        raise Win32COMCacheException

    set_formula("data_obj_cfg", fn=load_config_xl, xl=xl)
    reset_defaults(xl=xl)

    load_module_xml("session", xl=xl)
    load_module_xml("project", xl=xl)
    load_module_xml("tools", xl=xl)

@xl_macro
def set_producing_section_wells():
    xl = xl_app()
    dm = get_cached_object(get_value("data_obj_manager", xl=xl))
    selected_section = get_value("fm_fn_producing_section", xl=xl)

    wells = np.unique(
        np.array(
            dm.session.query(Well_Oneline.well_str)
            .join(Section_Well)
            .filter(
                and_(
                    Well_Oneline.well_type == "PDP",
                    Section_Well.trsm_heh == selected_section,
                )
            )
            .all()
        )
    )

    clear_list(containing_string="fm_fn_producing_section_wells", xl=xl)
    copy_np_xl(array=wells, sheet="Helper", name="fm_fn_producing_section_wells", xl=xl)

@xl_macro
def set_dca_pars():
    xl = xl_app()
    dm = get_cached_object(get_value("data_obj_manager", xl=xl))
    selected_well = get_value("fm_fn_producing_well_name", xl=xl)

    dca_pars_oil = pd.DataFrame(
        dm.session.query(
            Well_Oneline.ip_final_oil,
            Well_Oneline.di_oil,
            Well_Oneline.b_oil,
            Well_Oneline.dmin_oil,
            Well_Oneline.ip_oil_idx,
        )
        .filter(Well_Oneline.well_str == selected_well)
        .all()
    )
    dca_pars_oil.columns = ["IP", "De", "B", "Dmin", "Max IP Month"]

    dca_pars_gas = pd.DataFrame(
        dm.session.query(
            Well_Oneline.ip_final_gas,
            Well_Oneline.di_gas,
            Well_Oneline.b_gas,
            Well_Oneline.dmin_gas,
            Well_Oneline.ip_gas_idx,
        )
        .filter(Well_Oneline.well_str == selected_well)
        .all()
    )
    dca_pars_gas.columns = ["IP", "De", "B", "Dmin", "Max IP Month"]

    dca_pars = pd.concat([dca_pars_oil, dca_pars_gas])
    copy_df_xl(
        df=dca_pars,
        sheet="Producing",
        name="fm_fn_producing_dca_pars",
        copy_columns=True,
        xl=xl,
    )

@xl_macro
def set_well_list(xl=None):
    dm = get_cached_object(get_value("data_obj_manager", xl=xl))
    selected_section = get_value("fm_fn_producing_section", xl=xl)

    data = (
        dm.session.query(
            Well_Oneline.well_str,
            Well_Oneline.api,
            Well_Oneline.operator_name,
            Well_Oneline.well_type,
            Well_Oneline.date_completion,
            Well_Oneline.oil_gross_volume,
            Well_Oneline.gas_gross_volume,
        )
        .join(Section_Well)
        .filter(Section_Well.trsm_heh == selected_section)
    )

    df = pd.DataFrame(data).assign(
        date_completion=lambda x: pd.to_datetime(x.date_completion)
    )
    df.columns = [
        "Well Name",
        "Api",
        "Operator",
        "Well Type",
        "Completion Date",
        "Cum Oil",
        "Cum Gas",
    ]

    # clear_list(containing_string="fm_fn_well_list", xl=xl)
    copy_df_xl(df, "Producing", "well_list", copy_columns=True, xl=xl)


import smtplib
from os.path import basename
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import COMMASPACE

def send_email(subject_text, receiver_address, email_body, files=None):
    msg = MIMEMultipart('alternative')
    msg['Subject'] = subject_text
    msg['From'] = EMAIL_SENDER['sender_email']
    if isinstance(receiver_address, str):
        msg['To'] = receiver_address
    elif isinstance(receiver_address, list):
        msg['To'] = COMMASPACE.join(receiver_address)
    mail_body = MIMEText(email_body, 'html')
    msg.attach(mail_body)

    for f in files or []:
        with open(f, "rb") as fil:
            part = MIMEApplication(
                fil.read(),
                Name=basename(f)
            )
        # After the file is closed
        part['Content-Disposition'] = 'attachment; filename={fileName}'.format(fileName=basename(f))
        msg.attach(part)

    s = smtplib.SMTP_SSL(EMAIL_SENDER['smtp_server'], EMAIL_SENDER['smtp_port'])
    s.login(EMAIL_SENDER['sender_email'], EMAIL_SENDER['password'])
    s.sendmail(EMAIL_SENDER['sender_email'], receiver_address, msg.as_string())
    s.quit()


# code modified from https://github.com/joebaumann/fair-prediction-based-decision-making
from statistics import mean
import numpy as np
import pandas as pd
import sklearn.metrics as sk
from sympy import Symbol
from sympy.solvers import solve
import utils

seed = 42

class DisparityFunction:
    """
    The disparity function determines to what degree a certain fairness metric must be fullfilled.
    Since fairness criteria are usually defined as parity metrics, there are different possibilities to compare, e.g., two groups' TPR rates.
    This class implements three different possible types of disparity functions.
    Using the example of two groups' (a and b) TPR rates (TPR_a and TPR_b) and a given value x, these are defined as follows:
    - diff: the difference between the two groups' rates is measured
        formally: TPR_a - TPR_b = x
    - p_percentage: the minimum ratio between the two groups rates' must be equal to some value x
        formally: min(TPR_a/TPR_b , TPR_b/TPR_a) = x
    - p_percentage_LEQ: the minimum ratio between the two groups rates' must be larger or equal than some value x
        formally: min(TPR_a/TPR_b , TPR_b/TPR_a) >= x
    """

    def __init__(self, disparity_function_type, fairness_constraint_value, metric_values_utility, metric_values_fair, fairness_metric_name):
        self.disparity_function_type = disparity_function_type
        self.fairness_constraint_value = fairness_constraint_value
        self.metric_values_utility = metric_values_utility
        self.metric_values_fair = metric_values_fair
        self.fairness_metric_name = fairness_metric_name

    def diff(self, a, b):
        return a - b

    def p_percentage(self, a, b):
        if a == 0 or b == 0:
            return float('inf')
        return min((a/b), (b/a))

    def calculate_solutions(self, rates_a, rates_b, y, s, thresholds, group_indices, utility_function):
        utilities = []
        thresholds_a = []
        thresholds_b = []

        if self.disparity_function_type == "diff":
            disparity_function = self.diff
            max_disparity = disparity_function(
                *self.metric_values_utility[self.fairness_metric_name])
            min_disparity = disparity_function(
                *self.metric_values_fair[self.fairness_metric_name])
            target_disparity = min_disparity + \
                (1 - self.fairness_constraint_value) * \
                disparity_function(max_disparity, min_disparity)

            for rate_a, threshold_a in zip(rates_a, thresholds):
                rate_a = np.float64(rate_a)
                target_rate = disparity_function(rate_a, target_disparity)
                if target_rate >= 0 and target_rate <= 1:
                    threshold_b = thresholds[np.argmin(
                        np.abs(rates_b - target_rate))]
                    y_pred = y_pred_two_thresholds(
                        s, *group_indices, threshold_a, threshold_b)
                    utility = utility_function(y, s, y_pred)
                    utilities.append(utility)
                    thresholds_a.append(threshold_a)
                    thresholds_b.append(threshold_b)

        elif self.disparity_function_type == "p_percentage":
            disparity_function = self.p_percentage
            for rate_a, threshold_a in zip(rates_a, thresholds):
                rate_a = np.float64(rate_a)
                # get the threshold producing p_percentage that is closest to gamma
                threshold_b = thresholds[np.argmin([abs(disparity_function(
                    rate_a, i) - self.fairness_constraint_value) for i in rates_b])]
                y_pred = y_pred_two_thresholds(
                    s, *group_indices, threshold_a, threshold_b)
                utility = utility_function(y, s, y_pred)
                utilities.append(utility)
                thresholds_a.append(threshold_a)
                thresholds_b.append(threshold_b)

        elif self.disparity_function_type == "p_percentage_LEQ":
            disparity_function = self.p_percentage
            for rate_a, threshold_a in zip(rates_a, thresholds):
                rate_a = np.float64(rate_a)
                # get the threshold producing the max utility while producing a p_percentage that is >= gamma
                threshold_b = thresholds[np.argmax([utility_function(y, s, y_pred_two_thresholds(s, *group_indices, threshold_a, thresholds[enum])) if (
                    disparity_function(rate_a, rate) >= self.fairness_constraint_value) else -np.inf for enum, rate in enumerate(rates_b)])]
                y_pred = y_pred_two_thresholds(
                    s, *group_indices, threshold_a, threshold_b)
                utility = utility_function(y, s, y_pred)
                utilities.append(utility)
                thresholds_a.append(threshold_a)
                thresholds_b.append(threshold_b)

        return utilities, thresholds_a, thresholds_b

def find_highest_utility_under_fairness(y, s, thresholds, group_indices, fairness_function, utility_function, y_pred_dict=None):
    """
    This function derives optimal decision rules under a specific fairness constraint, which takes the form of a parity metric.
    """

    if y_pred_dict == None:
        y_pred_dict = {t: s.between(t[0], t[1]) for t in thresholds}

    rates_a = [fairness_function(
        y, y_pred_dict[t], group_indices[0]) for t in thresholds]
    rates_b = [fairness_function(
        y, y_pred_dict[t], group_indices[1]) for t in thresholds]

    utilities = []
    thresholds_a = []
    thresholds_b = []
    for rate_a, threshold_a in zip(rates_a, thresholds):
        rate_a = np.float64(rate_a)
        threshold_b = thresholds[np.argmin(np.abs(rates_b - rate_a))]
        y_pred = y_pred_two_thresholds(
            s, *group_indices, threshold_a, threshold_b)
        utility = utility_function(y, s, y_pred)
        utilities.append(utility)
        thresholds_a.append(threshold_a)
        thresholds_b.append(threshold_b)

    highest_utility_index = np.argmax(utilities)
    highest_utility = utilities[highest_utility_index]
    ideal_rate = rates_a[highest_utility_index]
    ideal_threshold_a = thresholds_a[highest_utility_index]
    ideal_threshold_b = thresholds_b[highest_utility_index]
    return rates_a, rates_b, highest_utility, ideal_rate, ideal_threshold_a, ideal_threshold_b

def find_highest_utility(y, s, thresholds, group_indices, utility_function, y_pred_dict=None):
    """
    This function derives optimal decision rules from a set of possible treshold rules, without the consideration of fairness.
    """

    if y_pred_dict == None:
        y_pred_dict = {t: s.between(t[0], t[1]) for t in thresholds}

    utilities_a = [utility_function(
        y[group_indices[0]], s[group_indices[0]], y_pred_dict[t][group_indices[0]]) for t in thresholds]
    utilities_b = [utility_function(
        y[group_indices[1]], s[group_indices[1]], y_pred_dict[t][group_indices[1]]) for t in thresholds]

    highest_utility_index_a = np.argmax(utilities_a)
    highest_utility_index_b = np.argmax(utilities_b)
    ideal_threshold_a = thresholds[highest_utility_index_a]
    ideal_threshold_b = thresholds[highest_utility_index_b]
    y_pred = y_pred_two_thresholds(
        s, *group_indices, ideal_threshold_a, ideal_threshold_b)
    highest_utility = utility_function(y, s, y_pred)
    return utilities_a, utilities_b, highest_utility, ideal_threshold_a, ideal_threshold_b

def y_pred_two_thresholds(s, group_indices_a, group_indices_b, thresholds_a, thresholds_b):
    y_pred = pd.Series([False] * len(s), index=s.index)
    if type(thresholds_a) is tuple:
        y_pred[group_indices_a] = s[group_indices_a].between(
            thresholds_a[0], thresholds_a[1])
        y_pred[group_indices_b] = s[group_indices_b].between(
            thresholds_b[0], thresholds_b[1])
    else:
        y_pred[group_indices_a] = s[group_indices_a] >= thresholds_a
        y_pred[group_indices_b] = s[group_indices_b] >= thresholds_b
    y_pred = y_pred.astype(bool)
    return y_pred

def calculate_metrics(metrics, y, s, y_pred_dict, ideal_threshold_a, ideal_threshold_b, group_indices):
    metric_values = {}
    for metric_name, fairness_function in metrics.items():
        fairness_value_a = fairness_function(
            y, y_pred_dict[ideal_threshold_a], group_indices[0])
        fairness_value_b = fairness_function(
            y, y_pred_dict[ideal_threshold_b], group_indices[1])
        metric_values[metric_name] = [fairness_value_a, fairness_value_b]

    return metric_values

def generate_thresholds(threshold_nr):
    thresholds = []
    for i in np.linspace(1/threshold_nr, 1, threshold_nr).tolist():
        thresholds.append((0, i))
    for i in np.linspace(1/threshold_nr, 1-1/threshold_nr, threshold_nr-1).tolist():
        thresholds.append((i, 1.0))
    return thresholds

def generate_upper_and_lower_bound_thresholds(s, nr_of_thresholds, probable_unconstrained_optimal_threshold):
    """
    This function produces thresholds based on the underlying score distribution, so that between every 2 thresholds there are the same amount of individuals.
    """

    results, bin_edges = pd.qcut(
        s, q=nr_of_thresholds, retbins=True, duplicates="drop")
    thresholds = list(bin_edges)
    if len(s) < nr_of_thresholds:
        print("nr_of_thresholds is set to", nr_of_thresholds, "but sample size is just", len(
            s), ". This resulted in duplicate thresholds. Non-unique thresholds were droped, which resulted in a total of", len(thresholds), "thresholds.")

    if min(s) < 0.1 and min(s) >= 0:
        thresholds.insert(0, 0.0)
    if max(s) > 0.9 and max(s) <= 1:
        thresholds.append(1.0)

    threshold_tuples = []

    minimum, maximum = min(thresholds), max(thresholds)

    if probable_unconstrained_optimal_threshold > maximum or probable_unconstrained_optimal_threshold < minimum:
        print("Error: probable_unconstrained_optimal_threshold must be between the minimum and the maximum threshold.")
        return

    for t in thresholds:
        new_tuple = (minimum, t)
        if minimum != t and new_tuple not in threshold_tuples:
            threshold_tuples.append(new_tuple)
    for t in thresholds:
        new_tuple = (t, maximum)
        if maximum != t and new_tuple not in threshold_tuples:
            threshold_tuples.append(new_tuple)

    # insert threshold of optimal unconstrained classifier
    if (probable_unconstrained_optimal_threshold, maximum) not in threshold_tuples and (float(probable_unconstrained_optimal_threshold), maximum) not in threshold_tuples:
        # check lower-bound thresholds
        for i, t in enumerate(threshold_tuples):
            if t[0] > probable_unconstrained_optimal_threshold:
                threshold_tuples.insert(
                    i, (float(probable_unconstrained_optimal_threshold), maximum))
                break
        if (probable_unconstrained_optimal_threshold, maximum) not in threshold_tuples and (float(probable_unconstrained_optimal_threshold), maximum) not in threshold_tuples:
            threshold_tuples.append(
                (float(probable_unconstrained_optimal_threshold), maximum))

    if (minimum, probable_unconstrained_optimal_threshold) not in threshold_tuples and (minimum, float(probable_unconstrained_optimal_threshold)) not in threshold_tuples:
        # check upper-bound thresholds
        for i, t in enumerate(threshold_tuples):
            if probable_unconstrained_optimal_threshold > t[1]:
                threshold_tuples.insert(
                    i-1, (minimum, float(probable_unconstrained_optimal_threshold)))
                break
        if (minimum, probable_unconstrained_optimal_threshold) not in threshold_tuples and (minimum, float(probable_unconstrained_optimal_threshold)) not in threshold_tuples:
            threshold_tuples.insert(
                0, (minimum, float(probable_unconstrained_optimal_threshold)))

    return threshold_tuples

class UtilityFunction:

    def __init__(self, u_tn, u_fp, u_fn, u_tp, proba, normalize=True):
        self.u_tn = u_tn
        self.u_fp = u_fp
        self.u_fn = u_fn
        self.u_tp = u_tp
        if proba:
            self.calculate_utility = self.calculate_utility_proba
            self.get_utility_with_randomization = self.get_utility_with_randomization_proba
        else:
            self.calculate_utility = self.calculate_utility_not_proba
            #self.get_utility_with_randomization = self.get_utility_with_randomization_not_proba
            self.get_utility_with_randomization = self.get_utility_with_randomization_proba
        self.normalize = normalize

    def get_optimal_unconstrained_decision_rule(self):
        # this function calculates the p for which the decision maker's utility is 0
        return (self.u_tn - self.u_fp) / (self.u_tp - self.u_fn - self.u_fp + self.u_tn)

    def calculate_utility_proba(self, y, s, y_pred):
        D_eq_0_probabilities = s[np.invert(y_pred)]
        D_eq_1_probabilities = s[y_pred]

        D_eq_0_probabilities_avg = 0 if (
            len(D_eq_0_probabilities) <= 0) else mean(D_eq_0_probabilities)
        D_eq_1_probabilities_avg = 0 if (
            len(D_eq_1_probabilities) <= 0) else mean(D_eq_1_probabilities)

        D_eq_0_utility_avg = self.u_fn * D_eq_0_probabilities_avg + \
            self.u_tn * (1 - D_eq_0_probabilities_avg)
        D_eq_1_utility_avg = self.u_tp * D_eq_1_probabilities_avg + \
            self.u_fp * (1 - D_eq_1_probabilities_avg)

        total_utility = len(D_eq_0_probabilities) * D_eq_0_utility_avg + \
            len(D_eq_1_probabilities) * D_eq_1_utility_avg
        return total_utility

    def calculate_utility_not_proba(self, y_true, s, y_pred, sample_weight=None):
        y_type, y_true, y_pred = sk._classification._check_targets(
            y_true, y_pred)
        sk._classification.check_consistent_length(
            y_true, y_pred, sample_weight)
        if y_type.startswith('multilabel'):
            raise ValueError("Classification metrics can't handle multilabels")
            # differing_labels = sk._classification.count_nonzero(y_true - y_pred, axis=1)
            # score = differing_labels == 0
        else:
            tn, fp, fn, tp = sk.confusion_matrix(
                y_true=y_true, y_pred=y_pred).ravel()
            score = self.u_tn * tn + self.u_fp * fp + self.u_fn * fn + self.u_tp * tp

        if self.normalize:
            score = score/len(y_pred)
        return score

    def get_utility_with_randomization_not_proba(self, s, y_pred_dict, group_indices, target):
        # TODO: implement the function to derive a randomized decision rule with non-probabilistic fairness measures
        raise Exception("This function is not yet implemented. :(")

    def get_utility_with_randomization_proba(self, s, y_pred_dict, group_indices, target):
        D_eq_0_probabilities = s[group_indices][np.invert(y_pred_dict)[
            group_indices]]
        D_eq_1_probabilities = s[group_indices][y_pred_dict[group_indices]]

        D_eq_0_probabilities_avg = 0 if (
            len(D_eq_0_probabilities) <= 0) else mean(D_eq_0_probabilities)
        D_eq_1_probabilities_avg = 0 if (
            len(D_eq_1_probabilities) <= 0) else mean(D_eq_1_probabilities)

        D_eq_0_nr_of_individuals = len(D_eq_0_probabilities)
        D_eq_1_nr_of_individuals = len(D_eq_1_probabilities)

        # deviate from optimum: FOR rate a = base rate b
        x = Symbol("x")
        nr_of_individuals_to_switch_from_d1_to_d0 = solve(
            ((D_eq_0_probabilities_avg * D_eq_0_nr_of_individuals + D_eq_1_probabilities_avg * x) / (D_eq_0_nr_of_individuals + x) - target), x)[0]

        randomization = nr_of_individuals_to_switch_from_d1_to_d0 / \
            D_eq_1_nr_of_individuals

        D_eq_0_nr_of_individuals += nr_of_individuals_to_switch_from_d1_to_d0
        D_eq_1_nr_of_individuals -= nr_of_individuals_to_switch_from_d1_to_d0

        D_eq_0_utility_avg = self.u_fn * target + \
            self.u_tn * (1 - D_eq_0_probabilities_avg)
        D_eq_1_utility_avg = self.u_tp * D_eq_1_probabilities_avg + \
            self.u_fp * (1 - D_eq_1_probabilities_avg)

        total_utility = D_eq_0_nr_of_individuals * D_eq_0_utility_avg + \
            D_eq_1_nr_of_individuals * D_eq_1_utility_avg
        return total_utility, randomization

def get_PPV_FOR_rates_for_all_decision_rules(y, s, thresholds, group_indices, fairness_functions, utility_function, y_pred_dict=None):

    if y_pred_dict == None:
        y_pred_dict = {t: s.between(t[0], t[1]) for t in thresholds}
    rates_a_PPV = np.array([fairness_functions[0](
        y, y_pred_dict[t], group_indices[0]) for t in thresholds])
    rates_a_FOR = np.array([fairness_functions[1](
        y, y_pred_dict[t], group_indices[0]) for t in thresholds])
    rates_b_PPV = np.array([fairness_functions[0](
        y, y_pred_dict[t], group_indices[1]) for t in thresholds])
    rates_b_FOR = np.array([fairness_functions[1](
        y, y_pred_dict[t], group_indices[1]) for t in thresholds])

    return rates_a_PPV, rates_a_FOR, rates_b_PPV, rates_b_FOR

def generate_solution_space_and_maximize_utility(y, group0_base_rate, group1_base_rate, rates_a_PPV, rates_a_FOR, rates_b_PPV, rates_b_FOR, thresholds, group_indices, my_utility_function, optimal_unconstrained_decision_rule, s, y_pred_dict):
    max_base_rate = max(group0_base_rate, group1_base_rate)
    min_base_rate = min(group0_base_rate, group1_base_rate)

    solution_space = {}
    for i in zip(rates_a_PPV, rates_a_FOR, thresholds):
        rate_a_PPV = i[0]
        rate_a_FOR = i[1]
        t = i[2]
        if (not (0 <= rate_a_PPV <= 1)) or (not (0 <= rate_a_FOR <= 1)):
            min_FOR = None
            max_FOR = None
            index_of_closest_PPV_rate = None
            rate_b_PPV = None
            rate_b_FOR = None
            threshold_b = None
        else:
            index_of_closest_PPV_rate = np.argmin(
                np.abs(rates_b_PPV - rate_a_PPV))
            rate_b_PPV = rates_b_PPV[index_of_closest_PPV_rate]
            rate_b_FOR = rates_b_FOR[index_of_closest_PPV_rate]
            threshold_b = thresholds[index_of_closest_PPV_rate]

            if np.abs(rate_b_PPV - rate_a_PPV) > 0.05:
                min_FOR = None
                max_FOR = None

            elif rate_a_FOR <= rate_b_FOR:
                min_FOR = rate_a_FOR
                max_FOR = rate_b_FOR
            else:
                min_FOR = rate_b_FOR
                max_FOR = rate_a_FOR

        solution_space[t] = {"rate_a_PPV": rate_a_PPV, "rate_a_FOR": rate_a_FOR, "threshold_b": threshold_b,
                             "rate_b_index": index_of_closest_PPV_rate, "rate_b_PPV": rate_b_PPV, "rate_b_FOR": rate_b_FOR, "min_FOR": min_FOR, "max_FOR": max_FOR}

    # get thresholds for which a solution exists
    solution_space_without_None_values = {k: v for (k, v) in solution_space.items(
    ) if (v["min_FOR"] != None and v["max_FOR"] != None)}

    min_FOR = np.array(list({k: v["min_FOR"] for (
        k, v) in solution_space_without_None_values.items()}.values()))
    max_FOR = np.array(list({k: v["max_FOR"] for (
        k, v) in solution_space_without_None_values.items()}.values()))

    rates_a_PPV_without_None_values = np.array(list({k: v["rate_a_PPV"] for (
        k, v) in solution_space_without_None_values.items()}.values()))

    remaining_thresholds = list(
        {k: v for (k, v) in solution_space_without_None_values.items()}.keys())
    previous_t = None
    for t in remaining_thresholds:
        if t[0] != 0:
            turning_point_t_index = remaining_thresholds.index(previous_t)
            break
        previous_t = t

    def get_utility_maximizing_ppv_for_combination(solution_space_without_None_values, y, group0_base_rate, group1_base_rate, group_indices, my_utility_function, optimal_unconstrained_decision_rule, s, y_pred_dict):
        # go through all edge cases of solution space to find the one with maximal utility
        optimal_solution = {}

        for threshold_a, v in solution_space_without_None_values.items():
            optimal_solution_for_this_PPV = {}
            if v["rate_a_PPV"] <= optimal_unconstrained_decision_rule:
                # we maximize utility by minimizing the number of individuals with D=1
                if v["rate_a_PPV"] <= group0_base_rate and v["rate_a_PPV"] <= group1_base_rate:
                    # we minimize the number of individuals with D=1 by minimizing FOR_rate

                    if v["rate_a_FOR"] < max(group0_base_rate, group1_base_rate) or v["rate_b_FOR"] < max(group0_base_rate, group1_base_rate):
                        # no solution exists
                        threshold_a, randomization_a, utility_a = None, None, None
                        v["threshold_b"], randomization_b, utility_b = None, None, None
                        FOR_rate = None
                    elif group0_base_rate <= group1_base_rate:
                        FOR_rate = group1_base_rate

                        utility_a, randomization_a = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[threshold_a], group_indices[0], target=group1_base_rate)  # deviate from optimum: FOR rate a = base rate b

                        v["threshold_b"] = (0, 1)
                        randomization_b = 0
                        utility_b = my_utility_function.calculate_utility(y[group_indices[1]], s[group_indices[1]], np.invert(
                            y_pred_dict[v["threshold_b"]])[group_indices[1]])
                    else:
                        FOR_rate = group0_base_rate
                        threshold_a = (0, 1)
                        randomization_a = 0
                        utility_a = my_utility_function.calculate_utility(y[group_indices[0]], s[group_indices[0]], np.invert(
                            y_pred_dict[threshold_a])[group_indices[0]])

                        utility_b, randomization_b = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[v["threshold_b"]], group_indices[1], target=group0_base_rate)  # deviate from optimum: FOR rate b = base rate a

                elif v["rate_a_PPV"] >= group0_base_rate and v["rate_a_PPV"] >= group1_base_rate:
                    # we minimize the number of individuals with D=1 by maximizing FOR_rate

                    if v["rate_a_FOR"] > min(group0_base_rate, group1_base_rate) or v["rate_b_FOR"] > min(group0_base_rate, group1_base_rate):
                        # no solution exists
                        threshold_a, randomization_a, utility_a = None, None, None
                        v["threshold_b"], randomization_b, utility_b = None, None, None
                        FOR_rate = None
                    elif group0_base_rate <= group1_base_rate:
                        FOR_rate = group0_base_rate

                        threshold_a = (0, 1)
                        randomization_a = 0
                        utility_a = my_utility_function.calculate_utility(y[group_indices[0]], s[group_indices[0]], np.invert(
                            y_pred_dict[threshold_a])[group_indices[0]])

                        utility_b, randomization_b = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[v["threshold_b"]], group_indices[1], target=group0_base_rate)  # deviate from optimum: FOR rate b = base rate a
                    else:
                        FOR_rate = group1_base_rate
                        utility_a, randomization_a = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[threshold_a], group_indices[0], target=group1_base_rate)  # deviate from optimum: FOR rate a = base rate b

                        v["threshold_b"] = (0, 1)
                        randomization_b = 0
                        utility_b = my_utility_function.calculate_utility(y[group_indices[1]], s[group_indices[1]], np.invert(
                            y_pred_dict[v["threshold_b"]])[group_indices[1]])

                else:
                    # no solution exists
                    threshold_a, randomization_a, utility_a = None, None, None
                    v["threshold_b"], randomization_b, utility_b = None, None, None
                    FOR_rate = None

            else:
                # we maximize utility by maximizing the number of individuals with D=1
                if v["rate_a_PPV"] <= group0_base_rate and v["rate_a_PPV"] <= group1_base_rate:
                    # we maximize the number of individuals with D=1 by maximizing FOR_rate

                    if v["rate_a_FOR"] < max(group0_base_rate, group1_base_rate) or v["rate_b_FOR"] < max(group0_base_rate, group1_base_rate):
                        # no solution exists
                        threshold_a, randomization_a, utility_a = None, None, None
                        v["threshold_b"], randomization_b, utility_b = None, None, None
                        FOR_rate = None
                    elif v["rate_a_FOR"] <= v["rate_b_FOR"]:
                        # rate_a_FOR is smaller so this is given
                        FOR_rate = v["rate_a_FOR"]
                        randomization_a = 1
                        utility_a = my_utility_function.calculate_utility(
                            y[group_indices[0]], s[group_indices[0]], y_pred_dict[threshold_a][group_indices[0]])

                        utility_b, randomization_b = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[v["threshold_b"]], group_indices[1], target=v["rate_a_FOR"])  # deviate from optimum: FOR rate b = FOR rate a
                    else:
                        # rate_b_FOR is smaller so this is given
                        FOR_rate = v["rate_b_FOR"]
                        utility_a, randomization_a = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[threshold_a], group_indices[0], target=v["rate_b_FOR"])  # deviate from optimum: FOR rate a = FOR rate b

                        randomization_b = 1
                        utility_b = my_utility_function.calculate_utility(
                            y[group_indices[1]], s[group_indices[1]], y_pred_dict[v["threshold_b"]][group_indices[1]])

                elif v["rate_a_PPV"] >= group0_base_rate and v["rate_a_PPV"] >= group1_base_rate:
                    # we maximize the number of individuals with D=1 by minimizing FOR_rate

                    if v["rate_a_FOR"] > min(group0_base_rate, group1_base_rate) or v["rate_b_FOR"] > min(group0_base_rate, group1_base_rate):
                        # no solution exists
                        threshold_a, randomization_a, utility_a = None, None, None
                        v["threshold_b"], randomization_b, utility_b = None, None, None
                        FOR_rate = None
                    elif v["rate_a_FOR"] <= v["rate_b_FOR"]:
                        FOR_rate = v["rate_b_FOR"]
                        utility_a, randomization_a = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[threshold_a], group_indices[0], target=v["rate_b_FOR"])  # deviate from optimum: FOR rate a = FOR rate b

                        randomization_b = 1
                        utility_b = my_utility_function.calculate_utility(
                            y[group_indices[1]], s[group_indices[1]], y_pred_dict[v["threshold_b"]][group_indices[1]])
                    else:
                        FOR_rate = v["rate_a_FOR"]
                        randomization_a = 1
                        utility_a = my_utility_function.calculate_utility(
                            y[group_indices[1]], s[group_indices[0]], y_pred_dict[threshold_a][group_indices[0]])

                        utility_b, randomization_b = my_utility_function.get_utility_with_randomization(
                            s, y_pred_dict[threshold_a], group_indices[1], target=v["rate_a_FOR"])  # deviate from optimum: FOR rate b = FOR rate a

                else:
                    # no solution exists
                    threshold_a, randomization_a, utility_a = None, None, None
                    threshold_b, randomization_b, utility_b = None, None, None
                    FOR_rate = None

            optimal_solution_for_this_PPV["threshold_a"] = threshold_a
            optimal_solution_for_this_PPV["threshold_b"] = v["threshold_b"]
            optimal_solution_for_this_PPV["randomization_a"] = randomization_a
            optimal_solution_for_this_PPV["randomization_b"] = randomization_b
            optimal_solution_for_this_PPV["utility_a"] = utility_a
            optimal_solution_for_this_PPV["utility_b"] = utility_b
            optimal_solution_for_this_PPV["PPV_rate"] = v["rate_a_PPV"]
            optimal_solution_for_this_PPV["FOR_rate"] = FOR_rate
            if utility_a != None and utility_b != None:
                optimal_solution_for_this_PPV["total_utility"] = utility_a + utility_b
            else:
                optimal_solution_for_this_PPV["total_utility"] = np.NINF

            # only add if solution exists
            if threshold_a != None:
                optimal_solution[threshold_a] = optimal_solution_for_this_PPV

        return optimal_solution

    optimal_solution = get_utility_maximizing_ppv_for_combination(
        solution_space_without_None_values, y, group0_base_rate, group1_base_rate, group_indices, my_utility_function, optimal_unconstrained_decision_rule, s, y_pred_dict)

    return optimal_solution, max_base_rate, min_base_rate, solution_space_without_None_values, rates_a_PPV_without_None_values, turning_point_t_index, min_FOR, max_FOR

def run_ppv_parity_and_for_parity(threshold_nr, s, y, group_indices):
    my_utility_function = UtilityFunction(
        1, 0, 0, 1, proba=False)  # corresponds to accuracy
    optimal_unconstrained_decision_rule = my_utility_function.get_optimal_unconstrained_decision_rule()
    thresholds = generate_upper_and_lower_bound_thresholds(
        s, threshold_nr, optimal_unconstrained_decision_rule)
    y_pred_dict = {t: s.between(t[0], t[1]) for t in thresholds}

    """
    # Maximize utility

    utilities_a, utilities_b, highest_utility, ideal_threshold_a, ideal_threshold_b = find_highest_utility(
        y=y, s=s, thresholds=thresholds, group_indices=group_indices, utility_function=my_utility_function.calculate_utility)

    metric_values_utility = calculate_metrics({"ppv": utils.ppv, "for": utils.forate}, y, s, y_pred_dict, ideal_threshold_a, ideal_threshold_b, group_indices)

    print("\nIf we maximize utility without fairness constraints, these are the optimal thresholds:",
          "\n  ideal_threshold_A0:", ideal_threshold_a, "\n  ideal_threshold_A1:", ideal_threshold_b)
    print("Thereby, we achieve the following fairness values:", metric_values_utility)

    no_fairness_PPV_rate_0, no_fairness_FOR_rate_0 = metric_values_utility["ppv"][0], metric_values_utility["for"][0]
    no_fairness_PPV_rate_1, no_fairness_FOR_rate_1 = metric_values_utility["ppv"][1], metric_values_utility["for"][1]
    group_selection_rate_0, group_selection_rate_1 = metric_values_utility["group_selection_rate"][0], metric_values_utility["group_selection_rate"][1]

    optimal_decision_rules_unconstrained = {
        "ideal_thresholds": (ideal_threshold_a, ideal_threshold_b),
        "no_fairness_PPV A0": no_fairness_PPV_rate_0,
        "no_fairness_FOR A0": no_fairness_FOR_rate_0,
        "no_fairness_PPV A1": no_fairness_PPV_rate_1,
        "no_fairness_FOR A1": no_fairness_FOR_rate_1,
        "group_selection_rate_A0": group_selection_rate_0,
        "group_selection_rate_A1": group_selection_rate_1
    }
    """

    # Find ideal thresholds under PPV parity

    fairness_function = utils.ppv

    rates_a, rates_b, highest_fair_utility, ideal_rate, ideal_threshold_a, ideal_threshold_b = find_highest_utility_under_fairness(
        y=y, s=s, thresholds=thresholds, group_indices=group_indices, fairness_function=fairness_function, utility_function=my_utility_function.calculate_utility, y_pred_dict=y_pred_dict)

    FOR_rates_a = [utils.forate(
        y, y_pred_dict[t], group_indices[0]) for t in thresholds]
    FOR_rates_b = [utils.forate(
        y, y_pred_dict[t], group_indices[1]) for t in thresholds]

    group_selection_rates_a = [utils.group_selection_rate(
        y, y_pred_dict[t], group_indices[0]) for t in thresholds]
    group_selection_rates_b = [utils.group_selection_rate(
        y, y_pred_dict[t], group_indices[1]) for t in thresholds]

    print("\nUnder PPV parity, these are the optimal thresholds:", "\n  ideal_threshold_A0:", ideal_threshold_a, "\n  ideal_threshold_A1:", ideal_threshold_b, "\n  PPV rates: ", ideal_rate,
          "\nThis results in the following FOR rates:", "\n  FOR rate group 0:", FOR_rates_a[thresholds.index(ideal_threshold_a)], "\n  FOR rate group 1:", FOR_rates_b[thresholds.index(ideal_threshold_b)])

    PPV_parity_ideal_PPV_rate = ideal_rate
    PPV_parity_ideal_FOR_rate_0 = FOR_rates_a[thresholds.index(
        ideal_threshold_a)]
    PPV_parity_ideal_FOR_rate_1 = FOR_rates_b[thresholds.index(
        ideal_threshold_b)]

    group_selection_rate_0 = group_selection_rates_a[thresholds.index(ideal_threshold_a)]
    group_selection_rate_1 = group_selection_rates_b[thresholds.index(ideal_threshold_b)]

    optimal_decision_rules_ppv = {
        "ideal_thresholds": (ideal_threshold_a, ideal_threshold_b),
        "PPV_parity_ideal_PPV_rate": PPV_parity_ideal_PPV_rate,
        "PPV_parity_ideal_FOR A0": PPV_parity_ideal_FOR_rate_0,
        "PPV_parity_ideal_FOR A1": PPV_parity_ideal_FOR_rate_1,
        "group_selection_rate_A0": group_selection_rate_0,
        "group_selection_rate_A1": group_selection_rate_1
    }

    # Find ideal thresholds for FOR parity

    fairness_function = utils.forate

    rates_a, rates_b, highest_fair_utility, ideal_rate, ideal_threshold_a, ideal_threshold_b = find_highest_utility_under_fairness(
        y=y, s=s, thresholds=thresholds, group_indices=group_indices, fairness_function=fairness_function, utility_function=my_utility_function.calculate_utility, y_pred_dict=y_pred_dict)

    PPV_rates_a = [utils.ppv(y, y_pred_dict[t], group_indices[0])
                   for t in thresholds]
    PPV_rates_b = [utils.ppv(y, y_pred_dict[t], group_indices[1])
                   for t in thresholds]

    group_selection_rates_a = [utils.group_selection_rate(
        y, y_pred_dict[t], group_indices[0]) for t in thresholds]
    group_selection_rates_b = [utils.group_selection_rate(
        y, y_pred_dict[t], group_indices[1]) for t in thresholds]

    print("\nUnder FOR parity, these are the optimal thresholds:", "\n  ideal_threshold_A0:", ideal_threshold_a, "\n  ideal_threshold_A1:", ideal_threshold_b, "\n  FOR rates: ", ideal_rate,
          ".\nThis results in the following PPV rates:", "\n  PPV rate group 0:", PPV_rates_a[thresholds.index(ideal_threshold_a)], "\n  PPV rate group 1:", PPV_rates_b[thresholds.index(ideal_threshold_b)])

    FOR_parity_ideal_FOR_rate = ideal_rate
    FOR_parity_ideal_PPV_rate_0 = PPV_rates_a[thresholds.index(
        ideal_threshold_a)]
    FOR_parity_ideal_PPV_rate_1 = PPV_rates_b[thresholds.index(
        ideal_threshold_b)]

    group_selection_rate_0 = group_selection_rates_a[thresholds.index(ideal_threshold_a)]
    group_selection_rate_1 = group_selection_rates_b[thresholds.index(ideal_threshold_b)]

    optimal_decision_rules_for = {
        "ideal_thresholds": (ideal_threshold_a, ideal_threshold_b),
        "FOR_parity_ideal_FOR_rate": FOR_parity_ideal_FOR_rate,
        "FOR_parity_ideal_PPV A0": FOR_parity_ideal_PPV_rate_0,
        "FOR_parity_ideal_PPV A1": FOR_parity_ideal_PPV_rate_1,
        "group_selection_rate_A0": group_selection_rate_0,
        "group_selection_rate_A1": group_selection_rate_1
    }

    return optimal_decision_rules_ppv, optimal_decision_rules_for

def apply_decision_rule(s, group_indices_a, group_indices_b, thresholds_a, thresholds_b):
    y_pred = pd.Series([False] * len(s), index=s.index)
    if type(thresholds_a) is tuple:
        y_pred[group_indices_a] = s[group_indices_a].between(
            thresholds_a[0], thresholds_a[1])
        y_pred[group_indices_b] = s[group_indices_b].between(
            thresholds_b[0], thresholds_b[1])
    else:
        y_pred[group_indices_a] = s[group_indices_a] >= thresholds_a
        y_pred[group_indices_b] = s[group_indices_b] >= thresholds_b
    y_pred = y_pred.astype(bool)
    return y_pred

def run_sufficiency(threshold_nr, s, y, group_indices):
    my_utility_function = UtilityFunction(
        1, 0, 0, 1, proba=False)  # corresponds to accuracy
    optimal_unconstrained_decision_rule = my_utility_function.get_optimal_unconstrained_decision_rule()
    thresholds = generate_upper_and_lower_bound_thresholds(
        s, threshold_nr, optimal_unconstrained_decision_rule)
    y_pred_dict = {t: s.between(t[0], t[1]) for t in thresholds}

    group0_base_rate, group1_base_rate = mean(s[group_indices[0]]), mean(s[group_indices[1]])

    # Find optimal decision rule under sufficiency

    fairness_functions = (utils.ppv, utils.forate)

    rates_a_PPV, rates_a_FOR, rates_b_PPV, rates_b_FOR = get_PPV_FOR_rates_for_all_decision_rules(
        y=y, s=s, thresholds=thresholds, group_indices=group_indices, fairness_functions=fairness_functions, utility_function=my_utility_function.calculate_utility, y_pred_dict=y_pred_dict)

    optimal_solution, max_base_rate, min_base_rate, solution_space_without_None_values, rates_a_PPV_without_None_values, turning_point_t_index, min_FOR, max_FOR = generate_solution_space_and_maximize_utility(
        y, group0_base_rate, group1_base_rate, rates_a_PPV, rates_a_FOR, rates_b_PPV, rates_b_FOR, thresholds, group_indices, my_utility_function, optimal_unconstrained_decision_rule, s, y_pred_dict)

    optimal_solution_total_utility = [v["total_utility"] for (k, v) in optimal_solution.items()]
    optimal_solution_PPV_rates = [v["PPV_rate"] for (k, v) in optimal_solution.items()]
    optimal_solution_FOR_rates = [v["FOR_rate"] for (k, v) in optimal_solution.items()]
    optimal_solution_threshold_a = [v["threshold_a"] for (k, v) in optimal_solution.items()]
    optimal_solution_threshold_b = [v["threshold_b"] for (k, v) in optimal_solution.items()]
    optimal_solution_randomization_a = [v["randomization_a"] for (k, v) in optimal_solution.items()]
    optimal_solution_randomization_b = [v["randomization_b"] for (k, v) in optimal_solution.items()]

    index_of_utils = np.argmax(optimal_solution_total_utility)

    print("\nUnder sufficiency, these are the optimal thresholds:", "\n  ideal_threshold_a:", optimal_solution_threshold_a[index_of_utils], "randomization_a:", optimal_solution_randomization_a[index_of_utils], "\n  ideal_threshold_b:", optimal_solution_threshold_b[index_of_utils], "randomization_b:",
        optimal_solution_randomization_b[index_of_utils], "\nThis results in the following rates (for both groups!):\n  PPV:", optimal_solution_PPV_rates[index_of_utils], "\n  FOR:", optimal_solution_FOR_rates[index_of_utils], "\nThis yields a total utility of:", optimal_solution_total_utility[index_of_utils])
    print("\nExplanation of randomized decision rules: A threshold of (t1, t2) with randomization q means: Individuals with p<t1 or p>t2 are not selected (i.e., assigned decision D=0). Individuals with t1<p<t2 are selected with probability q.")


from time import sleep
import network
from secret import ssid, password
from machine import Pin, WDT
import uasyncio as asyncio
import socket
from vl53l1x import VL53L1X
from inventor import Inventor2040W
import json

PINS_BREAKOUT_GARDEN = {"sda": 4, "scl": 5}
board = Inventor2040W()
i2c = board.i2c

sensor = VL53L1X(i2c)

onboard = Pin("LED", Pin.OUT, value=0)
wlan = network.WLAN(network.STA_IF)
wlan.active(True)
wlan.connect(ssid, password)

while wlan.isconnected() == False:
    print(".", end="")
    sleep(0.5)
print(wlan.ifconfig())

f = open("index.html","r")
html = f.read()
f.close()

distance = sensor.distance

async def serve_client(reader, writer):
    global sensor, html, distance
    sensor_reading = sensor.distance
    print("Client Connected")
    request_line = await reader.readline()
    print("Request:", request_line)
    while await reader.readline() != b"\r\n":
#         print("awaiting readline")
        pass

    request = str(request_line)
    print(request)
    url = request.find('/distance')
    percent = str(100 - (2600 // sensor_reading ))
    print(percent)
    if url == 6:
        distance = {'distance': sensor_reading //10, 'percent':percent}
        response = json.dumps(distance)
    else:
        response = html % (str(sensor_reading), percent)
    writer.write('HTTP/1.0 200 OK\r\nContent-type: text/html\r\n\r\n')
    writer.write(response)

    await writer.drain()
    await writer.wait_closed()
    print('Client Disconnected')

async def main():
    global sensor

    print ("setting up webserver")
    asyncio.create_task(asyncio.start_server(serve_client, "0.0.0.0", 80))
#     wdt = WDT(timeout=8000)
    count = 0
    while True:
        #distance = sensor.distance
        distance = 0
        onboard.on()
        ip = wlan.ifconfig()
        count += 1
        print(f"heartbeat {count}, distance {distance}, IP: {ip[0]}")
#         wdt.feed()
        await asyncio.sleep(0.25)
        onboard.off()
        await asyncio.sleep(0.25)

try:
    asyncio.run(main())
finally:
    asyncio.new_event_loop()


import random
import json
from generators.Star import Star
from generators.Planet import Planet
from generators.Generator import Generator

class StarSystem(Generator):
    def __init__(self, redis, features={}):
        Generator.__init__(self,redis, features)
        self.generate_stars()
        self.generate_planet()

    def generate_stars(self):
        """ Do stuff"""
        self.stars=[]

        positions=self.redis.lrange('starposition', 0,-1)

        for starId in xrange(self.starcount['count']):
            random.shuffle(positions)
            self.stars.append(Star(self.redis, {'pos': json.loads(positions.pop(0))  }))

    def generate_planet(self):
        """ Do stuff"""
        self.planet=Planet(self.redis)


from rdkit import Chem
from rdkit.Chem import AllChem

def reaction_smiles_split(reaction_smiles):
    """
    Parameters
    ----------
    reaction_smiles: str

    Returns
    -------
    reactants, products: tuple(list[str])
    """

    split_rxn = reaction_smiles.split(">>")

    reactants = split_rxn[0].split(".")
    products = split_rxn[1].split(".")

    reactants.sort()
    products.sort()

    return reactants, products

def smiles_to_rdkit_reaction(reaction_smiles):
    """
    For converting reaction SMILES strings into RDKit AllChem.ChemicalReaction
    objects.

    Parameters
    ----------
    reaction_smiles: str
        Reaction SMILES.

    Returns
    -------

    rdkit_reaction: RDKit AllChem.Reaction
        Converted reaction.
    """

    reactants, products = reaction_smiles_split(reaction_smiles)

    reactants_as_mol = [Chem.MolFromSmiles(r) for r in reactants]
    products_as_mol = [Chem.MolFromSmiles(p) for p in products]

    rdkit_reaction = AllChem.ChemicalReaction()

    [rdkit_reaction.AddReactantTemplate(r) for r in reactants_as_mol]
    [rdkit_reaction.AddProductTemplate(p) for p in products_as_mol]

    return rdkit_reaction


from datetime import datetime
from random import choice

from BucketLib.bucket import Bucket
from Cb_constants import DocLoading, CbServer
from SystemEventLogLib.Events import Event
from SystemEventLogLib.SystemEventOperations import SystemEventRestHelper
from SystemEventLogLib.data_service_events import DataServiceEvents
from membase.api.rest_client import RestConnection
from upgrade.upgrade_base import UpgradeBase

class SystemEventLogs(UpgradeBase):
    def setUp(self):
        super(SystemEventLogs, self).setUp()

    def tearDown(self):
        super(SystemEventLogs, self).tearDown()

    def test_upgrade(self):
        update_task = None
        internal_setting_error = "eventLogsLimit - not supported " \
                                 "in mixed version clusters"
        get_events_error = "This http API endpoint isn't " \
                           "supported in mixed version clusters"
        t_durability_level = choice([Bucket.DurabilityLevel.NONE,
                                     Bucket.DurabilityLevel.MAJORITY])

        if self.upgrade_with_data_load:
            self.log.info("Continuous doc updates with durability=%s"
                          % t_durability_level)
            update_task = self.task.async_continuous_doc_ops(
                self.cluster, self.bucket, self.gen_load,
                op_type=DocLoading.Bucket.DocOps.UPDATE, timeout_secs=30,
                process_concurrency=1, durability=t_durability_level)

        self.log.info("Upgrading cluster nodes to target version")
        node_to_upgrade = self.fetch_node_to_upgrade()
        while node_to_upgrade is not None:
            self.log.info("Selected node for upgrade: %s"
                          % node_to_upgrade.ip)
            self.upgrade_function[self.upgrade_type](node_to_upgrade,
                                                     self.upgrade_version)
            self.cluster_util.print_cluster_stats(self.cluster)

            node_to_upgrade = self.fetch_node_to_upgrade()

            # Create custom event to test system event logs
            event = DataServiceEvents.bucket_online(
                self.cluster.master.ip, "dummy_bucket", "dummy_uuid")
            event.pop(Event.Fields.EXTRA_ATTRS)
            event[Event.Fields.UUID] = self.system_events.get_rand_uuid()
            event[Event.Fields.TIMESTAMP] = \
                self.system_events.get_timestamp_format(datetime.now())
            nodes_in_cluster = \
                RestConnection(self.cluster.master).get_nodes()
            if node_to_upgrade is None \
                    or self.cluster_supports_system_event_logs:
                # Cluster fully upgraded / already supports system event logs
                self.log.info("Creating events for validation")
                self.system_events.set_test_start_time()
                self.system_events.events = list()
                col_name = "collection_1"
                self.bucket_util.create_collection(
                    self.cluster.master, self.bucket,
                    collection_spec={"name": col_name})
                self.bucket_util.drop_collection(
                    self.cluster.master, self.bucket, collection_name=col_name)

                self.system_events.add_event(
                    DataServiceEvents.collection_created(
                        self.cluster.master.ip, self.bucket.name,
                        CbServer.default_scope, col_name))
                self.system_events.add_event(
                    DataServiceEvents.collection_dropped(
                        self.cluster.master.ip, self.bucket.name,
                        CbServer.default_scope, col_name))
                event_helper = SystemEventRestHelper([self.cluster.master])
                status, content = event_helper.create_event(event)
                if status is False:
                    self.log_failure("Failed to create event: %s" % content)
                self.system_events.add_event(event)

                for node in nodes_in_cluster:
                    self.log.info("Validating events on %s" % node.ip)
                    self.system_events.validate(node)
            else:
                # Mixed mode cluster testing
                self.log.info("Creating custom event to validate "
                              "and trying to fetch events from cluster")
                for node in nodes_in_cluster:
                    if float(node.version[:3]) >= 7.1:
                        event_helper = SystemEventRestHelper([node])
                        status, content = event_helper.create_event(event)
                        if not status:
                            self.log_failure("Event creation returned '%s':%s"
                                             % (status, content))
                        events = event_helper.get_events(events_count=-1)
                        self.log.critical("Events: %s" % events)
                        if events != get_events_error:
                            self.log_failure("Unexpected response: %s"
                                             % events)

                        status, content = event_helper.update_max_events(
                            max_event_count=CbServer.sys_event_max_logs)
                        if status:
                            self.log_failure("Mixed mode - able to update "
                                             "max_events: %s" % content)
                        elif content["errors"][0] != internal_setting_error:
                            self.log_failure("Error mismatch in mixed-mode: %s"
                                             % content)

            # Halt further upgrade if test has failed during current upgrade
            if self.test_failure is not None:
                break

        if self.upgrade_with_data_load:
            # Wait for update_task to complete
            update_task.end_task()
            self.task_manager.get_task_result(update_task)

        # Perform final collection/doc_count validation
        self.validate_test_failure()


import time
import random
import functools

from concurrent.futures import ThreadPoolExecutor

def some_task(n):
    time.sleep(3)
    print("Task {} finished".format(n))
    return n

def f_callback(future):
    print("Future: {} - Status: {}, Error: {}".format(future, future.done(), future.exception()))

def process():
    result_data = {}
    with ThreadPoolExecutor(max_workers=1) as executor:
        try:
            future1 = executor.submit(some_task, (1))
            future2 = executor.submit(some_task, (2))
            future1.add_done_callback(functools.partial(print, "Future:"))
            # future1.add_done_callback(f_callback)
            # print("[notify] ", future1.set_running_or_notify_cancel())
            # print("[notify]", future2.set_running_or_notify_cancel())

            # print(future1.running())
            print(future2.cancel())

            # print(future1.exception(timeout=None))
            print("[1]", future1.result())
            print("[2]", future2.result())

            result_data['t1'] = future1.result()
            result_data['t2'] = future2.result()
        except Exception as e:
            print("Exception - {}".format(e))

    return result_data

def main():
    data = process()
    print(data)

if __name__ == "__main__":
    main()


import re
from typing import Union, List, Tuple

class StringCalculator:
    def add(self, input_str: str) -> int:
        if input_str == '':
            return 0
        str_vals = self._split_str(input_str)
        int_vals = self._convert_vals_to_ints(str_vals)
        self._validate(int_vals)
        return self._sum_vals(int_vals)

    @staticmethod
    def _string_to_number(value: str) -> int:
        return int(value)

    @staticmethod
    def _sum_vals(vals: List[int]) -> int:
        return sum(filter(lambda x: x <= 1000, vals))

    @staticmethod
    def _split_str(s: str) -> List[str]:
        separator, s_rest = StringCalculator.find_separator(s)
        return re.split(separator, s_rest)

    @staticmethod
    def _convert_vals_to_ints(vals: List[str]) -> List[int]:
        return [StringCalculator._string_to_number(v) for v in vals]

    @staticmethod
    def _validate(vals: List[int]) -> None:
        negs = list(filter(lambda x: x < 0, vals))
        if len(negs) > 0:
            negs_str = map(str, negs)
            raise ValueError('negatives not allowed ' + ','.join(negs_str))

    @staticmethod
    def find_separator(s: str) -> Tuple[str, str]:
        match = re.search(r'^//(.*)\n(.*)', s)
        default_separator = r'[\n,]'
        if match:
            return match.group(1), match.group(2)
        return default_separator, s


from datetime import datetime

from authlib.jose import jwt
from pytest import fixture

from app import app
from api.errors import INVALID_ARGUMENT

@fixture(scope='session')
def secret_key():
    # Generate some string based on the current datetime.
    return datetime.utcnow().isoformat()

@fixture(scope='session')
def client(secret_key):
    app.secret_key = secret_key

    app.testing = True

    with app.test_client() as client:
        yield client

@fixture(scope='session')
def valid_jwt(client):
    header = {'alg': 'HS256'}

    payload = {'key': 'some_key'}

    secret_key = client.application.secret_key

    return jwt.encode(header, payload, secret_key).decode('ascii')

@fixture(scope='module')
def invalid_json_expected_payload():
    def _make_message(message):
        return {
            'errors': [{
                'code': INVALID_ARGUMENT,
                'message': message,
                'type': 'fatal'
            }]
        }

    return _make_message


import turtle
from paddle import Paddle
from ball import Ball
from scoreboard import Scoreboard
import time

screen = turtle.Screen()
r_paddle = Paddle((350, 0))
l_paddle = Paddle((-350, 0))
ball = Ball()
scoreboard = Scoreboard()

screen.tracer(0)
screen.setup(width=800, height=600)
screen.bgcolor("black")
screen.title("Pong")

screen.listen()
screen.onkey(r_paddle.up, "Up")
screen.onkey(r_paddle.down, "Down")
screen.onkey(l_paddle.up, "w")
screen.onkey(l_paddle.down, "s")

game_is_on = True

while game_is_on:
    screen.update()
    ball.move()

    # Detect collision with top and bottom walls
    if ball.ycor() > 280 or ball.ycor() < -280:
        ball.bounce_y()

    # Detect goals
    if ball.xcor() > 380:
        ball.reset_position()
        scoreboard.score_lpoint()
    if ball.xcor() < -400:
        ball.reset_position()
        scoreboard.score_rpoint()

    # Detect collision with paddles
    if (ball.distance(r_paddle) < 50 and ball.xcor() > 330) or (ball.distance(l_paddle) < 50 and ball.xcor() < -330):
        ball.bounce_x()

screen.exitonclick()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Created on Wen Oct 26 05:22:00 2022

The class of kalmen filter based on OpenCV

@author: Xinyang Chen
"""

import cv2
import numpy as np

class KalmanFilter():
    def __init__(self, is3D=True, processNoise=0.003, measurementNoise=0.5):
        self.is3D = is3D# 3D filter or 2D filter

        # Initialize the filter
        if is3D:
            self.kalman = cv2.KalmanFilter(6, 3, cv2.CV_32F)
            self.kalman.measurementMatrix = np.array([[1,0,0,0,0,0],[0,1,0,0,0,0],[0,0,1,0,0,0]], np.float32)
            self.kalman.transitionMatrix = np.array([[1,0,0,1,0,0],[0,1,0,0,1,0],[0,0,1,0,0,1],[0,0,0,1,0,0],[0,0,0,0,1,0],[0,0,0,0,0,1]], np.float32)
            self.kalman.processNoiseCov = np.array([[1,0,0,0,0,0],[0,1,0,0,0,0],[0,0,1,0,0,0],[0,0,0,1,0,0],[0,0,0,0,1,0],[0,0,0,0,0,1]], np.float32) * processNoise
            self.kalman.measurementNoiseCov = np.array([[1,0,0],[0,1,0],[0,0,1]], np.float32) * measurementNoise
        else:
            self.kalman = cv2.KalmanFilter(4, 2, cv2.CV_32F)
            self.kalman.measurementMatrix = np.array([[1,0,0,0],[0,1,0,0]], np.float32)
            self.kalman.transitionMatrix = np.array([[1,0,1,0],[0,1,0,1],[0,0,1,0],[0,0,0,1]], np.float32)
            self.kalman.processNoiseCov = np.array([[1,0,0,0],[0,1,0,0],[0,0,1,0],[0,0,0,1]], np.float32) * processNoise
            self.kalman.measurementNoiseCov = np.array([[1,0],[0,1]], np.float32) * measurementNoise

    def process(self, measurement):
        """
        @brief: Process the mearsured data with Kalman filter
        param measurement: mearsured data
        """
        self.kalman.correct(measurement)
        predPos = self.kalman.predict()

        if self.is3D:
            return predPos[:3, 0]
        else:
            return predPos[:2, 0]


import sys
import re
import string
import json
import nltk
from nltk.stem import PorterStemmer

# global declarations for doclist, postings, vocabulary
docids = []
postings = {}
vocab = []
doclengths = []
errorPages = []
count = 0

# main is used for offline testing only
def main():
    # code for testing offline
    if len(sys.argv) != 2:
        print('usage: ./indexer.py file')
        sys.exit(1)
    filename = sys.argv[1]

    try:
        input_file = open(filename, 'r')
    except (IOError) as ex:
        print('Cannot open ', filename, '\n Error: ', ex)

    else:
        page_contents = input_file.read()  # read the input file
        url = 'http://www.' + filename + '/'
        #print(url, page_contents)
        make_index(url, page_contents)

    finally:
        input_file.close()

def write_index():
#
    # declare refs to global variables
    global docids
    global postings
    global vocab
    global doclengths
    # open files
    out_d = open('docids.txt', 'w')
    out_v = open('vocab.txt', 'w')
    out_l = open('doclengths.txt', 'w')
    out_p = open('postings.txt', 'w')

    # write to index files: docids, vocab, postings
    # use JSON as it preserves the dictionary structure (read/write treat it as a string)
    json.dump(docids, out_d)
    json.dump(vocab, out_v)
    json.dump(doclengths, out_l)
    json.dump(postings, out_p)
    # close files
    out_d.close()
    out_v.close()
    out_l.close()
    out_p.close()

    d = len(docids)
    v = len(vocab)
    p = len(postings)
    dl = len(doclengths)
    #print(doclengths)
    print ('===============================================')
    print ('Indexing: ', d, ' docs ', v, ' terms ', p,' postings written to file',dl,'doclengths')
    print ('===============================================')

    return

def read_index():

    # reads existing data into index files: docids, vocab, postings
    in_d = open('docids.txt', 'r')
    in_l = open('doclengths.txt', 'r')
    in_v = open('vocab.txt', 'r')
    in_p = open('postings.txt', 'r')

    docids = json.load(in_d)
    vocab = json.load(in_v)
    doclengths = json.load(in_l)
    postings = json.load(in_p)

    in_d.close()
    in_v.close()
    in_p.close()
    in_l.close()

	#returns array of docids, doclengths, vocab and postings to be read in retriever
    return [docids, doclengths, vocab, postings]

def clean_html(page_contents):
    # function to clean html

    # First we remove inline JavaScript/CSS:
    cleaned = re.sub(r"(?is)<(script|style).*?>.*?(</\1>)", "", page_contents.strip())
    # Then we remove html comments. This has to be done before removing regular
    # tags since comments can contain '>' characters.
    cleaned = re.sub(r"(?s)<!--(.*?)-->[\n]?", "", cleaned)
    # Next we can remove the remaining tags:
    cleaned = re.sub(r"(?s)<.*?>", " ", cleaned)
    # Deal with whitespace
    cleaned = re.sub(r" ", " ", cleaned)
    cleaned = re.sub(r"\t", " ", cleaned) # added DJS
    cleaned = re.sub(r"[ ]+", " ", cleaned) # modified DJS
    # and blank lines - added DJS
    cleaned = re.sub(r"[ ]*\n", "\n", cleaned)
    cleaned = re.sub(r"\n+", "\n", cleaned)

    return cleaned.strip()

def make_index(url, page_contents):
    # declare refs to global variables
    ps = PorterStemmer()
    global docids
    global postings
    global vocab
    global doclengths
    global count

    # first convert bytes to string if necessary
    if isinstance(page_contents, bytes):
        try:
            page_contents = page_contents.decode('utf-8')
        except:
			# if a page contains an error and can not continue pages are sent to a list of error pages
            errorPages.append(url)
            return

    page_text = clean_html(page_contents)
    count += 1

    print('===============================================')
    print('make_index: url = ', url)
    print('page: ', count)
    print('===============================================')

	# series of statements to remove duplicate pages
    if (re.search('https:..', url)):	# match and remove https://
        domain_url = re.sub('https://', '', url)
    elif (re.search('http:..', url)):	# match and remove http://
        domain_url = re.sub('http://', '', url)
    else:
        print ("make_index: no match for protocol url=", url)

    if (re.search('www.', domain_url)):	# match and remove www.
        domain_url = re.sub('www.', '', domain_url)

    page_text = clean_html(page_contents)

    if (domain_url in docids): # return if we've seen this before
        return
    else:
        docids.append(domain_url)				# add url to docids table
        docid = str(docids.index(domain_url))	# get a string version of the docid

    #tockenises words in page_text then stem the words in a list
    stemed = []

	#stems words in page_text and saves to a list
    for w in page_text:
        stemed.append(ps.stem(w))

	# list of named entities
    namedEntList = []

    #tokenizes page text
    words = nltk.word_tokenize(page_text)
    tagged = nltk.pos_tag(words)

	# named entities are turned from tree structure to list
    namedEnt = nltk.ne_chunk(tagged,binary=True)
    namedEntList += extract_entity_names(namedEnt)

	# replaces entities with blanks spaces
    for ent in namedEntList:
        page_text = page_text.replace(ent,"")

    token = nltk.word_tokenize(page_text)

    docwordcount = 0

	#appends
    token += namedEntList
	# constructs vocab
    for word in token:
        docwordcount += 1
        if word in vocab:
            index = vocab.index(word)
        else:
            vocab.append(word)
            index = vocab.index(word)
            postings[index] = []

        freq = token.count(word)

		# checks whether the word does not exist in postings to not allow duplicates in postings dictionary
        if index not in postings:
            postings[index] = [docid,freq]
        else:
            if [docid,freq] not in postings[index]:
                postings[index].update([docid,freq])

    doclengths.append(docwordcount)

    return

# Code used from https://gist.github.com/onyxfish/322906 for tree to list
def extract_entity_names(t):
    entity_names = []

    if hasattr(t, 'label') and t.label:
        if t.label() == 'NE':
            entity_names.append(' '.join([child[0] for child in t]))
        else:
            for child in t:
                entity_names.extend(extract_entity_names(child))

    return entity_names

# Standard boilerplate to call the main() function to begin
# the program.
if __name__ == '__main__':
    main()


#!/usr/bin/env python3

import setuptools

with open("README.md", "r") as f:
	long_description = f.read()

setuptools.setup(
	name='pi_si7021',
	version='0.5.4',
	author='Toni Kangas',
	description='Library to read si7021 sensor.',
	long_description=long_description,
	long_description_content_type="text/markdown",
	url="https://github.com/kangasta/pi_si7021",
	packages=setuptools.find_packages(),
	classifiers=(
		"Programming Language :: Python :: 3",
		"License :: OSI Approved :: MIT License",
		"Operating System :: OS Independent",
	)
)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.home, name='blog-home'),
    path('about/', views.about, name='blog-about'),
    path('posts/', views.posts, name='blog-posts'),
    path('posts/newer', views.newer, name='blog-newer'),
    path('posts/older', views.older, name='blog-older'),
    path('posts/<int:id>', views.post_detail, name='blog-detail'),

]