from django.http import JsonResponse
from django.db.models import Count, Q
from django.core.serializers import serialize
from rest_framework.decorators import api_view
from django.apps import apps
from django.forms.models import model_to_dict
from datetime import datetime
import json, uuid

App_User = apps.get_model('user_app', 'App_User')
Child = apps.get_model('user_app', 'Child')
Name = apps.get_model('user_app', 'Name')
Voted_Name = apps.get_model('user_app', 'Voted_Name')
Blacklist = apps.get_model('user_app', 'Blacklist')

# Handle viewing and adding children
@api_view(["GET", "PUT", "DELETE"])
def handle_child(request, uuid):

    if request.user.is_authenticated:
        # Find child from UUID

        if request.method == 'GET':

            # get specific child object based on UUID
            try:
                child = Child.objects.get(parent_url=uuid)
            except:
                child = Child.objects.get(guest_url=uuid)

            # Pull and clean up user data for parents
            parent1 = model_to_dict(child.parent_1, fields=['username', 'email', 'first_name', 'last_name'])
            if child.parent_2:
                parent2_obj = App_User.objects.get(email=child.parent_2)
                parent2 = model_to_dict(parent2_obj, fields=['username', 'email', 'first_name', 'last_name'])

            # serialize data and insert cleaned up parent data to final object
            child = model_to_dict(child)
            child['parent_1'] = parent1
            if child['parent_2']:
                child['parent_2'] = parent2

            # return child object, now with parent information
            return JsonResponse({'message': 'Found UUID', 'success': True, 'child': child})

        if request.method == 'PUT':
            # get specific child object based on UUID
            try:
                child = Child.objects.get(parent_url=uuid)
            except:
                child = Child.objects.get(guest_url=uuid)

            if request.data['parent2']:
                try:
                    updated_parent2 = App_User.objects.get(email=request.data['parent2'])
                except:
                    return JsonResponse({'error':'Update failed.  No user account associated with that e-mail exists'})

                if updated_parent2:
                    default_child = Name.objects.get(name='DEFAULT', gender=child.gender)
                    child.parent_2 = request.data['parent2']
                    Voted_Name.objects.create(name=default_child, liked=True, participant = updated_parent2, child=child)

            if request.data['nickname']:
                child.nickname = request.data['nickname']

            if request.data['lastname']:
                child.last_name = request.data['lastname']

            if request.data['due_date']:
                date_str = request.data['due_date']
                date_obj = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
                child.due_date = date_obj.strftime('%Y-%m-%d')

            print(child)
            child.save()

        if request.method == "DELETE":

            child = Child.objects.get(guest_url=uuid)

            print('delete this child: ', child)

            child.delete()

            return JsonResponse({'message':'Child Deleted Successfully'})

    return JsonResponse({'message': 'User is not logged in', 'success': False})

# Handle viewing and adding children
@api_view(["POST", "GET", "PUT"])
def handle_children(request):
    if request.user.is_authenticated:
        user_info = json.loads(serialize("json", [request.user], fields=["email"]))[0]
        curr_user_id=user_info['pk']
        data = request.data
        nickname = data['nickname']

        if request.method == "GET":
            # return all children objects for this parent
            kids = []
            curr_user = App_User.objects.get(id=curr_user_id)
            children_p1 = Child.objects.filter(parent_1=curr_user)
            kids.extend(json.loads(serialize("json", children_p1)))
            children_p2 = Child.objects.filter(parent_2=curr_user.email)
            kids.extend(json.loads(serialize("json", children_p2)))
            if len(kids) > 0:
                return JsonResponse({'kids': kids, 'success': True})
            return JsonResponse({'message': 'No children found', 'success': False})
        elif request.method == "PUT":
            # updates blacklist
            # checks for user entry for proposed blacklist user
            blacklist_user = App_User.objects.filter(email=data['blacklist_email'])
            if len(blacklist_user) == 0:
                return JsonResponse({'message': 'blacklist user not found', 'success': False})
            curr_user = App_User.objects.get(id=curr_user_id)
            curr_child_p1 = Child.objects.filter(nickname=nickname, parent_1=curr_user)
            # check if user is parent1
            if len(curr_child_p1) > 0:
                entry = Blacklist.objects.filter(user=blacklist_user[0], child=curr_child_p1[0])
                # check if black list entry already exists
                if len(entry) == 0:
                    Blacklist.objects.create(
                        user=blacklist_user[0],
                        child=curr_child_p1[0]
                    )
                    return JsonResponse({'message': 'black list entry added', 'success': True})
                return JsonResponse({'message': 'Black list entry already found', 'success': False})
            else:
                curr_child_p2 = Child.objects.filter(nickname=nickname,parent_2=user_info['fields']['email'])
                # check if user is parent2
                if len(curr_child_p2) > 0:
                    entry = Blacklist.objects.filter(user=blacklist_user[0], child=curr_child_p1[0])
                    # check if black list entry already exists
                    if len(entry) > 0:
                        Blacklist.objects.create(
                            user=blacklist_user[0],
                            child=curr_child_p2[0]
                        )
                        return JsonResponse({'message': 'black list entry added', 'success': True})
                    return JsonResponse({'message': 'Black list entry already found', 'success': False})
            return JsonResponse({'message': 'You are not a parent', 'success': False})
        elif request.method == "POST":
            # adds new child object
            parent_2 = None
            due_date = None
            p_url = uuid.uuid4()
            g_url = uuid.uuid4()
            gender = request.data['gender']
            lastname = request.data['lastname']
            if request.data['due_date']:
                date_str = request.data['due_date']
                date_obj = datetime.fromisoformat(date_str.replace('Z', '+00:00'))
                due_date = date_obj.strftime('%Y-%m-%d')
            if data['parent_2']:
                parent_2 = data['parent_2']

            # see if child already exists for parent_1
            p = App_User.objects.get(id=curr_user_id)
            children = Child.objects.filter(parent_1=p).filter(nickname=nickname)
            if len(children) == 0:
                # need to add try/except block here
                Child.objects.create(
                    nickname=nickname,
                    parent_1=p,
                    parent_2=parent_2,
                    last_name=lastname,
                    parent_url=p_url,
                    guest_url=g_url,
                    gender=gender,
                    due_date=due_date,
                )
                # get this new childs object to pass back to the front to be able to handle adding first voted_name
                new_kid_query = Child.objects.filter(parent_url=p_url)
                new_kid = json.loads(serialize("json", new_kid_query))[0]['fields']
                new_kid_id = json.loads(serialize("json", new_kid_query))[0]['pk']
                return JsonResponse({'message': 'child added to the DB', 'success': True, 'id': new_kid_id, 'child': new_kid})
            else:
                return JsonResponse({'message': 'child already exists', 'success': False})
        return JsonResponse({'message': 'You are logged in but something went wrong', 'success': False})
    return JsonResponse({'message': 'You are not logged in', 'success': False})

#Handle swiping name and create Voted_name objects
@api_view(["POST"])

def vote_name(request):
    if request.method=="POST":
        try:
            name=request.data['name']
            child_uuid=request.data['uuid']
            liked=request.data['liked']
            childIns=Child.objects.get(parent_url=child_uuid)
            nameIns=Name.objects.get(id=name['id'])
            participant=request.user
            if  Voted_Name.objects.filter(name=nameIns,participant=participant,child=childIns).exists():
                return JsonResponse({'voted': False, 'message': 'Vote already exists.'})
            else:
                Voted_Name.objects.create(name=nameIns,liked=liked,participant=participant,child=childIns)
                return JsonResponse({'voted':True})
        except Exception as e:
            print(e)
            return JsonResponse({'voted':False})

# Handle viewing, voting, and adding singular names
@api_view(["POST", "GET","PUT"])
def handle_name(request):
# get name list
    if request.method == "PUT":
        try:
            uuid=request.data['uuid']
            child=Child.objects.get(parent_url=uuid)
            gender=child.gender
            names=Name.objects.filter(gender=gender)
            user=request.user
            name_list=[model_to_dict (name) for name in names]
            voted_list=[model_to_dict (voted_name)for voted_name in (Voted_Name.objects.filter(participant_id=user.id))]
            voted_list_id=[i['name'] for i in voted_list]
            unshown_list=[name for name in name_list if name['id'] not in voted_list_id][:20]
            return JsonResponse({'unshown_list':unshown_list})
        except Exception as e:
            print(e)
            return JsonResponse({'unshown_list':[]})
# add new name object
    if request.method=="POST":
        #take name,gender,pk as request data
        name=request.data['name']
        gender=request.data['gender']
        child=request.data['child']
        child=Child.objects.get(parent_url=child['parent_url'])
        participant=request.user
        if request.data['parent_2']:
            try:
                participant=App_User.objects.get(email=request.data['parent_2'])
            except Exception as e:
                participant=None
                print(e)

        if participant and Name.objects.filter(name=name,gender=gender).exists():
            try:
# get Name instance and create Voted_Name
                nameIns=Name.objects.filter(name=name,gender=gender).first()
                Voted_Name.objects.create(name=nameIns,liked=True,participant=participant,child=child)
                return JsonResponse({'voted':True})
            except Exception as e:
                print(e)
                return JsonResponse({'voted':False})
        elif participant:
            try:
                new_name=Name.objects.create(name=name,popularity=None,gender=gender)
                Voted_Name.objects.create(name=new_name,liked=True,participant=participant,child=child)
                return JsonResponse({'added and voted':True})
            except Exception as e:
                print(e)
                return JsonResponse({'added and voted':False})

# handle retrieving and displaying lists of voted on names sorted by vote type
@api_view(["GET"])
def handle_voted_names(request, uuid):

    #define parameters
    user = request.user
    child = Child.objects.get(guest_url=uuid)
    parent1 = child.parent_1
    parent2 = None
    if child.parent_2:
        parent2 = App_User.objects.get(email=child.parent_2)

    # Declare variables liked / disliked / other parent liked names
    all_names = []
    all_names_set = set(all_names)
    duplicate_names = []
    liked_names = []
    disliked_names = []
    other_parent_liked_names = []
    alt_parent_liked_names = []
    agreed_names = []

    #pull all voted names for user
    names_query = Voted_Name.objects.filter(participant = user, child_id = child.id)

    # create list of all names and duplicate names
    for name in names_query:
        if name.name.name == 'DEFAULT':
            pass
        elif name.name.name not in all_names_set:
            all_names.append(name)
            all_names_set.add(name.name.name)
        else:
            duplicate_names.append(name)

    # delete duplicate names from DB
    for name in duplicate_names:
        name.delete()

    # create like / disliked name lists
    for name in all_names:
        if name.liked == True:
            liked_name = model_to_dict(name.name)
            liked_names.append(liked_name)
        else:
            disliked_name = model_to_dict(name.name)
            disliked_names.append(disliked_name)

    # Define liked names for other parent as long as parent 2 exists
    # If current user is parent 1, pull liked names for parent 2
    if parent2 and user == parent1:
        alt_parent_liked_names = Voted_Name.objects.filter(participant = parent2, child = child.id, liked = True)

    # If current user is parent 2, pull liked names for parent 1
    elif parent2 and user.email == child.parent_2:
        alt_parent_liked_names = Voted_Name.objects.filter(participant = parent1, child = child.id, liked = True)

    # serialize other parent liked names query set, and add name string for rendering on front end if parent 2 exists
    if alt_parent_liked_names:
        for name in alt_parent_liked_names:
            alt_like = model_to_dict(name.name)
            other_parent_liked_names.append(alt_like)

    # compare liked names lists and compile agreed names if parent2 exists and format response
    if parent2:
        for name in liked_names:
            if name in other_parent_liked_names:
                agreed_names.append(name)

        response = {
            'liked': liked_names,
            'disliked': disliked_names,
            'agreed': agreed_names,
        }

    # format response with liked / disliked and null value for agreed since parent2 doesn't exist yet
    else:
        response = {
            'liked': liked_names,
            'disliked': disliked_names,
            'agreed': None,
        }

    return JsonResponse({'names': response})

# Handle viewing and ranking of pairs of names
@api_view(["GET", "PUT"])
def handle_results(request, uuid):

    if request.method == "GET":

        child = Child.objects.get(parent_url=uuid)
        parent1 = App_User.objects.get(email=child.parent_1.email)
        parent2 = App_User.objects.get(email=child.parent_2)

        #
        p1 = Voted_Name.objects.filter(participant=parent1,child=child, liked=True)
        p2 = Voted_Name.objects.filter(participant=parent2,child=child, liked=True)

        agreed = []

        for vote in p1:
            if vote.name.name == 'DEFAULT':
                continue
            for vote2 in p2:
                if vote.name == vote2.name:
                    weight = vote.weight + vote2.weight
                    agreed.append([weight, vote.name.name])

        agreed.sort(reverse=True)

        return JsonResponse({'weighted':agreed})

    elif request.method == "PUT":
        # update names objects with ranking
        # this will require future model update most likely
        pass


from datetime import timedelta
from operator import eq

from pyphonetics.distance_metrics import levenshtein_distance
from couchdbkit.ext.django.schema import (
    DecimalProperty,
    ListProperty,
    StringProperty,
)
from dateutil.parser import parse as parse_date

from dimagi.ext.couchdbkit import DocumentSchema

def le_days_diff(max_days, date1, date2):
    """
    Returns True if date1 is less than or equal to max_days away from
    date2, otherwise returns False

    >>> from datetime import date
    >>> le_days_diff(364, '2018-01-01', '2017-01-20')
    True
    >>> le_days_diff(364, date(2017, 1, 20), date(2018, 1, 1))
    True
    >>> le_days_diff(364, '2018-01-01', '2020-01-20')
    False

    """
    if isinstance(date1, str):
        date1 = parse_date(date1)
    if isinstance(date2, str):
        date2 = parse_date(date2)
    return abs(date1 - date2) <= timedelta(max_days)

def le_levenshtein_percent(percent, string1, string2):
    """
    Returns True if Levenshtein distance between string1 and string2,
    divided by max length, is less than or equal to the given percent.

    :param percent: Percent expressed as a decimal between 0 and 1
    :param string1: First string to compare
    :param string2: Second string to compare
    :return: True or False

    >>> le_levenshtein_percent(0.2, 'Riyaz', 'Riaz')
    True
    >>> le_levenshtein_percent(0.2, 'Riyaz', 'Riazz')
    False

    """
    if not 0 <= percent < 1:
        raise ValueError('percent must be greater that or equal to 0 and less than 1')
    dist = levenshtein_distance(string1, string2)
    max_len = max(len(string1), len(string2))
    return dist / max_len <= percent

MATCH_TYPE_EXACT = 'exact'
MATCH_TYPE_LEVENSHTEIN = 'levenshtein'  # Useful for words translated across alphabets
MATCH_TYPE_DAYS_DIFF = 'days_diff'  # Useful for estimated dates of birth
MATCH_FUNCTIONS = {
    MATCH_TYPE_EXACT: eq,
    MATCH_TYPE_LEVENSHTEIN: le_levenshtein_percent,
    MATCH_TYPE_DAYS_DIFF: le_days_diff,
}
MATCH_TYPES = tuple(MATCH_FUNCTIONS)
MATCH_TYPE_DEFAULT = MATCH_TYPE_EXACT

class PropertyWeight(DocumentSchema):
    case_property = StringProperty()
    weight = DecimalProperty()
    match_type = StringProperty(required=False, choices=MATCH_TYPES, default=MATCH_TYPE_DEFAULT)
    match_params = ListProperty(required=False)


import pygame
from bomb import Bomb
pygame.init()
pygame.mixer.pre_init(44100, 16, 2)
pygame.font.init()
size = (824, 674)
screen = pygame.display.set_mode(size)

pygame.display.set_caption("BOH(Brunova Osobna Hra)")
running = True
clock = pygame.time.Clock()

bomb = Bomb(pygame, screen)

while True:
    bomb.drawstuff()
    bomb.ticktimer += clock.tick(60)
    bomb.beep()
    if bomb.getevents() == False:
        break

pygame.quit()


"""
Miscellaneous utilities to read and work with the json files and such.
Stuff multiple functions use.
"""
import sys
import os
import errno
import time
import codecs, json
import logging
import tables

import numpy as np
import torch
from transformers import AutoModel

# Use mpl on remote.
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from . import predict_utils as pu
from .retrieval_models import content_profile_models

def load_aspire_config(expanded_model_name):
    """
    Given the name of a model to load, load it and return its
    json config.
    :param expanded_model_name: string of the form <dataset>/<model_name>/<run_name> which is part
        of the path to a trained model.
    """
    trained_model_path = os.path.join(os.environ['CUR_PROJ_DIR'], 'model_runs', expanded_model_name)
    with codecs.open(os.path.join(trained_model_path, 'run_info.json'), 'r', 'utf-8') as fp:
        run_info = json.load(fp)
        all_hparams = run_info['all_hparams']
    return all_hparams

def load_aspire_model(expanded_model_name):
    """
    Given the name of a model to load, load it and return its
    base bert encoder.
    :param expanded_model_name: string of the form <dataset>/<model_name>/<run_name> which is part
        of the path to a trained model.
    """
    if 'gypsum' in os.environ['CUR_PROJ_DIR']:  # This is running on unity (in interactive mode)
        trained_model_path = os.path.join('/gypsum/work1/mccallum/smysore/2021-ai2-scisim', 'model_runs', expanded_model_name)
    else:  # This is running on gypsum.
        trained_model_path = os.path.join(os.environ['CUR_PROJ_DIR'], 'model_runs', expanded_model_name)
    model_name = expanded_model_name.split('/')[1]
    with codecs.open(os.path.join(trained_model_path, 'run_info.json'), 'r', 'utf-8') as fp:
        run_info = json.load(fp)
        all_hparams = run_info['all_hparams']
        # Init model:
    if model_name in {'myspecter', 'cospecter'}:
        model = disent_models.MySPECTER(model_hparams=all_hparams)
    elif model_name in {'miswordbienc'}:
        model = disent_models.WordSentAlignBiEnc(model_hparams=all_hparams)
    elif model_name in {'sbalisentbienc'}:
        model = disent_models.WordSentAbsSupAlignBiEnc(model_hparams=all_hparams)
    model_fname = os.path.join(trained_model_path, 'model_{:s}.pt'.format('cur_best'))
    if torch.cuda.is_available():
        model.load_state_dict(torch.load(model_fname))
    else:
        model.load_state_dict(torch.load(model_fname, map_location=torch.device('cpu')))
    print(f'Loaded model: {trained_model_path}')
    return model.bert_encoder

def load_kpenc_model(expanded_model_name):
    """
    Given the name of a model to load, load it and return its
    base bert encoder.
    :param expanded_model_name: string of the form <dataset>/<model_name>/<run_name> which is part
        of the path to a trained model.
    """
    trained_model_path = os.path.join(os.environ['CUR_PROJ_DIR'], 'model_runs', expanded_model_name)
    with codecs.open(os.path.join(trained_model_path, 'run_info.json'), 'r', 'utf-8') as fp:
        run_info = json.load(fp)
        all_hparams = run_info['all_hparams']

    kp_encoder = AutoModel.from_pretrained(all_hparams['kp-base-pt-layer'])
    model_fname = os.path.join(trained_model_path, 'kp_encoder_{:s}.pt'.format('cur_best'))
    if torch.cuda.is_available():
        kp_encoder.load_state_dict(torch.load(model_fname))
    else:
        kp_encoder.load_state_dict(torch.load(model_fname, map_location=torch.device('cpu')))
    return kp_encoder

def print_sorted_dict(d, out_file):
    for k in sorted(d, key=d.get, reverse=True):
        try:
            out_file.write("{}, {}\n".format(k, d[k]))
        except UnicodeError:
            pass

def create_dir(dir_name):
    """
    Create the directory whose name is passed.
    :param dir_name: String saying the name of directory to create.
    :return: None.
    """
    # Create output directory if it doesnt exist.
    try:
        os.makedirs(dir_name)
        print('Created: {}.'.format(dir_name))
    except OSError as ose:
        # For the case of *file* by name of out_dir existing
        if (not os.path.isdir(dir_name)) and (ose.errno == errno.EEXIST):
            sys.stderr.write('IO ERROR: Could not create output directory\n')
            sys.exit(1)
        # If its something else you don't know; report it and exit.
        if ose.errno != errno.EEXIST:
            sys.stderr.write('OS ERROR: {:d}: {:s}: {:s}\n'.format(ose.errno,
                                                                   ose.strerror,
                                                                   dir_name))
            sys.exit(1)

def read_json(json_file):
    """
    Read per line JSON and yield.
    :param json_file: File-like with a next() method.
    :return: yield one json object.
    """
    for json_line in json_file:
        # Try to manually skip bad chars.
        # https://stackoverflow.com/a/9295597/3262406
        try:
            f_dict = json.loads(json_line.replace('\r\n', '\\r\\n'),
                                encoding='utf-8')
            yield f_dict
        # Skip case which crazy escape characters.
        except ValueError:
            raise

def plot_train_hist(y_vals, checked_iters, fig_path, ylabel, suffix=None):
    """
    Plot y_vals against the number of iterations.
    :param y_vals: list; values along the y-axis.
    :param checked_iters: list; len(y_vals)==len(checked_iters); the iterations
        the values in y_vals correspond to.
    :param fig_path: string; the directory to write the plots to.
    :param ylabel: string; the label for the y-axis.
    :param suffix: string; string to add to the figure filename.
    :return: None.
    """
    # If there is nothing to plot just return.
    if len(checked_iters) <= 3:
        return
    x_vals = np.array(checked_iters)
    y_vals = np.vstack(y_vals)
    plt.plot(x_vals, y_vals, '-', linewidth=2)
    plt.xlabel('Training iteration')
    plt.ylabel(ylabel)
    plt.title('Evaluated every: {:d} iterations'.format(
        checked_iters[1]-checked_iters[0]))
    plt.tight_layout()
    ylabel = '_'.join(ylabel.lower().split())
    if suffix:
        fig_file = os.path.join(fig_path, '{:s}_history-{:s}.eps'.format(ylabel, suffix))
    else:
        fig_file = os.path.join(fig_path, '{:s}_history.eps'.format(ylabel))
    plt.savefig(fig_file)
    if suffix:
        plt.savefig(os.path.join(fig_path, '{:s}_history-{:s}.png'.format(ylabel, suffix)))
    else:
        plt.savefig(os.path.join(fig_path, '{:s}_history.png'.format(ylabel)))
    plt.clf()
    logging.info('Wrote: {:s}'.format(fig_file))


import pandas as pd
import numpy as np
import sklearn
import time
from sklearn.tree import DecisionTreeClassifier

# function from Assignment 1

# column filter
def create_column_filter(df):
    df1 = df.copy()
    column_filter = ['CLASS']
    for col in df.columns:
        if col not in ['CLASS', 'ID']:
            if len(df[col].dropna().unique()) > 1:
                column_filter.append(col)
    df1 = df1[column_filter]
    return df1, column_filter

def apply_column_filter(df, column_filter):
    df1 = df.copy()
    df1 = df[column_filter]
    return df1

# normalization
def create_normalization(df,normalizationtype="minmax"):
    df1 = df.copy()
    normalization = {}
    for col in df.columns:
        if col not in ['CLASS', 'ID']:
            if df[col].dtype == 'float64' or df[col].dtype == 'int64':
                if normalizationtype == "minmax":
                    normalization[col] = ("minmax", df[col].min(), df[col].max())
                    df1[col] = (df[col] - df[col].min()) / (df[col].max() - df[col].min())
                elif normalizationtype == "zscore":
                    normalization[col] = ("zscore", df[col].mean(), df[col].std())
                    df1[col] = (df[col] - df[col].mean()) / df[col].std()
    return df1, normalization

def apply_normalization(df,normalization):
    df1 = df.copy()
    for col in df.columns:
        if col in normalization.keys():
            if normalization[col][0] == "minmax":
                df1[col] = (df[col] - normalization[col][1]) / (normalization[col][2] - normalization[col][1])
            elif normalization[col][0] == "zscore":
                df1[col] = (df[col] - normalization[col][1]) / normalization[col][2]
    return df1

# imputation
def create_imputation(df):
    df1 = df.copy()
    imputation = {}
    for col in df.columns:
        if col not in ['CLASS', 'ID']:
            if df[col].dtype == 'float64' or df[col].dtype == 'int64':
                imputation[col] = df[col].mean()
                df1[col] = df[col].fillna(imputation[col])
            elif df[col].dtype == 'object' or df[col].dtype == 'category':
                imputation[col] = df[col].mode()[0]
                df1[col] = df[col].fillna(imputation[col])
    return df1, imputation

def apply_imputation(df,imputation):
    df1 = df.copy()
    for col in df.columns:
        if col in imputation.keys():
            df1[col] = df[col].fillna(imputation[col])
    return df1

# binning
def create_bins(df, nobins=10, bintype="equal-width"):
    df1 = df.copy()
    binning = {}

    for col in df1.columns:
        if col not in ['CLASS', 'ID']:
            if df1[col].dtype == "int64" or df1[col].dtype == "float64":
                if bintype == "equal-width":
                    df1[col], binning[col] = pd.cut(df1[col],nobins,labels=False, retbins=True)
                elif bintype == "equal-size":
                    df1[col], binning[col] = pd.qcut(df[col],nobins, labels=False, retbins=True, duplicates='drop')

                binning[col][0] = -np.inf
                binning[col][-1] = np.inf
                df1[col] = df1[col].astype('category')
                df1[col] = df1[col].cat.set_categories([str(i) for i in df1[col].cat.categories], rename = True)

    return df1, binning

def apply_bins(df, binning):
    df1 = df.copy()

    for col in df1:
        if col in binning.keys():
            df1[col] = pd.cut(df1[col], binning[col], labels=False)
            df1[col] = df1[col].astype('category')
            df1[col] = df1[col].cat.set_categories([str(i) for i in df1[col].cat.categories], rename = True)

    return df1

# one-hot encoding
def create_one_hot(df):
    df1 = df.copy()
    one_hot = {}
    for col in df.columns:
        if col not in ['CLASS', 'ID']:
            if df[col].dtype == 'object' or df[col].dtype == 'category':
                one_hot[col] = df[col].unique()
                for val in one_hot[col]:
                    df1[col + '_' + str(val)] = (df1[col] == val).astype('float')
                df1.drop(col, axis=1, inplace=True)
    return df1, one_hot

def apply_one_hot(df,one_hot):
    df1 = df.copy()
    for col in one_hot.keys():
        for val in one_hot[col]:
            df1[col + '_' + str(val)] = (df1[col] == val).astype('float')
        df1.drop(col, axis=1, inplace=True)
    return df1

# split
def split(df, testfraction=0.5):
    randind = np.random.permutation(df.index)
    testind = randind[:int(len(randind)*testfraction)]
    testdf = df.iloc[testind]
    traindf = df.drop(testind)
    return traindf, testdf

# accuracy prediction
def accuracy(df, correctlabels):

    correct_pred = 0

    for index, row in df.iterrows():
        maxind = row.argmax()
        predlabel = df.columns[maxind]
        if predlabel == correctlabels[index]:
            correct_pred += 1

    accuracy = correct_pred / len(correctlabels)
    return accuracy

# folds
def folds(df,nofolds=10):
    shuffle = df.sample(frac=1)
    folds = np.array_split(shuffle, nofolds)
    return folds

# brier score
def brier_score(df, correctlabels):
    brier_score = 0
    n = len(df.index)

    for i,p in df.iterrows():
        true_label = correctlabels[i]
        o = np.zeros(len(p))
        o[df.columns==true_label] = 1
        brier_score += ((p-o)**2).sum(axis=0)
    brier_score = brier_score/n

    return brier_score

def auc_binary(predictions, correctlabels, threshold, c):
    # array with true for correct labels for class c (by row index)
    correctlabels_class = np.array(correctlabels)==predictions.columns[c]
    # array with predictions for all instances that should be classified class c
    predictions_class = predictions[predictions.columns[c]]
    # array with true for all correctly predicted labels according to threshold
    predicted_labels = predictions_class[correctlabels_class] >= threshold
    pos = sum(predicted_labels) # number of correctly predicted labels
    # tp / (tp + fn)
    tpr = pos / sum(correctlabels_class)

    # same reasoning for negative class
    not_correctlabels_class = np.array(correctlabels)!=predictions.columns[c]
    predictions_class = predictions[ predictions.columns[c] ]
    predicted_labels = predictions_class[not_correctlabels_class] >= threshold
    neg = sum(predicted_labels)
    # fpr = fp / (fp + tn)
    fpr = neg / sum(not_correctlabels_class)

    return tpr, fpr

def auc(predictions, correctlabels):
    thresholds = np.unique(predictions)
    AUC_d = {}

    # iterate over all classes and calculate the area under the ROC(tpr/fpr) curve (AUC)
    for (index,c) in enumerate(np.unique(correctlabels)):
        roc_points = [auc_binary(predictions, correctlabels, th, index) for th in reversed(thresholds)]

        # calculate AUC as area under the curve
        AUC = 0
        tpr_last = 0
        fpr_last = 0

        # iterate over all thresholds
        for r in roc_points:
            tpr, fpr = r
            # Add area under triangle
            if tpr > tpr_last and fpr > fpr_last:
                AUC += (fpr-fpr_last)*tpr_last + (fpr-fpr_last)*(tpr-tpr_last) / 2
            # Add area under rectangle
            elif fpr > fpr_last:
                AUC += (fpr-fpr_last)*tpr
            # update point coordinates (tpr, fpr) of curve
            tpr_last = tpr
            fpr_last = fpr

        AUC_d[c] = AUC

    # take the weighted average for all classes
    AUC_total = 0
    for (cName,auc) in AUC_d.items():
        number_of_labels = np.sum(np.array(correctlabels) == cName)
        weight = number_of_labels / len(correctlabels)
        AUC_total += weight * auc

    return AUC_total

# Define the class RandomForest with three functions __init__, fit and predict (after the comments):
#
# Input to __init__:
# self - the object itself
#
# Output from __init__:
# <nothing>
#
# This function does not return anything but just initializes the following attributes of the object (self) to None:
# column_filter, imputation, one_hot, labels, model
#
# Input to fit:
# self      - the object itself
# df        - a dataframe (where the column names "CLASS" and "ID" have special meaning)
# no_trees  - no. of trees in the random forest (default = 100)
#
# Output from fit:
# <nothing>
#
# The result of applying this function should be:
#
# self.column_filter - a column filter (see Assignment 1) from df
# self.imputation    - an imputation mapping (see Assignment 1) from df
# self.one_hot       - a one-hot mapping (see Assignment 1) from df
# self.labels        - a (sorted) list of the categories of the "CLASS" column of df
# self.model         - a random forest, consisting of no_trees trees, where each tree is generated from a bootstrap sample
#                      and the number of evaluated features is log2|F| where |F| is the total number of features
#                      (for details, see lecture slides)
#
# Note that the function does not return anything but just assigns values to the attributes of the object.
#
# Hint 1: First create the column filter, imputation and one-hot mappings
#
# Hint 2: Then get the class labels and the numerical values (as an ndarray) from the dataframe after dropping the class labels
#
# Hint 3: Generate no_trees classification trees, where each tree is generated using DecisionTreeClassifier
#         from a bootstrap sample (see lecture slides), e.g., generated by np.random.choice (with replacement)
#         from the row numbers of the ndarray, and where a random sample of the features are evaluated in
#         each node of each tree, of size log2(|F|), where |F| is the total number of features;
#         see the parameter max_features of DecisionTreeClassifier
#
# Input to predict:
# self - the object itself
# df   - a dataframe
#
# Output from predict:
# predictions - a dataframe with class labels as column names and the rows corresponding to
#               predictions with estimated class probabilities for each row in df, where the class probabilities
#               are the averaged probabilities output by each decision tree in the forest
#
# Hint 1: Drop any "CLASS" and "ID" columns of the dataframe first and then apply column filter, imputation and one_hot
#
# Hint 2: Iterate over the trees in the forest to get the prediction of each tree by the method predict_proba(X) where
#         X are the (numerical) values of the transformed dataframe; you may get the average predictions of all trees,
#         by first creating a zero-matrix with one row for each test instance and one column for each class label,
#         to which you add the prediction of each tree on each iteration, and then finally divide the prediction matrix
#         by the number of trees.
#
# Hint 3: You may assume that each bootstrap sample that was used to generate each tree has included all possible
#         class labels and hence the prediction of each tree will contain probabilities for all class labels
#         (in the same order). Note that this assumption may be violated, and this limitation will be addressed
#         in the next part of the assignment.

class RandomForest:

    def __init__(self):
        self.column_filter = None
        self.imputation = None
        self.one_hot = None
        self.labels = None
        self.model = None

    def fit(self, dt, no_trees=100):
        df1 = dt.copy()
        df1, self.column_filter  = create_column_filter(df1)
        df1, self.imputation = create_imputation(df1)
        df1, self.one_hot = create_one_hot(df1)
        self.labels = list(df1["CLASS"].astype("category").cat.categories)
        features = df1.drop(columns=["ID", "CLASS"], errors='ignore').to_numpy()

        models = []

        # Hint 3: Generate trees from bootstrap replicas of the data
        for i in range(no_trees):
            row_num=np.random.choice(len(features),len(features), replace=True)
            data = features[row_num]
            # get the labels for the fit of the model
            labels = df1['CLASS'].to_numpy()[row_num]
            tree = DecisionTreeClassifier(max_features='log2')
            # fit the model with data and labels
            tree.fit(data,labels)
            models.append(tree)

        self.model=models

    def predict(self, df):
        df1 = df.copy()
        df1 = apply_column_filter(df1, self.column_filter)
        df1 = apply_imputation(df1, self.imputation)
        df1 = apply_one_hot(df1,self.one_hot)
        df1.drop(columns=["CLASS"], inplace=True)

        # matrix of zeros with one row for each test instance and one column for each class label
        probabilities = np.zeros((len(df1),len(self.labels)))

        # iterate over the trees in the forest and calculate probabilities (hint 2)
        for tree in self.model:
            for i, row in enumerate(df1.values):
                result = tree.predict_proba(row.reshape(1,-1))
                probabilities[i] = probabilities[i] + result

        probabilities = probabilities / len(self.model)
        predictions = pd.DataFrame(probabilities, columns=self.labels)

        return predictions

# check results
def check_1():
    train_df = pd.read_csv("Assignment_3/tic-tac-toe_train.csv")

    test_df = pd.read_csv("Assignment_3/tic-tac-toe_test.csv")

    rf = RandomForest()

    t0 = time.perf_counter()
    rf.fit(train_df)
    print("Training time: {:.2f} s.".format(time.perf_counter()-t0))

    test_labels = test_df["CLASS"]

    t0 = time.perf_counter()
    predictions = rf.predict(test_df)

    print("Testing time: {:.2f} s.".format(time.perf_counter()-t0))

    print("Accuracy: {:.4f}".format(accuracy(predictions,test_labels)))
    print("AUC: {:.4f}".format(auc(predictions,test_labels))) # Comment this out if not implemented in assignment 1
    print("Brier score: {:.4f}".format(brier_score(predictions,test_labels))) # Comment this out if not implemented in assignment 1

    train_labels = train_df["CLASS"]
    predictions = rf.predict(train_df)
    print("Accuracy on training set: {0:.4f}".format(accuracy(predictions,train_labels)))
    print("AUC on training set: {0:.4f}".format(auc(predictions,train_labels))) # Comment this out if not implemented in assignment 1
    print("Brier score on training set: {0:.4f}".format(brier_score(predictions,train_labels))) # Comment this out if not implemented in assignment 1

# Define a revised version of the class RandomForest with the same input and output as described in part 1 above,
# where the predict function is able to handle the case where the individual trees are trained on bootstrap samples
# that do not include all class labels in the original training set. This leads to that the class probabilities output
# by the individual trees in the forest do not refer to the same set of class labels.
#
# Hint 1: The categories obtained with <pandas series>.cat.categories are sorted in the same way as the class labels
#         of a DecisionTreeClassifier; the latter are obtained by <DecisionTreeClassifier>.classes_
#         The problem is that classes_ may not include all possible labels, and hence the individual predictions
#         obtained by <DecisionTreeClassifier>.predict_proba may be of different length or even if they are of the same
#         length do not necessarily refer to the same class labels. You may assume that each class label that is not included
#         in a bootstrap sample should be assigned zero probability by the tree generated from the bootstrap sample.
#
# Hint 2: Create a mapping from the complete (and sorted) set of class labels l0, ..., lk-1 to a set of indexes 0, ..., k-1,
#         where k is the number of classes
#
# Hint 3: For each tree t in the forest, create a (zero) matrix with one row per test instance and one column per class label,
#         to which one column is added at a time from the output of t.predict_proba
#
# Hint 4: For each column output by t.predict_proba, its index i may be used to obtain its label by t.classes_[i];
#         you may then obtain the index of this label in the ordered list of all possible labels from the above mapping (hint 2);
#         this index points to which column in the prediction matrix the output column should be added to

class RandomForest2a:

    def __init__(self):
        self.column_filter = None
        self.imputation = None
        self.one_hot = None
        self.labels = None
        self.model = None
        self.label_mapping = None

    def fit(self, df, no_trees=100):
        df1 = df.copy()

        df1, self.column_filter = create_column_filter(df1)
        df1, self.imputation = create_imputation(df1)
        df1, self.one_hot = create_one_hot(df1)
        self.labels = list(df1["CLASS"].astype("category").cat.categories)
        self.label_mapping = dict(zip(self.labels, list(range(len(self.labels))))) # mapping hint 2
        features = df1.drop(columns=["ID", "CLASS"], errors='ignore').to_numpy()

        models = []

        # Same thing as point 1
        for i in range(no_trees):
            row_num = np.random.choice(len(features), len(features), replace=True)
            data = features[row_num]
            labels = df1['CLASS'].to_numpy()[row_num]
            tree = DecisionTreeClassifier(max_features='log2')
            # fit the model with data and labels
            tree.fit(data,labels)
            models.append(tree)

        self.model = models

    def predict(self, df):
        df1 = df.copy()

        df1 = apply_column_filter(df1, self.column_filter)
        df1 = apply_imputation(df1, self.imputation)
        df1 = apply_one_hot(df1,self.one_hot)
        df1.drop(columns=["CLASS"], inplace=True)

        # matrix of zeros (hint 3)
        probabilities = np.zeros((len(df1),len(self.labels)))

        for tree in self.model:
            for i, row in enumerate(df1.values):
                result = tree.predict_proba(row.reshape(1,-1))

                # hint 4
                for j, col in enumerate(tree.classes_):
                    label_i = self.label_mapping[col]
                    probabilities[i][label_i] = probabilities[i][label_i] + result[0][j]

        probabilities = probabilities / len(self.model)
        predictions = pd.DataFrame(probabilities, columns=self.labels)

        return predictions

# check results
def check_2a():
    train_df = pd.read_csv("Assignment_3/anneal_train.csv")

    test_df = pd.read_csv("Assignment_3/anneal_test.csv")

    rf = RandomForest2a()

    t0 = time.perf_counter()
    rf.fit(train_df)
    print("Training time: {:.2f} s.".format(time.perf_counter()-t0))

    test_labels = test_df["CLASS"]

    t0 = time.perf_counter()
    predictions = rf.predict(test_df)
    print("Testing time: {:.2f} s.".format(time.perf_counter()-t0))

    print("Accuracy: {:.4f}".format(accuracy(predictions,test_labels)))
    print("AUC: {:.4f}".format(auc(predictions,test_labels))) # Comment this out if not implemented in assignment 1
    print("Brier score: {:.4f}".format(brier_score(predictions,test_labels))) # Comment this out if not implemented in assignment 1

# Define an extended version of the class RandomForest with the same input and output as described in part 2a above,
# where the results of the fit function also should include:
# self.oob_acc - the accuracy estimated on the out-of-bag predictions, i.e., the fraction of training instances for
#                which the given (correct) label is the same as the predicted label when using only trees for which
#                the instance is out-of-bag
#
# Hint 1: You may first create a zero matrix with one row for each training instance and one column for each class label
#         and one zero vector to allow for storing aggregated out-of-bag predictions and the number of out-of-bag predictions
#         for each training instance, respectively. By "aggregated out-of-bag predictions" is here meant the sum of all
#         predicted probabilities (one sum per class and instance). These sums should be divided by the number of predictions
#         (stored in the vector) in order to obtain a single class probability distribution per training instance.
#         This distribution is considered to be the out-of-bag prediction for each instance, and e.g., the class that
#         receives the highest probability for each instance can be compared to the correct label of the instance,
#         when calculating the accuracy using the out-of-bag predictions.
#
# Hint 2: After generating a tree in the forest, iterate over the indexes that were not included in the bootstrap sample
#         and add a prediction of the tree to the out-of-bag prediction matrix and update the count vector
#
# Hint 3: Note that the input to predict_proba has to be a matrix; from a single vector (row) x, a matrix with one row
#         can be obtained by x[None,:]
#
# Hint 4: Finally, divide each row in the out-of-bag prediction matrix with the corresponding element of the count vector
#
#         For example, assuming that we have two class labels, then we may end up with the following matrix:
#
#         2 4
#         4 4
#         5 0
#         ...
#
#         and the vector (no. of predictions) (6, 8, 5, ...)
#
#         The resulting class probability distributions are:
#
#         0.333... 0.666...
#         0.5 0.5
#         1.0 0

class RandomForest2b:

    def __init__(self):
        self.column_filter = None
        self.imputation = None
        self.one_hot = None
        self.labels = None
        self.model = None
        self.label_mapping = None
        self.oob_acc = None

    def fit(self, df, no_trees=100):
        df1 = df.copy()

        df1, self.column_filter = create_column_filter(df1)
        df1, self.imputation = create_imputation(df1)
        df1, self.one_hot = create_one_hot(df1)
        self.labels = list(df1["CLASS"].astype("category").cat.categories)
        self.label_mapping = dict(zip(self.labels, list(range(len(self.labels)))))
        features = df1.drop(columns=["ID", "CLASS"], errors='ignore').to_numpy()

        models = []
        missing_trees = []

        # same as point1 and 2a
        for i in range(no_trees):
            row_num = np.random.choice(len(features), len(features), replace=True)
            data = features[row_num]
            labels = df1['CLASS'].to_numpy()[row_num]
            tree = DecisionTreeClassifier(max_features='log2')
            # fit the model with data and labels
            tree.fit(data,labels)
            models.append(tree)

            # Checking out of the bag features and populate the array
            missing = [x for x in range(len(features)) if x not in row_num]
            missing_trees.append(missing)

        self.model = models
        self.oob_acc = self.calc_oob_acc(missing_trees, features, df1)

    def calc_oob_acc(self, missing_trees, features, df1):
        # Hint 1, create matrix and vector of zeros
        probabilities = np.zeros((len(features), len(self.labels)))
        vector = np.zeros(len(features))

        for i , curr_miss in enumerate(missing_trees):
            tree = self.model[i]

            # hint 2
            for row in curr_miss:
                curr_train = features[row].reshape(1,-1)
                result = tree.predict_proba(curr_train)

                # Same way as in predict function
                for j, col in enumerate(tree.classes_):
                    label_i = self.label_mapping[col]
                    probabilities[row][label_i] = probabilities[row][label_i] + result[0][j]

                vector[row] += 1

        # hint 3 and 4
        acc = probabilities / vector[:, None]
        acc = pd.DataFrame(acc, columns=self.labels)
        result = accuracy(acc, df1['CLASS'].astype("category"))

        return result

    def predict(self, df):
        df1 = df.copy()

        df1 = apply_column_filter(df1, self.column_filter)
        df1 = apply_imputation(df1, self.imputation)
        df1 = apply_one_hot(df1, self.one_hot)
        df1.drop(columns=["CLASS"], inplace=True)

        probabilities = np.zeros((len(df1), len(self.labels)))

        for tree in self.model:
            for i, row in enumerate(df1.values):
                classes = tree.classes_
                result = tree.predict_proba(row.reshape(1,-1))

                for j, col in enumerate(classes):
                    label_i = self.label_mapping[col]
                    probabilities[i][label_i] = probabilities[i][label_i] + result[0][j]

        probabilities = probabilities / len(self.model)
        predictions = pd.DataFrame(probabilities, columns=self.labels)

        return predictions

# check results
def check_2b():
    train_df = pd.read_csv("Assignment_3/anneal_train.csv")

    test_df = pd.read_csv("Assignment_3/anneal_test.csv")

    rf = RandomForest2b()

    t0 = time.perf_counter()
    rf.fit(train_df)
    print("Training time: {:.2f} s.".format(time.perf_counter()-t0))

    print("OOB accuracy: {:.4f}".format(rf.oob_acc))

    test_labels = test_df["CLASS"]

    t0 = time.perf_counter()
    predictions = rf.predict(test_df)
    print("Testing time: {:.2f} s.".format(time.perf_counter()-t0))

    print("Accuracy: {:.4f}".format(accuracy(predictions,test_labels)))
    print("AUC: {:.4f}".format(auc(predictions,test_labels))) # Comment this out if not implemented in assignment 1
    print("Brier score: {:.4f}".format(brier_score(predictions,test_labels))) # Comment this out if not implemented in assignment 1

    train_labels = train_df["CLASS"]
    rf = RandomForest2b()
    rf.fit(train_df)
    predictions = rf.predict(train_df)
    print("Accuracy on training set: {0:.2f}".format(accuracy(predictions,train_labels)))
    print("AUC on training set: {0:.2f}".format(auc(predictions,train_labels)))
    print("Brier score on training set: {0:.2f}".format(brier_score(predictions,train_labels)))

if __name__ == "__main__":
    print("------ Assignment 1 ---------")
    check_1()
    print("------ Assignment 2a ---------")
    check_2a()
    print("------ Assignment 2b ---------")
    check_2b()


import pybind11_stubgen
from pathlib import Path

if __name__ == '__main__':

    pybind11_stubgen.main()

    stubs_path = (Path(__file__).parents[0].resolve() / 'stubs').resolve()
    stubs = list(stubs_path.rglob('*.pyi'))
    for x in stubs:
        x.replace(x.with_suffix('.py'))


from unittest.mock import patch
from django.test import TestCase

from corehq.apps.data_dictionary.models import (
    CaseProperty,
    CasePropertyGroup,
    CaseType,
)
from corehq.apps.reports.standard.cases.case_data import (
    _get_case_property_tables,
)

@patch('corehq.apps.reports.standard.cases.case_data.domain_has_privilege', lambda _, __: True)
class TestCaseData(TestCase):

    def setUp(self) -> None:
        self.domain = "test-dd-domain"
        self.case_type = "case-test"
        self.timezone = "Asia/Kolkata"

    def tearDown(self) -> None:
        CaseProperty.objects.all().delete()
        CasePropertyGroup.objects.all().delete()
        CaseType.objects.all().delete()

    def test_no_data_dictionary(self):
        dynamic_data = {"prop1": None, "prop2": None}
        result = _get_case_property_tables(self.domain, self.case_type, dynamic_data, self.timezone)
        self._assert_grouping_order_tables(result, [{"name": None, "properties": ["prop1", "prop2"]}])

    def test_dd_ordering_when_single_group(self):
        """
        Tests that when a data dictionary contains only one user defined group,
        properties within the group is retained.
        """
        # Create group1 properties
        self._create_case_property("prop1", "group1")
        self._create_case_property("prop2", "group1")
        result = _get_case_property_tables(self.domain, self.case_type, {}, self.timezone)
        self._assert_grouping_order_tables(result, [{"name": "group1", "properties": ["prop1", "prop2"]}])

    def test_dd_ordering_when_multiple_groups(self):
        """
        Tests that when a data dictionary contains multiple user defined groups,
        ordering of groups and properties within the group is retained.
        """
        # Create group1 properties
        self._create_case_property("prop1", "group1")
        self._create_case_property("prop2", "group1")
        self._create_case_property("prop3", "group2")
        self._create_case_property("prop4", "group2")
        result = _get_case_property_tables(self.domain, self.case_type, {}, self.timezone)
        self._assert_grouping_order_tables(result, [{"name": "group1", "properties": ["prop1", "prop2"]},
                                                    {"name": "group2", "properties": ["prop3", "prop4"]}])

    def test_dd_ordering_when_multiple_groups_and_unrecognized_group(self):
        """
        Tests that when a data dictionary contains multiple user defined groups and a unrecognized group,
        ordering of groups and properties within the group is retained. Ordering is not retained in
        unrecognized group.
        """
        # Create group1 properties
        self._create_case_property("prop1", "group1")
        self._create_case_property("prop2", "group1")
        self._create_case_property("prop3", "group2")
        self._create_case_property("prop4", "group2")

        # Dynamic data with additional property not present in any of the groups
        dynamic_data = {"prop1": None, "prop2": None, "prop3": None, "prop4": None, "prop5": None, "prop6": None}

        result = _get_case_property_tables(self.domain, self.case_type, dynamic_data, self.timezone)
        self._assert_grouping_order_tables(result, [{"name": "group1", "properties": ["prop1", "prop2"]},
                                                    {"name": "group2", "properties": ["prop3", "prop4"]},
                                                    {"name": "Unrecognized", "properties": ["prop5", "prop6"]}
                                                    ])

    def _assert_grouping_order_tables(self, result, expected_result):
        """
        Sample result
        [{'name': 'group1', 'rows': [[{'expr': 'prop1', 'name': 'prop1', 'description': '', 'value': '---',
        'has_history': True}, {'expr': 'prop2', 'name': 'prop2', 'description': '', 'value': '---',
        'has_history': True}]]}, {'name': 'group2', 'rows': [[{'expr': 'prop3', 'name': 'prop3',
        'description': '', 'value': '---', 'has_history': True}, {'expr': 'prop4', 'name': 'prop4',
        'description': '', 'value': '---', 'has_history': True}]]}]
        """
        for group_data, expected_group_data in zip(result, expected_result):
            group_name_actual = group_data.get("name")
            group_name_expected = expected_group_data.get("name")
            self.assertEqual(group_name_expected, group_name_actual)
            rows_actual = group_data.get("rows")[0]
            props_actual = [prop.get("name") for prop in rows_actual]
            if not group_name_actual or group_name_actual == "Unrecognized":
                # Ordering of properties in unrecognized group is not user defined
                self.assertEqual(set(expected_group_data.get("properties")), set(props_actual))
            else:
                self.assertEqual(expected_group_data.get("properties"), props_actual)

    def _create_case_property(self, prop_name, group=None):
        case_type_obj = CaseType.get_or_create(self.domain, self.case_type)
        group_obj = None
        if group:
            group_obj, _ = CasePropertyGroup.objects.get_or_create(name=group, case_type=case_type_obj)
        CaseProperty.objects.get_or_create(case_type=case_type_obj, name=prop_name, group=group_obj)


#!iaskdata/bin/python
# -*- coding: utf8 -*-

from datetime import datetime, timedelta
from config import BASEDIR
from app import app, db
from app.models import BaseUser, Customer, Expert, Topic, Category, TopicRequest, Comment
from app.model_utils import *

def build_topic_1():
    test_topic_1 = Topic(
	topic_id = 1,
        body = 'Our company needs a real-time recommendation system, who can help us?',
        title = u'推荐系统求助',
        created_time = datetime.utcnow(),
        rate = 100.0,
        expert_id = 101
    )
    return test_topic_1

def build_topic_2():
    test_topic_2 = Topic(
	topic_id = 2,
        body = 'A VR company is seeking computer vision experts',
        title = u'VR & AR',
        created_time = datetime.utcnow(),
        rate = 150.0,
        expert_id = 101
    )
    return test_topic_2

def build_customer_1():
    return Customer(
        user_id = 1,
        email = 'larry@iaskdata.com',
        last_seen = datetime.utcnow(),
        name = 'larry',
        company = 'google',
        title = 'CEO',
        about_me = 'i love my job',
        avatar_thumbnail_url = 'thumbnail',
        avatar_url = 'image',
        phone_number = '13880089000'
    )

def build_customer_2():
    return Customer(
        user_id = 2,
        email = 'zuck@iaskdata.com',
        last_seen = datetime.utcnow(),
        name = 'zuck',
        company = 'facebook',
        title = 'CEO',
        about_me = 'i love my job',
        avatar_thumbnail_url = 'thumbnail',
        avatar_url = 'image',
        phone_number = '13880089001'
    )

def build_customer_3():
    return Customer(
        user_id = 3,
        email = 'jack@iaskdata.com',
        last_seen = datetime.utcnow(),
        name = 'jack',
        company = 'Twitter',
        title = 'CEO',
        about_me = 'i love my job',
        phone_number = '13880089002'
    )

def build_expert_1():
    return Expert(
        user_id = 101,
        email = 'andrewng@iaskdata.com',
        last_seen = datetime.utcnow(),
        name = 'Andrew NG',
        company = 'Coursera',
        title = 'Founder',
        about_me = 'I am just GOOOOD',
        degree = 'Ph.D.',
        university = 'Stanford University',
        major = 'Computer Science',
        rating = 4.9,
        bio = 'I graduated from the best university and I created the best company in the field',
        credits = 5
    )

def build_category_1():
    category_id = build_category_id(1, 2)
    return Category(category_id = category_id)

def build_category_2():
    category_id = build_category_id(1, 3)
    return Category(category_id = category_id)

def build_comment_1():
    return Comment(
        comment_id = 1,
    	content = 'this guy is really good at his major',
        request_id = 1,
        rating = 4.6
    )

def build_comment_2():
    return Comment(
        comment_id = 2,
    	content = 'bravo, this expert really knows his field',
        request_id = 2,
        rating = 4.8
    )

def build_topic_request_1():
    return TopicRequest(
        request_id = 1,
        customer_id = 1,
        topic_id = 1,
        request_stage = 0,
    	topic_requested_time = datetime.utcnow(),
    	topic_accepted_time = datetime.utcnow(),
        topic_paid_time = datetime.utcnow(),
        topic_scheduled_time = datetime.utcnow(),
        topic_served_time = datetime.utcnow(),
        topic_rated_time = datetime.utcnow()
    )

def build_topic_request_2():
    return TopicRequest(
        request_id = 2,
        customer_id = 2,
        topic_id = 1,
        request_stage = 0,
    	topic_requested_time = datetime.utcnow(),
    	topic_accepted_time = datetime.utcnow(),
        topic_paid_time = datetime.utcnow(),
        topic_scheduled_time = datetime.utcnow(),
        topic_served_time = datetime.utcnow(),
        topic_rated_time = datetime.utcnow()
    )


import pytest
from Day3.Rectangle import Rectangle

class TestRectangle:
    @pytest.mark.parametrize("from_left,from_top,height,width,result", [
        (0, 0, 1, 1, [(0,0)]),
        (0, 1, 1, 1, [(0,1)]),
        (0, 0, 1, 2, [(0,0), (0,1)]),
        (0, 0, 2, 1, [(0,0), (1,0)]),
        (1, 0, 1, 1, [(1,0)]),
        (2, 2, 3, 3, [
            (2,2), (3,2), (4,2),
            (2,3), (3,3), (4,3),
            (2,4), (3,4), (4,4),
        ]),
    ])
    def test_should_have_areas(self, from_left, from_top, height, width, result):
        rectangle = Rectangle(from_left, from_top, width, height)

        assert rectangle.areas() == result

    class Test_overlap:
        def test_should_return_false_when_not_overlap(self):
            rectangle_one = Rectangle(0, 0, 1, 1)
            rectangle_two = Rectangle(2, 2, 2, 2)

            assert rectangle_one.overlap(rectangle_two) == False
            assert rectangle_two.overlap(rectangle_one) == False

        def test_should_return_true_when_not_overlap(self):
            rectangle_one = Rectangle(0, 0, 3, 3)
            rectangle_two = Rectangle(2, 2, 2, 2)

            assert rectangle_one.overlap(rectangle_two) == True
            assert rectangle_two.overlap(rectangle_one) == True


import time
from datetime import datetime
from logging import Logger
from lib.kafka_connect.kafka_connectors import KafkaConsumer, KafkaProducer
import json
from typing import Dict, Optional

from lib.redis.redis_client import  RedisClient
from stg_loader.repository.stg_repository import StgRepository

import json

class StgMessageProcessor:
    def __init__(self,
                 consumer: KafkaConsumer,
                 producer: KafkaProducer,
                 redis:RedisClient,
                 stg_repository:StgRepository,
                 batch_size: int,
                 logger: Logger
                 ) -> None:

        self._consumer=consumer
        self._producer=producer
        self._redis=redis
        self._stg_repository=stg_repository
        self._batch_size=batch_size
        self._logger = logger

    # функция, которая будет вызываться по расписанию.
    def run(self) -> None:
        # Пишем в лог, что джоб был запущен.
        self._logger.info(f"{datetime.utcnow()}: START")

        i=0
        while i< self._batch_size:
            self._logger.info(f"{datetime.utcnow()}: {self._consumer}")
            self._logger.info(f"{datetime.utcnow()}: {type(self._stg_repository)}")
            message=self._consumer.consume()
            self._logger.info(f"{datetime.utcnow()}: {message}")

            if message is None:
                break
            else:
                self._stg_repository.order_events_insert(message['object_id'],
                                                     message['object_type'],
                                                     message['sent_dttm'],
                                                     json.dumps(message['payload']))
                self._logger.info(f"{datetime.utcnow()}: POSTGRES FINISHED")

                products_list=[]
                for product in message['payload']['order_items']:
                        for item in self._redis.get(message['payload']['restaurant']['id'])['menu']:
                            if item['_id']==product["id"]:
                                product['name']=item['name']
                                product['category']=item['category']

                        products_list.append(product)

                result={
                    "object_id": message['object_id'],
                    "object_type":message['object_type'],
                    "payload": {
                        "id": message['object_id'],
                        "date": message['payload']['date'],
                        "cost": message['payload']['cost'],
                        "payment": message['payload']['payment'],
                        "status": message['payload']['final_status'],
                        "restaurant": {
                                        "id": message['payload']['restaurant']['id'],
                                        "name": self._redis.get(message['payload']['restaurant']['id'])['name']
                                    },
                        "user": {
                            "id": message['payload']['user']['id'],
                            "name": self._redis.get(message['payload']['user']['id'])['name']
                        },
                        "products": products_list
                        }
                    }
                self._producer.produce(result)
            i+=1

        # Пишем в лог, что джоб успешно завершен.
        self._logger.info(f"{datetime.utcnow()}: FINISH")


import numpy as np
from scipy.special import logsumexp
from scipy.special import expit as logistic_sigmoid

from matplotlib import pyplot as plt

from LRGradientDescent import LogisticRegressionGradientDescent

import sklearn.preprocessing
import sklearn.pipeline
from sklearn.base import BaseEstimator, TransformerMixin

class LRGDWithFeatureTransform(LogisticRegressionGradientDescent):
    ''' Logistic Regression classifier using special feature transform preprocessing!

    Implements the standard sklearn binary classifier API:
    * fit : train the model and set internal trainable attributes
    * predict : produce hard binary predictions
    * predict_proba : produce probabilistic predictions for both labels (0 and 1)

    Inherits all methods from LogisticRegressionGradientDescent, including:
    * constructor
    * gradient descent training of weights via fit
    * utility methods like initialize_w

    '''

    def fit_feature_pipeline(self, x_NF, y_N=None):
        ''' Train the feature-processing pipeline that happens before the LR classifier

        Args
        ----
        x_NF : 2D array, size N x F (n_examples x n_features)
            Each row is a feature vector
        y_N : 1D array, size N (n_examples)
            Each entry is a binary label

        Returns
        -------
        Nothing. Internal attribute 'feature_transform_pipeline' updated.
        '''
        # TODO edit this pipeline!
        # Add whatever additional preprocessing would be useful!
        self.feature_transform_pipeline = sklearn.pipeline.Pipeline(
            [('feature_transform', sklearn.pipeline.FeatureUnion(transformer_list=[
                    ('original_x', sklearn.preprocessing.PolynomialFeatures(degree=1, include_bias=False)),
                    ('squared_x', SquaredFeatureExtractor()),
                    ('avg_x', AverageValueFeatureExtractor()),
                    ]),
            )])
        self.feature_transform_pipeline.fit(x_NF, y_N)

    def predict_proba(self, x_NF):
        ''' Produce soft probabilistic predictions for provided input features

        Args
        ----
        x_NF : 2D array, size N x F (n_examples x n_features_excluding_bias)
            Input features (one row per example).

        Returns
        -------
        yproba_N2 : 2D array, size N x 2
            First column gives probability of zero label (negative)
            Second column gives probability of one label (positive)
            Each entry is a non-negative probability value within (0.0, 1.0)
            Each row sums to one
        '''
        # Transform each original F-dim feature vector to a new H-dim vector!
        feat_NH = self.feature_transform_pipeline.transform(x_NF)
        # Now call your original predict_proba code defined in parent class
        # but passing in the transformed feats!
        return super(LRGDWithFeatureTransform, self).predict_proba(feat_NH)

    ### Method for training

    def fit(self, x_NF, y_N):
        ''' Fit logistic regression model to provided training data

        Will minimize the loss function defined by calc_loss

        Returns
        -------
        Nothing. Only internal instance attributes updated.

        Post Condition
        --------------
        Internal attributes are updated:
        * w_G contains the optimal weights
        * trace_loss contains loss at every step of gradient descent
        * trace_L1_norm_of_grad contains L1 norm of grad after every step
        '''
        # Train the feature processing pipeline
        self.fit_feature_pipeline(x_NF, y_N)
        # Transform each original F-dim feature vector to a new H-dim vector!
        feat_NH = self.feature_transform_pipeline.transform(x_NF)
        ## Now call your original gradient descent code,
        # as defined in the parent class LogisticRegressionGradientDescent,
        # but use the transformed features as input to LR classifier
        return super(LRGDWithFeatureTransform, self).fit(feat_NH, y_N)

class SquaredFeatureExtractor(BaseEstimator, TransformerMixin):
    """ Extracts feature equal to square of each original feature
    """

    def __init__(self):
        pass

    def transform(self, x, y=None):
        """ Average all feature values into a new feature column

        Args
        ----
        x : 2D array, size F

        Returns
        -------
        feat : 2D array, size N x F
            One feature extracted for each example
        """
        return np.square(x)

    def fit(self, x, y=None):
        """ Nothing happens when fitting
        """
        return self

class AverageValueFeatureExtractor(BaseEstimator, TransformerMixin):
    """ Extracts feature equal to the *sum* of all pixels in image
    """

    def __init__(self):
        pass

    def transform(self, x, y=None):
        """ Average all feature values into a new feature column

        Returns
        -------
        feat : 2D array, size N x 1
            One feature extracted for each example
        """
        return np.sum(x, axis=1)[:,np.newaxis]

    def fit(self, x, y=None):
        """ Nothing happens when fitting
        """
        return self

if __name__ == '__main__':
    ## Toy problem
    #
    # Generate negative-label points in a tight cluster centered at zero
    # General positive-label points in a wide ring centered at zero
    #
    # Logistic regression on original features should struggle
    # But using our *squared* features, we should get better performance

    C = 50 # num examples per class
    N = 2 * C # num examples total
    random_state = np.random.RandomState(0)
    x_pos_C2 = 0.5 * random_state.randn(C, 2)
    x_neg_M2 = 5.0 * random_state.randn(100 * C, 2)
    radius_M = np.sqrt(np.sum(np.square(x_neg_M2), axis=1))
    fits_ring_M = np.logical_and(radius_M >= 1.8, radius_M <= 2.5)
    x_neg_C2 = x_neg_M2[fits_ring_M][:C]

    x_N2 = np.vstack([x_neg_C2, x_pos_C2])
    y_N = np.hstack([np.zeros(C), 1.0 * np.ones(C)])

    plt.plot(x_N2[y_N == 1,0], x_N2[y_N == 1,1], color='b', marker='+', ls='')
    plt.plot(x_N2[y_N == 0,0], x_N2[y_N == 0,1], color='r', marker='x', ls='')

    ## Run LR on original features!
    orig_lr = LogisticRegressionGradientDescent(alpha=10.0, step_size=0.1)
    orig_lr.fit(x_N2, y_N)

    ## Run LR on transformed features!
    new_lr = LRGDWithFeatureTransform(alpha=10.0, step_size=0.1)
    new_lr.fit(x_N2, y_N)

    np.set_printoptions(precision=4, suppress=1, linewidth=100)
    for nn in range(3):
        print("Example %d" % nn)
        print("Original feature vector: %s" % x_N2[nn])
        print("     New feature vector: %s" % (
            new_lr.feature_transform_pipeline.transform(x_N2[nn:nn+1])[0]))

    yproba1_orig_N = orig_lr.predict_proba(x_N2)[:,1]
    yproba1_new_N = new_lr.predict_proba(x_N2)[:,1]

    print("Training Error with orig features: %.3f" % (
        np.mean(np.logical_xor(y_N, yproba1_orig_N >= 0.5))))
    print("Training Error with new  features: %.3f" % (
        np.mean(np.logical_xor(y_N, yproba1_new_N >= 0.5))))

    plt.show()


import os
import cv2
import SimpleITK as sitk
from radiomics import featureextractor
import pandas as pd
import numpy as np
# Defina os rótulos das classes
labels = {
    'Healthy': 0,
    'Die Back': 1,
    'Cutting Weevil': 2
}

def extrair_caracteristicas(imagem):
    # Verifique se a imagem já está em escala de cinza
    if len(imagem.shape) == 2 or (len(imagem.shape) == 3 and imagem.shape[2] == 1):
        image_cv2 = imagem
    else:
        # A imagem não está em escala de cinza, converta para escala de cinza
        image_cv2 = cv2.cvtColor(imagem, cv2.COLOR_RGB2GRAY)

    # Crie uma imagem SimpleITK
    image = sitk.GetImageFromArray(image_cv2)

    # Crie uma máscara que cubra toda a imagem
    mask = sitk.Image(image.GetSize(), sitk.sitkUInt8)
    mask.CopyInformation(image)
    mask = sitk.BinaryThreshold(mask, lowerThreshold=0, upperThreshold=1, insideValue=1, outsideValue=0)

    # Crie o extrator de características Radiomics
    extractor = featureextractor.RadiomicsFeatureExtractor(shape2D=True)

    # Calcule as características radiômicas
    result = extractor.execute(image, mask)

    # Converta as características para um array numpy
    features_array = np.concatenate([np.atleast_1d(result[key]).ravel() for key in result.keys()])

    return features_array


#!/usr/bin/python

# Create dataset for toy linear regression problems
# Convert dataset to TFRecords format

import argparse
import os
import sys

import tensorflow as tf
import numpy as np

FLAGS = None

def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _float_feature(value):
    return tf.train.Feature(float_list=tf.train.FloatList(value=[value]))

def generate_dataset(size):
    x = np.random.rand(1, size)
    noise = np.random.rand(1, size) * 0.1
    y = x + noise

    return x[0], y[0]

def convert_to(size, name):
    x, y = generate_dataset(size)

    filename = os.path.join(FLAGS.directory, name + '.tfrecords')
    print('Writing', filename)
    writer = tf.python_io.TFRecordWriter(filename)
    for index in range(size):
        example = tf.train.Example(features=tf.train.Features(feature={
            'x': _float_feature(x[index]),
            'y': _float_feature(y[index])}))
        writer.write(example.SerializeToString())
    writer.close()

def main(unused_argv):
    convert_to(FLAGS.train_size, 'train')
    convert_to(FLAGS.validation_size, 'validation')
    convert_to(FLAGS.test_size, 'test')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--directory',
        type=str,
        default='/tmp/data',
        help='Directory to generate data set and convert to'
    )
    parser.add_argument(
        '--train_size',
        type=float,
        default=1000,
        help='Number of training samples'
    )
    parser.add_argument(
        '--validation_size',
        type=float,
        default=100,
        help='Number of validation samples'
    )
    parser.add_argument(
        '--test_size',
        type=float,
        default=100,
        help='Number of test samples'
    )
    FLAGS, unparsed = parser.parse_known_args()
    tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


import configparser
#from refextract import extract_references_from_file
from bib_parser import GenRequest, recupID, ShowBar, GenDirectories, sort, ShowBar, Mergefinal
from references import dump_extract, Merge_All_List, filtre, Filtre_title, genBib
import argparse
import shutil
import sys
import subprocess
import os as os

#TODO VERBOSE COMMAND

BASE_DIR="./Results/"
pathname = os.path.dirname(sys.argv[0])
SOURCE_DIR=pathname = os.path.abspath(pathname)+"/"

def ConfigParse():
    config = configparser.ConfigParser()
    config.read('SLR.conf')
    requests=[]
    for i in range(len(config['Request'])):
        r="r"+str(i)
        requests.append(config['Request'][r].split(','))
    CSV=config['RQ']['CSV'].split(',')
    TEXTE=config['RQ']['Text'].split(',')
    COMPARISON=int(config['Filter']['comparison'])
    APPEARANCE=int(config['Filter']['appearance'])
    NKWORDS=int(config['Filter']['NKwords'])
    KWORDS=config['Filter']['Kwords'].split(',')
    #print("Requests = "+str(requests))
    #print("CSV = "+str(CSV))
    #print("Texte = "+str(TEXTE))
    #print("COMPARISON = "+str(COMPARISON))
    #print("APPEARANCE = "+str(APPEARANCE))
    #print("KWORDS = "+str(KWORDS))
    return requests,CSV,TEXTE, COMPARISON, APPEARANCE, KWORDS, NKWORDS

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('command')
    arg = parser.parse_args()
    if arg.command not in ["snowball","mergeBib","start","final"]:
        print("Error, unknow command")
        print("Use : 'slrh start' to create a directory and a empty config file")
        print("Use : 'slrh mergeBib' to merge all the bib file in the 'bib' folder, print some stats about the bib files merged, and create folders for each paper to analyse.")
        print("Use : 'slrh snowball' to dump all references from all PDF in all sub folders of the 'Results' folder, create a CSV file with all extracted references, and a bib file with all the filtered references.")
    else :
        if arg.command == "start":
            if os.path.isdir(BASE_DIR) == False:
                os.mkdir(BASE_DIR)
                print("[+] The Results Folder will contain later the folders for the pdf to analyse")
                srcConf=SOURCE_DIR+"empty.conf"
                if os.path.isfile("SLR.conf"):
                    print("[!] Error SLR.conf already created")
                    print("[!] Please complete it")
                else :
                    shutil.copyfile(srcConf,"SLR.conf")
                if os.path.isdir("./bib") :
                    print("[!] Error bib folder already created")
                else :
                    os.mkdir("./bib")
                    print("[+] SLRHelper create a folder 'bib' to put your bibfile in")
                print("[+] NOW COMPLETE THE SLR.conf FILE FOR FURTHER USE !")
            else :
                print("Error Results folder already exist")
        if arg.command == "mergeBib":
            requests,CSV,TEXTE, COMPARISON, APPEARANCE, KWORDS, NKWORDS = ConfigParse()
            r = GenRequest(requests)
            print("[+] The request have been generated")
            path=os.getcwd()
            ida,db = recupID(r,path)
            idt=sort(ida)
            ShowBar(idt)
            GenDirectories(ida,r,idt,path,CSV,TEXTE,db)
            print("[+] Directories created in the Results folder. Please fill them with the rigth pdf paper")
        if arg.command == "snowball":
            requests,CSV,TEXTE, COMPARISON, APPEARANCE, KWORDS,NKWORDS = ConfigParse()
            print("[*] Reference extraction from all pdf (may take severals minutes)")
            cmd=subprocess.run(["/usr/bin/SLRHelper_0.1/ref_extractor.py",BASE_DIR,"ref.csv"])
            # TODO VERIF
            print("[+] extraction complete, results in 'ref.csv'")
            all = dump_extract("ref.csv")
            print("[*] Snowball strating (may take severals minutes)")
            snowball=Merge_All_List(all,COMPARISON)
            #TODO VERIF
            print("[*] Merge completed")
            print("[*] "+str(len(snowball))+" differents references extracted")
            final=filtre(snowball,APPEARANCE)
            final=Filtre_title(final,KWORDS,NKWORDS)
            print("[*] "+str(len(final))+" references add after filter")
            genBib(final,"snowball.bib")
            print("[+] BibFile 'snowball.bib' created with all the references to add")
            print("[*] Please check the file and change the unknow ref")
        if arg.command == "final":
            if not os.path.isfile("final.bib"):
                print("[!] Error, missing final.bib")
            else :
                if not os.path.isfile("snowball.bib"):
                    print("[!] Error, missing snowball.bib")
                else:
                    db=Mergefinal("final.bib","snowball.bib")

if __name__ == '__main__':
    main()