from ..player import BasePlayer
from ..utils import count_min

class Agachao(BasePlayer):
    '''
    This player doesn't want to pass,
    so if there is a number that appears in only one piece,
    this piece will not be used until there's no other choice.
    '''
    def __init__(self, name):
         super().__init__(f"Agachao::{name}")

    def filter(self, valids=None):
        valids = super().filter(valids)

        heads = []
        first_move = True
        amount_played = {}

        for e, *d in self.history:
            if e.name == 'MOVE':
                player, piece, head = d
                if player == self.me:
                    amount_played[piece[0]] = amount_played.get(piece[0], 0) + 1
                    if piece[0] != piece[1]:
                        amount_played[piece[1]] = amount_played.get(piece[1], 0) + 1
                if first_move:
                    heads = list(piece)
                    first_move = False
                else:
                    heads[head] = piece[piece[0] == heads[head]]

        best, data = (-1, -1), []
        for piece, head in valids:
            mn, i = count_min(self, piece)
            value = (mn, amount_played.get(piece[i], 0))
            if value > best:
                best = value
                data.clear()
            if value == best:
                data.append((piece, head))

        return data


import sys
input = lambda: sys.stdin.readline().rstrip("\n\r")
from collections import Counter

def solve():
    n = int(input())
    a = list(map(int, input().split()))

    pre = [a[0]]
    for i in range(1, n):
        pre.append(pre[-1] + a[i])

    zind = []
    for i in range(n):
        if a[i] == 0:
            zind.append(i)
    zind.append(n)

    ans = pre[:zind[0]].count(0)
    for i in range(len(zind) - 1):
        x = sorted(pre[zind[i]: zind[i + 1]])
        ans += max(Counter(x).values())

    print(ans)

for _ in range(int(input())):
    solve()


# 1225 - 암호 생성기
import sys
sys.stdin = open('input.txt')
from collections import deque

T = 10
for t in range(1, T+1) :
    tc = int(input())
    queue = deque(list(map(int, input().split())))  # 큐

    while True :
        for i in range(1, 6) :  # 한 사이클
            if (queue[0] - i) <= 0 :    # i 만큼 뺀 값이 0보다 작거나 같아질 때
                queue[0] = 0    # 0 으로 바꾸고
                queue.append(queue.popleft())   # 뒤로 보내준 후 브레이크
                break
            else :
                queue[0] -= i   # i 만큼 빼준 후
                queue.append(queue.popleft())   # 뒤로 보내기

        if queue[-1] == 0 : # while문 빠져나가기
            break

    print(f'#{t}', *queue)


from django.shortcuts import render

# Create your views here.
from django.http import HttpResponse

from app1.models import Board
from app1.forms import ChessForm

def home(request):

    if(Board.objects.all().count() == 0):
        reset(request)
    if(request.method == 'GET' and 'Reset' in request.GET):
        reset(request)
    column_labels = ['0', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h']
    page_data = {"rows": [], "chess_form": ChessForm}
    if(request.method == 'POST'):
        chess_form = ChessForm(request.POST)
        if(chess_form.is_valid()):
            Source = chess_form.cleaned_data["Source"]
            pieces = Board.objects.filter(Source=Source).values()[0]
            Destination = chess_form.cleaned_data["Destination"]
            Board(Source=Source, Destination="&nbsp").save()
            Board(Source=Destination, Destination=pieces['Destination']).save()
        else:
            page_data["chess_form"] = chess_form
    for row in range(1,9):
        row_data = {}
        for col in range(1,10):#2,10
            id="{}{}".format(column_labels[col-1],9-row)
            try:
                record = Board.objects.get(Source=id)
                row_data[id] = record.Destination
            except Board.DoesNotExist:
                    row_data[id] = "&nbsp"
        page_data.get("rows").append(row_data)

    return render(request, 'app1/home.html', page_data)

def reset(request):
    page_data= {
    "rows": [
        {"08":8, "a8":"&#9820", "b8":"&#9822", "c8":"&#9821", "d8":"&#9819", "e8":"&#9818", "f8":"&#9821", "g8":"&#9822", "h8":"&#9820"},
        {"07":7, "a7":"&#9823", "b7":"&#9823", "c7":"&#9823", "d7":"&#9823", "e7":"&#9823", "f7":"&#9823", "g7":"&#9823", "h7":"&#9823"},
        {"06":6, "a6":"", "b6":"", "c6":"", "d6":"", "e6":"", "f6":"", "g6":"", "h6":""},
        {"05":5, "a5":"", "b5":"", "c5":"", "d5":"", "e5":"", "f5":"", "g5":"", "h5":""},
        {"04":4, "a4":"", "b4":"", "c4":"", "d4":"", "e4":"", "f4":"", "g4":"", "h4":""},
        {"03":3, "a3":"", "b3":"", "c3":"", "d3":"", "e3":"", "f3":"", "g3":"", "h3":""},
        {"02":2, "a2":"&#9817", "b2":"&#9817", "c2":"&#9817", "d2":"&#9817", "e2":"&#9817", "f2":"&#9817", "g2":"&#9817", "h2":"&#9817"},
        {"01":1, "a1":"&#9814", "b1":"&#9816", "c1":"&#9815", "d1":"&#9813", "e1":"&#9812", "f1":"&#9815", "g1":"&#9816", "h1":"&#9814"},
    ]
    }
    Board.objects.all().delete()
    for row in page_data.get("rows"):
        for Source, Destination in row.items():
            Board(Source=Source, Destination=Destination).save()

def history(request):
    return render(request, 'app1/history.html')

def rules(request):
    return render(request, 'app1/rules.html')

def about(request):
    return render(request, 'app1/about.html')


from django.contrib.auth import get_user_model
from django.conf import settings
from django.template.loader import render_to_string

def email_or_mobile(value):
    UserModel = get_user_model()
    if settings.EMAIL_PATTERN.search(value):
        return {UserModel.EMAIL_FIELD: value}
    if settings.MOBILE_PATTERN.search(value):
        return {UserModel.USERNAME_FIELD: value}
    return None

def user_exists(**kwargs):
    model = get_user_model()
    try:
        user = model.objects.get(**kwargs)
    except model.DoesNotExist:
        return False
    return user

def verification_mail(user, token, time_validity):
    # token_message = _("SMAS OTP Verification {token}")
    # token_message = token_message.format(token=token)
    try:
        html_message = render_to_string(
            'mailer/verification.html',
            {'name': user.get_full_name().title(), 'otp': token, 'time_validity': time_validity})
        user.email_user("SMAS OTP Verification", None, "admin@smas.com", html_message=html_message)
    except Exception as ex:
        raise ex.args
    return True


import os
import numpy as np
import torch
from torch.autograd import Variable
from torch.nn import functional as F
from logging import getLogger

from .utils import get_optimizer, clip_grad_norm, get_lambda, reload_model
from .model import get_attr_loss, flip_attributes

logger = getLogger()

class Trainer(object):

    def __init__(self, ae, lat_dis, ptc_dis, clf_dis, data, params):
        """
        Trainer initialization.
        """
        # data / parameters
        self.data = data
        self.params = params

        # modules
        self.ae = ae
        self.lat_dis = lat_dis
        self.ptc_dis = ptc_dis
        self.clf_dis = clf_dis

        # optimizers
        self.ae_optimizer = get_optimizer(ae, params.ae_optimizer)
        logger.info(ae)
        logger.info('%i parameters in the autoencoder. '
                    % sum([p.nelement() for p in ae.parameters()]))
        if params.n_lat_dis:
            logger.info(lat_dis)
            logger.info('%i parameters in the latent discriminator. '
                        % sum([p.nelement() for p in lat_dis.parameters()]))
            self.lat_dis_optimizer = get_optimizer(lat_dis, params.dis_optimizer)
        if params.n_ptc_dis:
            logger.info(ptc_dis)
            logger.info('%i parameters in the patch discriminator. '
                        % sum([p.nelement() for p in ptc_dis.parameters()]))
            self.ptc_dis_optimizer = get_optimizer(ptc_dis, params.dis_optimizer)
        if params.n_clf_dis:
            logger.info(clf_dis)
            logger.info('%i parameters in the classifier discriminator. '
                        % sum([p.nelement() for p in clf_dis.parameters()]))
            self.clf_dis_optimizer = get_optimizer(clf_dis, params.dis_optimizer)

        # reload pretrained models
        if params.ae_reload:
            reload_model(ae, params.ae_reload,
                         ['img_sz', 'img_fm', 'init_fm', 'n_layers', 'n_skip', 'attr', 'n_attr'])
        if params.lat_dis_reload:
            reload_model(lat_dis, params.lat_dis_reload,
                         ['enc_dim', 'attr', 'n_attr'])
        if params.ptc_dis_reload:
            reload_model(ptc_dis, params.ptc_dis_reload,
                         ['img_sz', 'img_fm', 'init_fm', 'max_fm', 'n_patch_dis_layers'])
        if params.clf_dis_reload:
            reload_model(clf_dis, params.clf_dis_reload,
                         ['img_sz', 'img_fm', 'init_fm', 'max_fm', 'hid_dim', 'attr', 'n_attr'])

        # training statistics
        self.stats = {}
        self.stats['rec_costs'] = []
        self.stats['lat_dis_costs'] = []
        self.stats['ptc_dis_costs'] = []
        self.stats['clf_dis_costs'] = []

        # best reconstruction loss / best accuracy
        self.best_loss = 1e12
        self.best_accu = -1e12
        self.params.n_total_iter = 0

    def lat_dis_step(self):
        """
        Train the latent discriminator.
        """
        data = self.data
        params = self.params
        self.ae.eval()
        self.lat_dis.train()
        bs = params.batch_size
        # batch / encode / discriminate
        batch_x, batch_y = data.train_batch(bs)
        enc_outputs = self.ae.encode(Variable(batch_x.data, volatile=True))
        preds = self.lat_dis(Variable(enc_outputs[-1 - params.n_skip].data))
        # loss / optimize
        loss = get_attr_loss(preds, batch_y, False, params)
        self.stats['lat_dis_costs'].append(loss.data[0])
        self.lat_dis_optimizer.zero_grad()
        loss.backward()
        if params.clip_grad_norm:
            clip_grad_norm(self.lat_dis.parameters(), params.clip_grad_norm)
        self.lat_dis_optimizer.step()

    def ptc_dis_step(self):
        """
        Train the patch discriminator.
        """
        data = self.data
        params = self.params
        self.ae.eval()
        self.ptc_dis.train()
        bs = params.batch_size
        # batch / encode / discriminate
        batch_x, batch_y = data.train_batch(bs)
        flipped = flip_attributes(batch_y, params, 'all')
        _, dec_outputs = self.ae(Variable(batch_x.data, volatile=True), flipped)
        real_preds = self.ptc_dis(batch_x)
        fake_preds = self.ptc_dis(Variable(dec_outputs[-1].data))
        y_fake = Variable(torch.FloatTensor(real_preds.size())
                               .fill_(params.smooth_label).cuda())
        # loss / optimize
        loss = F.binary_cross_entropy(real_preds, 1 - y_fake)
        loss += F.binary_cross_entropy(fake_preds, y_fake)
        self.stats['ptc_dis_costs'].append(loss.data[0])
        self.ptc_dis_optimizer.zero_grad()
        loss.backward()
        if params.clip_grad_norm:
            clip_grad_norm(self.ptc_dis.parameters(), params.clip_grad_norm)
        self.ptc_dis_optimizer.step()

    def clf_dis_step(self):
        """
        Train the classifier discriminator.
        """
        data = self.data
        params = self.params
        self.clf_dis.train()
        bs = params.batch_size
        # batch / predict
        batch_x, batch_y = data.train_batch(bs)
        preds = self.clf_dis(batch_x)
        # loss / optimize
        loss = get_attr_loss(preds, batch_y, False, params)
        self.stats['clf_dis_costs'].append(loss.data[0])
        self.clf_dis_optimizer.zero_grad()
        loss.backward()
        if params.clip_grad_norm:
            clip_grad_norm(self.clf_dis.parameters(), params.clip_grad_norm)
        self.clf_dis_optimizer.step()

    def autoencoder_step(self):
        """
        Train the autoencoder with cross-entropy loss.
        Train the encoder with discriminator loss.
        """
        data = self.data
        params = self.params
        self.ae.train()
        if params.n_lat_dis:
            self.lat_dis.eval()
        if params.n_ptc_dis:
            self.ptc_dis.eval()
        if params.n_clf_dis:
            self.clf_dis.eval()
        bs = params.batch_size
        # batch / encode / decode
        batch_x, batch_y = data.train_batch(bs)
        enc_outputs, dec_outputs = self.ae(batch_x, batch_y)
        # autoencoder loss from reconstruction
        loss = params.lambda_ae * ((batch_x - dec_outputs[-1]) ** 2).mean()
        self.stats['rec_costs'].append(loss.data[0])
        # encoder loss from the latent discriminator
        if params.lambda_lat_dis:
            lat_dis_preds = self.lat_dis(enc_outputs[-1 - params.n_skip])
            lat_dis_loss = get_attr_loss(lat_dis_preds, batch_y, True, params)
            loss = loss + get_lambda(params.lambda_lat_dis, params) * lat_dis_loss
        # decoding with random labels
        if params.lambda_ptc_dis + params.lambda_clf_dis > 0:
            flipped = flip_attributes(batch_y, params, 'all')
            dec_outputs_flipped = self.ae.decode(enc_outputs, flipped)
        # autoencoder loss from the patch discriminator
        if params.lambda_ptc_dis:
            ptc_dis_preds = self.ptc_dis(dec_outputs_flipped[-1])
            y_fake = Variable(torch.FloatTensor(ptc_dis_preds.size())
                                   .fill_(params.smooth_label).cuda())
            ptc_dis_loss = F.binary_cross_entropy(ptc_dis_preds, 1 - y_fake)
            loss = loss + get_lambda(params.lambda_ptc_dis, params) * ptc_dis_loss
        # autoencoder loss from the classifier discriminator
        if params.lambda_clf_dis:
            clf_dis_preds = self.clf_dis(dec_outputs_flipped[-1])
            clf_dis_loss = get_attr_loss(clf_dis_preds, flipped, False, params)
            loss = loss + get_lambda(params.lambda_clf_dis, params) * clf_dis_loss
        # check NaN
        if (loss != loss).data.any():
            logger.error("NaN detected")
            exit()
        # optimize
        self.ae_optimizer.zero_grad()
        loss.backward()
        if params.clip_grad_norm:
            clip_grad_norm(self.ae.parameters(), params.clip_grad_norm)
        self.ae_optimizer.step()

    def step(self, n_iter):
        """
        End training iteration / print training statistics.
        """
        # average loss
        if len(self.stats['rec_costs']) >= 25:
            mean_loss = [
                ('Latent discriminator', 'lat_dis_costs'),
                ('Patch discriminator', 'ptc_dis_costs'),
                ('Classifier discriminator', 'clf_dis_costs'),
                ('Reconstruction loss', 'rec_costs'),
            ]
            logger.info(('%06i - ' % n_iter) +
                        ' / '.join(['%s : %.5f' % (a, np.mean(self.stats[b]))
                                    for a, b in mean_loss if len(self.stats[b]) > 0]))
            del self.stats['rec_costs'][:]
            del self.stats['lat_dis_costs'][:]
            del self.stats['ptc_dis_costs'][:]
            del self.stats['clf_dis_costs'][:]

        self.params.n_total_iter += 1

    def save_model(self, name):
        """
        Save the model.
        """
        def save(model, filename):
            path = os.path.join(self.params.dump_path, '%s_%s.pth' % (name, filename))
            logger.info('Saving %s to %s ...' % (filename, path))
            torch.save(model, path)
        save(self.ae, 'ae')
        if self.params.n_lat_dis:
            save(self.lat_dis, 'lat_dis')
        if self.params.n_ptc_dis:
            save(self.ptc_dis, 'ptc_dis')
        if self.params.n_clf_dis:
            save(self.clf_dis, 'clf_dis')

    def save_best_periodic(self, to_log):
        """
        Save the best models / periodically save the models.
        """
        if to_log['ae_loss'] < self.best_loss:
            self.best_loss = to_log['ae_loss']
            logger.info('Best reconstruction loss: %.5f' % self.best_loss)
            self.save_model('best_rec')
        if self.params.eval_clf and np.mean(to_log['clf_accu']) > self.best_accu:
            self.best_accu = np.mean(to_log['clf_accu'])
            logger.info('Best evaluation accuracy: %.5f' % self.best_accu)
            self.save_model('best_accu')
        if to_log['n_epoch'] % 5 == 0 and to_log['n_epoch'] > 0:
            self.save_model('periodic-%i' % to_log['n_epoch'])

def classifier_step(classifier, optimizer, data, params, costs):
    """
    Train the classifier.
    """
    classifier.train()
    bs = params.batch_size

    # batch / classify
    batch_x, batch_y = data.train_batch(bs)
    preds = classifier(batch_x)
    # loss / optimize
    loss = get_attr_loss(preds, batch_y, False, params)
    costs.append(loss.data[0])
    optimizer.zero_grad()
    loss.backward()
    if params.clip_grad_norm:
        clip_grad_norm(classifier.parameters(), params.clip_grad_norm)
    optimizer.step()


import xlrd
import collections

class CalculateJIRAStats(object):

    def __init__(self,type):
        # give JIRA dumps path
        if type == 'XL':
            path = 'C:\\Users\\sballary\\Documents\\Sanjay\\PycharmProjects\\JIRA-Stat-Automation\\dumps_Aug-20-2020.xlsx'
            # give resolved date
            resolved_date = ''
            jira_issue_list = []
            input_work_book = xlrd.open_workbook(path)
            input_work_sheet = input_work_book.sheet_by_index(1)
            self.start_reading(input_work_sheet)
    def start_reading(self,input_work_sheet):
        jira_issue_list = []
        for i in range(1, input_work_sheet.nrows):
            current_row = input_work_sheet.row_values(i, 0, 100)
            jira_issue_object = {'jira_id': int(input_work_sheet.cell_value(i, 2)), 'assigned_to': input_work_sheet.cell_value(i, 13),
                                 'priority': self.calculate_priority(current_row),
                                 'application': self.calculate_application(current_row),
                                 'isEnhancement': self.find_if_enhancement(current_row), 'status': input_work_sheet.cell_value(i, 4)}
            jira_issue_list.append(jira_issue_object)
        self.report(jira_issue_list)
    @staticmethod
    def report(jira_issue_list):
        jira_issues = []
        jira_enhancements = []
        for object in jira_issue_list:
            if object['isEnhancement']:
                jira_enhancements.append(object)
            else:
                jira_issues.append(object)

        all_jira = []
        all_jira.extend(jira_issues)
        all_jira.extend(jira_enhancements)
        print("Total Issue/Enhancements: {}".format(len(jira_issues)+len(jira_enhancements)))
        print(collections.Counter([jira['priority'] for jira in all_jira]))
        print("Issues : {}".format(len(jira_issues)))
        print("Enhancements : {}".format(len(jira_enhancements)))
        populate_stat_issues = PopulateStats('Issues', jira_issues)
        populate_stat_issues.print_report()
        populate_stat_enhancements = PopulateStats('Enhancements', jira_enhancements)
        populate_stat_enhancements.print_report()
        print("--------------------------------------------------------------------------------")
        # print("Issues")
        for ticket in jira_issues:
                # if ticket['application'] in ['UC4', 'uc4'] and ticket['priority']  in ['P1','p1']:
                print("{},{},{},{},{}".format(ticket['application'].upper(),ticket['jira_id'],ticket['priority'],ticket['status'],'Issue'))
        # print("Enhancements")
        for ticket in jira_enhancements:
                # if ticket['application'] in ['UC4', 'uc4'] and ticket['priority']  in ['P1','p1']:
                print("{},{},{},{},{}".format(ticket['application'].upper(), ticket['jira_id'], ticket['priority'],ticket['status'],'Enhancement'))

    @staticmethod
    def calculate_priority(excel_row_array):
        track_priority_occurrence = False
        priority = 'NA'
        for i in range(len(excel_row_array)):
            if excel_row_array[i] == 'P1' or excel_row_array[i] == 'P2' or excel_row_array[i] == 'P3':
                track_priority_occurrence = True
                priority = excel_row_array[i]
                if track_priority_occurrence:
                    return priority
        if not track_priority_occurrence:
            raise Exception('Jira Ticket {} does not have any Priority assigned. For correct report priority'
                            'needs to be assigned'.format(excel_row_array[1]))
        return priority

    @staticmethod
    def calculate_application(excel_row_array):
        track_application_occurrence = False
        application = 'NA'
        import re
        pattern = re.compile("UC\d",re.IGNORECASE)
        for i in range(len(excel_row_array)):
            if pattern.match(excel_row_array[i]):
                track_application_occurrence = True
                application = excel_row_array[i]
                if track_application_occurrence:
                    return application
        if not track_application_occurrence:
            raise Exception('Jira Ticket {} does not have application name against it.'.
                            format(excel_row_array[1]))
        return application

    @staticmethod
    def find_if_enhancement(excel_row_array):
        """
            To find an enhancement we need to look at below occurrence:
            'Issue_Type_Issue' return False
            'Issue_Type_Enhancements' return True
            'Issue_Type_Additional_Requirements' return True
             If nothing is found look for
             Return as Issue i.e. False
        """
        for i in range(len(excel_row_array)):
            if excel_row_array[i] in ['Issue_Type_Enhancements', 'Issue_Type_Additional_Requirements']:
                return True
            elif excel_row_array[i] == 'Issue_Type_Issue':
                return False
        return False

class PopulateStats(object):
    def __init__(self, defect_type, jira_object):
        self.defect_type = defect_type
        self.jira_object = jira_object

    def print_report(self):
        print('####################################################################################')
        print('##############################' + self.defect_type + '###############################')
        print('####################################################################################')
        print('-------------------------------------------------------------------------------------')
        print('---------------------------------------USE CASE 1------------------------------------')
        print('---------Blocked---------Not Started---------In Progress----------UAT----------Closed--------Total')
        self.user_case_specific_stat('UC1')
        print('-------------------------------------------------------------------------------------')
        print('---------------------------------------USE CASE 2------------------------------------')
        print('---------Blocked---------Not Started---------In Progress----------UAT----------Closed--------Total')
        self.user_case_specific_stat('UC2')
        print('-------------------------------------------------------------------------------------')
        print('---------------------------------------USE CASE 3------------------------------------')
        print('---------Blocked---------Not Started---------In Progress----------UAT----------Closed--------Total')
        self.user_case_specific_stat('UC3')
        print('-------------------------------------------------------------------------------------')
        print('---------------------------------------USE CASE 4------------------------------------')
        print('---------Blocked---------Not Started---------In Progress----------UAT----------Closed--------Total')
        self.user_case_specific_stat('UC4')
        print('-------------------------------------------------------------------------------------')
        print('---------------------------------------USE CASE 5------------------------------------')
        print('---------Blocked---------Not Started---------In Progress----------UAT----------Closed--------Total')
        self.user_case_specific_stat('UC5')
        print('-------------------------------------------------------------------------------------')
        print('---------------------------------------USE CASE 6------------------------------------')
        print('---------Blocked---------Not Started---------In Progress----------UAT----------Closed--------Total')
        self.user_case_specific_stat('UC6')

    def user_case_specific_stat(self, use_case):
        use_case_wise_jira_list = []
        for jiraobject in self.jira_object:
            if jiraobject['application'] == use_case:
                use_case_wise_jira_list.append(jiraobject)
        self.process_stat(use_case_wise_jira_list)

    def process_stat(self, usecase_jira_list):
        p1_jira_tickets = []
        p2_jira_tickets = []
        p3_jira_tickets = []
        for jiraobject in usecase_jira_list:
            if jiraobject['priority'] == 'P1':
                p1_jira_tickets.append(jiraobject)
            elif jiraobject['priority'] == 'P2':
                p2_jira_tickets.append(jiraobject)
            elif jiraobject['priority'] == 'P3':
                p3_jira_tickets.append(jiraobject)
            else:
                raise Exception('Stats cannot be generated as Jira Ticket {} does not have any priority'
                                .format(jiraobject['jira_id']))

        blocked = 0
        not_started = 0
        in_progress = 0
        uat = 0
        closed = 0
        for ticket in p1_jira_tickets:
            if ticket['status'] == 'New Requests':
                not_started = not_started + 1
            elif ticket['status'] == 'Done':
                closed = closed + 1
            elif ticket['status'] == 'Blocked':
                blocked = blocked + 1
            elif ticket['status'] == 'UAT':
                uat = uat + 1
            else:
                in_progress = in_progress + 1
        total = blocked + not_started + in_progress + uat + closed
        print('P1          ' + str(blocked) + '                    ' + str(not_started) + '                  '
              + str(in_progress) + '            ' + str(uat) + '             ' + str(closed) + '             '
              + str(total))
        blocked = 0
        not_started = 0
        in_progress = 0
        uat = 0
        closed = 0
        for ticket in p2_jira_tickets:
            if ticket['status'] == 'New Requests' or ticket['status'] == 'Backlog':
                not_started = not_started + 1
            elif ticket['status'] == 'Done':
                closed = closed + 1
            elif ticket['status'] == 'Blocked':
                blocked = blocked + 1
            elif ticket['status'] == 'UAT':
                uat = uat + 1
            else:
                in_progress = in_progress + 1
        total = blocked + not_started + in_progress + uat + closed
        print('P2          ' + str(blocked) + '                    ' + str(not_started) + '                  '
              + str(in_progress) + '            ' + str(uat) + '             ' + str(closed) + '             '
              + str(total))
        blocked = 0
        not_started = 0
        in_progress = 0
        uat = 0
        closed = 0

        for ticket in p3_jira_tickets:
            if ticket['status'] == 'New Requests' or ticket['status'] == 'Backlog':
                not_started = not_started + 1
            elif ticket['status'] == 'Done':
                closed = closed + 1
            elif ticket['status'] == 'Blocked':
                blocked = blocked + 1
            elif ticket['status'] == 'UAT':
                uat = uat + 1
            else:
                in_progress = in_progress + 1
        total = blocked + not_started + in_progress + uat + closed
        print('P3          ' + str(blocked) + '                    ' + str(not_started) + '                  '
              + str(in_progress) + '            ' + str(uat) + '             ' + str(closed) + '             '
              + str(total))

if __name__ == '__main__':
    CalculateJIRAStats('XL')


import re

from docx import Document

from Helpers import to_float, without_whitespace, to_float_or, without_lines

class ParseMachines:
    def __init__(self, model):
        self.model = model
        self.last_id = None
        pass

    def run(self, doc: Document, collection_id: int, id_prefix: str, collection_type: int):
        self.doc = doc
        self.collection_id = collection_id
        self.id_prefix = id_prefix
        self.collection_type = collection_type
        for table in doc.tables:
            self.parse_caption(table)

    def parse_caption(self, table):
        rows = table.rows
        continue_n = 0
        for row in range(0, len(rows)):
            if continue_n > 0:
                continue_n -= 1
                continue
            try:
                cells = rows[row].cells
            except Exception as e:
                continue
            self.model.commit()
            for cell in range(0, len(cells)):
                cells = rows[row].cells
                text = cells[cell].text
                text_all = ' '
                for i in cells:
                    text_all += i.text
                text_all = re.sub(r'\n', '', text_all)
                text_all = re.sub(r'\t', '', text_all)
                if self.check_kniga(text_all):
                    self.last_kniga_id = self.model.insert_caption(self.collection_id, None, text)
                    break
                elif self.check_razdel(text_all):
                    self.last_razdel_id = self.model.insert_caption(self.collection_id, self.last_kniga_id, text)
                    break
                elif self.check_gruppa(text_all):
                    if cells[cell].text:
                        self.last_table_id = self.model.insert_caption(self.collection_id, self.last_razdel_id, cells[cell].text)
                    else:
                        self.last_table_id = self.model.insert_caption(self.collection_id, self.last_razdel_id,
                                                                       cells[cell + 1].text)
                    continue_n = row - self.parse_costs(table, row + 1)

                    cells = rows[row].cells
                    break
                else:
                    break

    def parse_costs(self, table, row_i):
        addon_string = ''
        rows = table.rows
        continue_n = 0
        codes = None
        for row in range(row_i, len(rows)):
            if continue_n > 0:
                continue_n -= 1
                continue
            cells = rows[row].cells
            for cell in range(0, len(cells)):
                cell_text = rows[row].cells[cell].text
                if self.check_kniga(cell_text):
                    return row - 1
                if self.check_razdel(cell_text):
                    return row - 1
                if self.check_gruppa(cell_text):
                    return row - 1

            if len(rows[row].cells) == 5:
                if cells[1].text == cells[2].text:
                    addon_string = without_lines(cells[1].text)
                    continue
                id = without_lines(without_whitespace(cells[0].text))
                name = addon_string + ' ' + without_lines(cells[1].text)
                unit = without_lines(without_whitespace(cells[2].text))
                price = to_float(cells[3].text)
                price_driver = to_float(cells[4].text)

                self.model.insert_machine(id, name, unit, price, price_driver, self.last_table_id)

                continue
            if len(rows[row].cells) > 6:
                print('sghafolksfjlhlgaf;gdsjhfj               {}'.format(len(rows[row].cells)))
                pass
        return row

    def check_kniga(self, row_str: str):
        row_str = re.sub(r'\n', '', row_str)
        row_str = re.sub(r'\t', '', row_str)
        result = re.search(r'(Книга(.*?)$)', row_str, re.IGNORECASE)
        if result:
            return result.group(1)
        return None

    def check_razdel(self, row_str: str):
        result = re.search(r'(Раздел(.*?)$)', row_str, re.IGNORECASE)
        if result:
            return result.group(1)
        return None

    def check_gruppa(self, row_str: str):
        result = re.search(r'((Группа(.*?))$)', row_str, re.IGNORECASE)
        if result:
            return result.group(1)
        return None


import logging
import json

from cpc.network.server_to_server_message import ServerToServerMessage
from cpc.network.com.input import Input
from cpc.network.server_request import ServerRequest
from cpc.util import json_serializer

log=logging.getLogger(__name__)

class ServerMessage(ServerToServerMessage):
    """Server-to-server messages.
       These messages can be sent to any node in the network
    """
    def pingServer(self,serverId):
        cmdstring='ping'
        fields = []
        input = Input('cmd', cmdstring)
        fields.append(input)
        fields.append(Input('version', "1"))
        headers = dict()
        if serverId!= None:
            headers['server-id'] = serverId
        msg = ServerRequest.prepareRequest(fields,[],headers)
        response= self.putRequest(msg)
        return response

    def workerReadyForwardedRequest(self, workerID, archdata, topology,
                                    originatingServer, heartbeatInterval,
                                    originatingClient=None):
        cmdstring='worker-ready-forward'
        fields = []
        fields.append(Input('cmd', cmdstring))
        fields.append(Input('version', "1"))
        fields.append(Input('worker', archdata))
        fields.append(Input('worker-id', workerID))
        fields.append(Input('heartbeat-interval', str(heartbeatInterval)))
        topologyInput = Input('topology',
            # a json structure that needs to be dumped
            json.dumps(topology,
                default = json_serializer.toJson,
                indent=4))
        fields.append(topologyInput)
        headers = dict()
        headers['originating-server-id'] = originatingServer
        if originatingClient is not None:
            headers['originating-client'] = originatingClient
        response= self.putRequest(ServerRequest.prepareRequest(fields, [],
            headers))
        return response

    def commandFinishedForwardedRequest(self, cmdID, workerServer,
                                        projectServer, returncode,
                                        cputime, haveData):
        """A server-to-sever request doing command-finished. Used in
            forwarding non-local command-finished requests."""
        cmdstring='command-finished-forward'
        fields = []
        input = Input('cmd', cmdstring)
        fields.append(input)
        fields.append(Input('cmd_id', cmdID))
        fields.append(Input('version', "1"))
        fields.append(Input('worker_server', workerServer))
        fields.append(Input('project_server', projectServer))
        if returncode is not None:
            fields.append(Input('return_code', returncode))
        fields.append(Input('used_cpu_time', cputime))
        if haveData:
            fields.append(Input('run_data', 1))

        #the files are not sent in this message, instead they are pulled
        # from the project server upon receiving this message (for now)
        files = []
        headers = dict()
        #headers['end-node'] = self.host
        #headers['end-node-port'] = self.port
        #self.connect()
        #log.debug("forwarding command finished to %s"%self.endNode)
        response= self.putRequest(ServerRequest.prepareRequest(fields, files,
            headers))
        return response

    def heartbeatForwardedRequest(self, workerID, workerDir, workerServer,
                                  iteration, heartbeatItemsXML):
        """A server-to-sever request doing heartbeat signal. Used in
           forwarding non-local heartbeat requests."""
        cmdstring='heartbeat-forward'
        fields = []
        fields.append(Input('cmd', cmdstring))
        fields.append(Input('version', "1"))
        fields.append(Input('worker_id', workerID))
        fields.append(Input('worker_dir', workerDir))
        fields.append(Input('iteration', iteration))
        fields.append(Input('worker_server', workerServer))
        fields.append(Input('heartbeat_items', heartbeatItemsXML))
        files = []
        headers = dict()
        #headers['end-node'] = self.host
        #headers['end-node-port'] = self.port
        #self.connect()
        #log.debug("forwarding command finished to %s"%self.endNode)
        response= self.putRequest(ServerRequest.prepareRequest(fields, files,
            headers))
        return response

    def deadWorkerFetchRequest(self, workerDir, runDir):
        """A server-to-sever request for fetching a set of run directories
           from a dead worker's output."""
        cmdstring='dead-worker-fetch'
        fields = []
        fields.append(Input('cmd', cmdstring))
        fields.append(Input('version', "1"))
        fields.append(Input('worker_dir', workerDir))
        fields.append(Input('run_dir', runDir))
        files = []
        headers = dict()
        #self.connect()
        response= self.putRequest(ServerRequest.prepareRequest(fields, files,
            headers))
        return response

    def pullAssetRequest(self, cmdID, assetType):
        cmdstring='pull-asset'
        fields = []
        fields.append(Input('cmd', cmdstring))
        fields.append(Input('version', "1"))
        fields.append(Input('cmd_id', cmdID))
        fields.append(Input('asset_type', assetType))
        headers = dict()
        headers['end-node'] = self.host
        headers['end-node-port'] = self.port

        #self.connect()
        response= self.putRequest(ServerRequest.prepareRequest(fields, [],
            headers))
        return response

    def clearAssetRequest(self, cmdID):
        cmdstring='clear-asset'
        fields = []
        fields.append(Input('cmd', cmdstring))
        fields.append(Input('version', "1"))
        fields.append(Input('cmd_id', cmdID))
        headers = dict()
        headers['end-node'] = self.host
        headers['end-node-port'] = self.port
        #self.connect()
        response= self.putRequest(ServerRequest.prepareRequest(fields, [],
            headers))
        return response


# -*- coding: utf-8 -*-
"""

data engineering functions

"""

import sqlite3
import sys

# -*- coding: utf-8 -*-
import numpy as np
import pandas as pd
import pylab as plt
from scipy.sparse import coo_matrix
from sklearn.model_selection import train_test_split

from auxiliary import *

def list_channels_to_df(portfolio, channel_types=['web', 'email', 'mobile', 'social'], debug=False):
    """
    extracts information from the dictionary located in the channels column of the portfolio database to separate columns

    :param portfolio:
    :param channel_types:
    :param debug:
    :return:
    """
    #

    # initialize
    for channel in channel_types:
        portfolio[channel] = False

    # assign values
    for ind in portfolio.index:
        for channel in channel_types:
            if debug:
                print(portfolio.loc[ind, 'channels'])
            if channel in portfolio.loc[ind, 'channels']:
                portfolio.loc[ind, channel] = True
                if debug:
                    print(channel, portfolio[channel])
    return (portfolio)

def load_portfolio(debug=False, location='colab'):
    """
    loads and processes the portfolio database

    :param debug: print debug information
    :param location: portfolio.json location (Google Colab or local)
    :return:
    """

    if location == 'colab':
        portfolio = pd.read_json(r'/content/gdrive/My Drive/UD/Sbux/data/portfolio.json', orient='records', lines=True)
    else:
        portfolio = pd.read_json(r'portfolio.json', orient='records', lines=True)

    # extracts channel (web, email, ...) information
    portfolio = list_channels_to_df(portfolio);
    portfolio = portfolio.drop(columns=['channels'])
    if debug:
        print('portfolio.shape:', portfolio.shape)

    assert portfolio.shape[0] == 10, 'wrong portfolio size'
    n_offers = np.unique(portfolio['id']).shape[0]
    assert n_offers == 10, 'wrong portfolio size'
    portfolio.rename(columns={'id': 'offer', 'reward': 'offer_reward'}, inplace=True)
    portfolio['duration_hours'] = 24. * portfolio['duration']

    return (portfolio)

def load_transcript(debug=False, location='colab'):
    """
    loads and processes the transcript database

    :param debug: print debug information
    :param location: portfolio.json location (Google Colab or local)
    :return:
    """

    if location == 'colab':
        transcript = pd.read_json(r'/content/gdrive/My Drive/UD/Sbux/data/transcript.json', orient='records',
                                  lines=True)
    else:
        transcript = pd.read_json(r'transcript.json', orient='records', lines=True)

    # extract offers + transactions
    if debug:
        print('transcript.shape:', transcript.shape)

    # extract if/when the offer was viewed etc
    values = (pd.DataFrame(transcript['value'].values.tolist()))  # .stack()
    values.columns = ['transaction', 'offer', 'offer viewed', 'reward']

    transcript = pd.concat([transcript, values], axis=1)
    transcript['offer'] = transcript['offer'].fillna(transcript['offer viewed'])
    transcript = transcript.drop(columns=['offer viewed'])

    if debug:
        print('transcript.shape:', transcript.shape)

    n_persons_transcript = np.unique(transcript['person']).shape[0]

    if debug:
        print('n_persons_transcript:', n_persons_transcript)

    # separate into two different DF to handle better time splits
    t_offers = transcript[transcript.event.str.contains('offer')]
    t_purchases = transcript[transcript.event.str.contains('transaction')]

    return (t_offers, t_purchases)

def load_profile(debug=False, location='colab'):
    """
    loads and processes the customer profile db

    :param debug: print debug information
    :param location: portfolio.json location (Google Colab or local)
    :return:
    """

    if location == 'colab':
        profile = pd.read_json(r'/content/gdrive/My Drive/UD/Sbux/data/profile.json', orient='records', lines=True)
    else:
        profile = pd.read_json(r'profile.json', orient='records', lines=True)

    profile['became_member_on'] = pd.to_datetime(profile['became_member_on'].astype(str), format='%Y%m%d').astype(
        "datetime64")
    profile['joined_year'] = profile['became_member_on'].dt.year
    profile['joined_month'] = profile['became_member_on'].dt.month
    profile['joined_week'] = profile['became_member_on'].dt.week

    profile['gender_num'] = profile.gender.astype("category", categories=['F', 'M', 'O']).cat.codes

    time_start = profile.became_member_on.min()
    time_end = profile.became_member_on.max()
    time_delta = time_end - time_start

    if debug:
        print()
        print('time span in profile DB:', time_start, time_end)
        print('time_delta in profile DB:', time_delta.days, ' days ~ %d years' % np.round(time_delta.days / 365))

    profile['user_time'] = (time_end - profile['became_member_on']).dt.days
    profile = profile.rename(columns={'id': 'person'})

    return (profile)

def create_person_offer_table(transcript_offers, transcript_transactions, portfolio, add_missing=False, debug=False):
    """
    combines the transcript_offers, transcript_transactions, portfolio DBs into a single combined one that can be
    indexed by (person, offer, time)

    :param transcript_offers:
    :param transcript_transactions:
    :param portfolio:
    :param add_missing: adds back the persons (6) that did not receive an offer in the timeframe of transcript_offers
    :param debug:
    :return: s
    """
    # generates a main pivot table with customer and offer information (received, viewed, completed)

    # time of offer received / viewed / completed / reward
    combined_pot = extract_offer_times(transcript_offers, portfolio, debug=True);

    # adding the persons that don't receive any offer
    missing_persons = set(transcript_transactions.person) - set(combined_pot.person)
    print('persons with no offers received in timeframe: ', len(missing_persons))

    if add_missing:
        # adding back the persons that did not receive an offer in the timeframe of transcript_offers

        tmp1 = np.array(list(missing_persons))
        tmp = np.vstack((tmp1, np.empty(tmp1.shape) * np.nan)).T

        piv_missing = pd.DataFrame(tmp, columns=['person', 'offer'])
        # display(piv_missing)

        combined = pd.concat((combined_pot, piv_missing), sort=False)
    else:
        combined = combined_pot

    return combined

def add_offers_df(combined, portfolio, debug=False):
    """
    adds the offer details to the main pivot table

    :param combined:
    :param portfolio: portfolio of offers
    :param debug:
    :return:
    """

    #

    # merge the offers on (person, offer)
    if debug:
        print('combined.shape:', combined.shape)
    combined = combined.merge(portfolio[['difficulty', 'duration', 'offer', 'offer_type', 'web',
                                         'email', 'mobile', 'social', 'offer_reward', 'duration_hours']], on='offer',
                              how='left')
    combined.rename(columns={'offer completed': 'offer_completed', 'offer received': 'offer_received',
                             'offer viewed': 'offer_viewed'}, inplace=True)

    combined['offer_end'] = combined['offer_received'] + combined['duration_hours']
    combined['duration_viewed'] = combined['offer_end'] - combined['offer_viewed']
    combined['duration_viewed'] = combined['duration_viewed'].fillna(0)

    combined['duration_effective'] = combined[['duration_viewed', 'duration_hours']].min(axis=1, skipna=False)

    combined['viewed'] = np.invert(combined['offer_viewed'].isnull())
    combined['rewarded'] = np.invert(combined['reward'].isnull())
    combined['received'] = np.invert(combined['offer_received'].isnull())

    combined['comp_not_viewed'] = ((-combined['viewed']) & combined['rewarded'])
    # if an offer was viewed after being completed it also should not be rewarded
    combined['comp_not_viewed'] = combined.offer_viewed > combined.offer_completed

    combined['completed'] = combined['rewarded'] & (-combined['comp_not_viewed'])

    if debug:
        print('combined.shape:', combined.shape)

    return (combined)

def extract_offer_times(transcript_offers, portfolio, debug=True, ov_same_keep_earliest_viewed=True):
    """
    extracts the offer received, viewed, completed information from the transcript dataframe

    :param transcript_offers:
    :param portfolio:
    :param debug:
    :param ov_same_keep_earliest_viewed:
    :return: combined database by person, offer --> times (received, viewed, completed,...)
    """

    if debug:
        events_type = transcript_offers.event.value_counts()
        print(events_type)

    er = transcript_offers.loc[transcript_offers.event == 'offer received', ['person', 'offer', 'time']].rename(
        columns={'time': 'offer_received'})
    ec = transcript_offers.loc[
        transcript_offers.event == 'offer completed', ['person', 'offer', 'time', 'reward']].rename(
        columns={'time': 'offer_completed'})
    ev = transcript_offers.loc[transcript_offers.event == 'offer viewed', ['person', 'offer', 'time']].rename(
        columns={'time': 'offer_viewed'})

    if debug:
        print('er,ec,ev')
        print(er.shape, ec.shape, ev.shape)

    er = er.merge(portfolio[['offer', 'duration_hours']], on='offer')
    er['offer_end'] = er['offer_received'] + er['duration_hours'] - 1

    er.index.name = 'idx'
    er = er.reset_index()

    if debug:
        print('er ', er.shape)
    conn = sqlite3.connect(':memory:')

    er.to_sql('er', conn, index=False, if_exists='replace')  # ,index_label='idx')
    ec.to_sql('ec', conn, index=False, if_exists='replace')
    ev.to_sql('ev', conn, index=False, if_exists='replace')

    # SQL query for speed
    qry = '''
      SELECT
          er.person, er.offer, er.offer_received, ev.offer_viewed, ec.offer_completed, ec.reward, er.idx, er.offer_end
      FROM
          er
          left join  ec on
          er.person = ec.person and er.offer = ec.offer and ec.offer_completed <= er.offer_end and ec.offer_completed >= er.offer_received

          left join  ev on
          er.person = ev.person and er.offer = ev.offer and ev.offer_viewed <= er.offer_end and ev.offer_viewed >= er.offer_received

      '''
    combined = pd.read_sql_query(qry, conn)  # .drop(columns=['idx'])

    # remove duplicates from SQL
    combined.drop_duplicates(subset=['idx'], inplace=True)
    combined = combined.drop(columns=['idx'])

    #   # dealing with overlapping offers of the same type

    #   if ov_same_keep_earliest_viewed != 'skip':
    #     piv['distance']=piv['offer_viewed']-piv['offer_received']

    #     ordered=piv.loc[:,['person','offer','offer_viewed','distance']].sort_values(by='distance',ascending=True)
    #     ordered=ordered.drop(columns=['distance'])

    #     if ov_same_keep_earliest_viewed:
    #       keep='first'
    #     else:
    #       keep='last'
    #     duplicated=ordered.loc[ordered.duplicated(keep=keep)].index

    #     piv=piv.drop(index=duplicated)
    #     piv=piv.drop(columns=['distance'])

    #     piv=piv.reindex()

    #   if debug:
    #     print(piv.shape)
    return (combined)

def extract_transactions(combined, transcript, add_transactions=False, skip_overlap=True, debug=False):
    """
    extract_transactions
    separates purchases from transcript_offers in those during/outside offers

    :param combined:
    :param transcript:
    :param add_transactions:
    :param skip_overlap: if offers are overlapping the transaction is assigned to the first offer received
    :return:
    """

    # filter the transactions for purchases
    trans = transcript[transcript['event'] == 'transaction'][['person', 'time', 'transaction']].sort_values(
        by=['person', 'time'])
    trans.rename(columns={'transaction': 'payments'}, inplace=True)
    trans.reset_index(inplace=True)

    if debug:
        print('trans.shape:', trans.shape)

    # extracts the transactions during an offer period, uses group by to avoid repeated rows in the SQL query
    conn = sqlite3.connect(':memory:')

    combined.to_sql('piv', conn, index=False, if_exists='replace')

    # (tr_id) index is the real index in the transcript DataFrame, idx is the reindexed index
    trans = trans.rename(columns={'index': 'tr_id'})

    # to check for duplicate lines after SQL query
    trans.index.name = 'idx'
    trans.reset_index().to_sql('trans', conn, index=False, if_exists='replace', index_label='idx')

    if debug:
        print(trans.columns)

    # https://stackoverflow.com/questions/30627968/merge-pandas-dataframes-where-one-value-is-between-two-others
    if skip_overlap:
        qry = '''
        SELECT
            trans.person, trans.time, trans.payments, trans.tr_id, piv.offer, idx , piv.offer_viewed
        FROM
            trans inner join  piv on
            trans.person = piv.person and trans.time between offer_viewed and offer_end+1
        WHERE
            viewed = 1
        GROUP BY
            idx
        '''
    else:
        qry = '''
    SELECT
        trans.person, trans.time, trans.payments, trans.tr_id, piv.offer, idx , piv.offer_viewed
    FROM
        trans inner join  piv on
        trans.person = piv.person and trans.time between offer_viewed and offer_end+1
    WHERE
        viewed = 1
    '''

    transactions_during_offer = pd.read_sql_query(qry, conn)

    # check for duplicates and remove them. there should be none
    print('transactions_during_offer.shape:', transactions_during_offer.shape)

    if skip_overlap:
        print('skip_overlap: True')

        print(transactions_during_offer.columns)

        # each transaction is attributed to the first offer received
        #transactions_during_offer.set_index('idx', inplace=True)
        #transactions_during_offer = transactions_during_offer.groupby(by='idx').first()
        transactions_during_offer = transactions_during_offer.drop_duplicates(subset=['idx'])
        transactions_during_offer['overlaps'] = 0

        print(transactions_during_offer.columns)

    else:
        print('skip_overlap: False')
        print(transactions_during_offer.columns)

        # fix the issue that some transactions are counted multiple times on the same offer
        transactions_during_offer = transactions_during_offer.drop_duplicates(
            subset=['person', 'time', 'payments', 'tr_id', 'offer'])
        # however this does not resolve the issue that the same transactions may be assigned to two different offers.

        # each transaction is attributed to all the valid offers
        vc = transactions_during_offer['tr_id'].value_counts().rename('overlaps')
        transactions_during_offer = transactions_during_offer.merge(vc, left_on='tr_id', right_index=True)
        # overlaps = value_counts -1
        transactions_during_offer['overlaps'] -= 1

        print(transactions_during_offer.columns)

    if debug:
        print('combined.shape:', combined.shape)
        print('trans.shape:', trans.shape)

        print('transactions_during_offer.shape:', transactions_during_offer.shape)
        print('transactions_during_offer index unique :', len(set(transactions_during_offer.index)))
    n_trans_do = transactions_during_offer.shape[0]

    if add_transactions:
        combined = combined.merge(transactions_during_offer, on=['person', 'offer'], how='left')
        print('piv.shape:', combined.shape)

    #  get the remaining transactions
    ind = trans.index.difference(transactions_during_offer.index)
    transactions_outside_offer = trans.loc[ind, :]

    # check validity
    if debug:
        print()
        #   print('trans.shape:',trans.shape)
        print('transactions_outside_offer.shape:', transactions_outside_offer.shape)
        print('transactions_during_offer.shape:', transactions_during_offer.shape)
        print('total transactions: ', transactions_outside_offer.shape[0] + transactions_during_offer.shape[0])
        print('difference: ', transactions_outside_offer.shape[0] + transactions_during_offer.shape[0] - trans.shape[0])

    assert trans.shape[0] == transactions_outside_offer.shape[0] + transactions_during_offer.shape[
        0], "transactions not matching "

    # trans is the full transaction database
    return (combined, trans, transactions_during_offer, transactions_outside_offer)

def assign_overlapping_purchases(transactions_during_offer, assign_to='first'):
    """
    assign_overlapping_purchases
    resolve case of overlapping offers by the criteria assign_to

    :param transactions_during_offer:
    :param assign_to: first, last, ignore
    :return:
    """

    # resolve case of overlapping offers by
    conn = sqlite3.connect(':memory:')

    # find purchases with overlapping offers
    tdo_ov = transactions_during_offer.loc[(transactions_during_offer.overlaps > 0)]

    tdo_ov.to_sql('tdo', conn, index=False, if_exists='replace')

    # assign to first viewed offer
    if assign_to == 'first':
        qry = '''
    SELECT
         person,  time,  payments, tr_id,  offer,   idx, MIN(offer_viewed) AS offer_viewed
    FROM
        tdo
    GROUP BY
        tr_id
    '''

    # assign to last viewed offer
    elif assign_to == 'last':
        qry = '''
    SELECT
         person,  time,  payments, tr_id,  offer,   idx, MAX(offer_viewed) AS offer_viewed
    FROM
        tdo
    GROUP BY
        tr_id
    '''
    elif assign_to == 'ignore':
        return transactions_during_offer
    else:
        print('method not selected!')
        return (None)

    tdo_nov = pd.read_sql_query(qry, conn)
    if len(tdo_nov) > 0:
        tdo_nov['overlaps'] = 0
    else:
        print('tdo_nov:', tdo_nov)

    tdo_others = transactions_during_offer.loc[(transactions_during_offer.overlaps == 0)]

    return pd.concat([tdo_nov, tdo_others], ignore_index=True)

def add_stats_by_person(combined, transactions_outside_offer, transactions_during_offer, calc_net_offer_time=False):
    """
    add_stats_by_person

    :param combined:
    :param transactions_outside_offer:
    :param transactions_during_offer:
    :param calc_net_offer_time:
    :return:
    """

    # customers who made purchases but in times when no offer is received  (total $ sum and number of purchases)
    total_purchase_outside = transactions_outside_offer.groupby(by=['person'])['payments'].agg('sum').to_frame()
    total_purchase_outside.rename(columns={'payments': 'Tpay_out'}, inplace=True)

    total_purchase_outside_num = transactions_outside_offer.groupby(by=['person'])['payments'].agg('count').to_frame()
    total_purchase_outside_num.rename(columns={'payments': 'Npay_out'}, inplace=True)

    t_offers = transactions_during_offer.groupby(by=['person'])['payments'].agg(['sum', 'count', 'max', 'min'])
    t_offers.rename(columns={'sum': 'Tpay_offers_tot', 'count': 'Npay_offers_tot', 'max': 'Maxpay_offers_tot',
                             'min': 'Minpay_offers_tot'}, inplace=True)
    t_offers = t_offers.fillna(0)

    t_rewards = combined.groupby(by=['person'])['reward'].agg(['sum', 'count'])
    t_rewards.rename(columns={'sum': 'Trewards_tot', 'count': 'Nrewards_tot'}, inplace=True)
    t_rewards = t_rewards.fillna(0)

    ## SPECIFIC FOR A CUSTOMER
    # total offers time (gross, ie counting multiple times overlapping time intervals only once)
    offer_times = combined.groupby(by='person')['duration_effective'].agg('sum').to_frame()
    offer_times.rename(columns={'duration_effective': 'tot_gross_offer_time'}, inplace=True)

    # total offers time (gross, ie counting multiple times overlapping offer intervals)
    offer_timesN = combined.groupby(by=['person', 'offer'])[['offer_viewed', 'offer_end']].agg(np.mean)
    offer_timesN = offer_timesN[pd.notnull(offer_timesN['offer_viewed'])]

    if calc_net_offer_time:
        tot_net_offers_time = overlapping_offer_intevals(combined, final_time=744, return_person_occupacy=False)

    # completed offers
    p_offers_completed = combined.groupby(by='person')['completed', 'comp_not_viewed', 'received','viewed'].agg('sum')
    p_offers_completed['p_c_r_ratio'] = p_offers_completed['completed'] / p_offers_completed['received']
    p_offers_completed['p_c_v_ratio'] = p_offers_completed['completed'] / p_offers_completed['viewed']

    p_offers_completed['p_cnv_r_ratio'] = p_offers_completed['comp_not_viewed'] / p_offers_completed['received']
    p_offers_completed=p_offers_completed.fillna(0)

    print('add_stats_by_person:')
    print(combined.shape)
    #   print('transactions_during_offer',transactions_during_offer.shape)
    # add overlaps
    combined_with_person_stats = combined.merge(total_purchase_outside, on=['person'], how='left')

    combined_with_person_stats = combined_with_person_stats.merge(total_purchase_outside_num, on=['person'], how='left')
    combined_with_person_stats = combined_with_person_stats.merge(p_offers_completed[['p_c_r_ratio', 'p_cnv_r_ratio','p_c_v_ratio']],
                                                                  on=['person'], how='left')

    overlaps = transactions_during_offer[['person', 'offer', 'offer_viewed', 'overlaps']].groupby(
        by=['person', 'offer', 'offer_viewed']).agg('sum')
    combined_with_person_stats = combined_with_person_stats.merge(overlaps, on=['person', 'offer', 'offer_viewed'],
                                                                  how='left')
    #   piv2=piv2.merge( transactions_during_offer[['person', 'offer',  'offer_viewed', 'overlaps']].drop_duplicates() ,on=['person','offer','offer_viewed'],how='left')

    print(combined_with_person_stats.shape)

    combined_with_person_stats = combined_with_person_stats.merge(t_offers, on=['person'], how='left')
    combined_with_person_stats = combined_with_person_stats.merge(t_rewards, on=['person'], how='left')
    #   piv2=piv2.merge(t_offers_num,on=['person'],how='left')
    print(combined_with_person_stats.shape)

    combined_with_person_stats = combined_with_person_stats.merge(offer_times, on='person', how='left')
    print(combined_with_person_stats.shape)

    # piv2[['tot_net_offers_time'],'tot_net_offers_time','reward','Tpay_offer','Tpay_out','Npay_out','Npay_offer'
    if calc_net_offer_time:
        combined_with_person_stats = combined_with_person_stats.merge(tot_net_offers_time.reset_index(), on=['person'],
                                                                      how='left')
        combined_with_person_stats['tot_net_offers_time'] = combined_with_person_stats['tot_net_offers_time'].fillna(0)
        combined_with_person_stats['tot_not_offers_time'] = 744. - combined_with_person_stats['tot_net_offers_time']

        combined_with_person_stats['Avg_pay_offers'] = combined_with_person_stats['Tpay_offers_tot'] / \
                                                       combined_with_person_stats['tot_net_offers_time']
        combined_with_person_stats['Net_pay_offers'] = combined_with_person_stats['Tpay_offers_tot'] - \
                                                       combined_with_person_stats['Trewards_tot']
        combined_with_person_stats['Avg_net_pay_offers'] = (combined_with_person_stats['Tpay_offers_tot'] -
                                                            combined_with_person_stats['Trewards_tot']) / \
                                                           combined_with_person_stats['tot_net_offers_time']

        combined_with_person_stats['Avg_pay_outside'] = combined_with_person_stats['Tpay_out'] / \
                                                        combined_with_person_stats['tot_not_offers_time']
        combined_with_person_stats['Avg_pay_offers'] = combined_with_person_stats['Avg_pay_offers'].fillna(0)
        combined_with_person_stats['Avg_pay_outside'] = combined_with_person_stats['Avg_pay_outside'].fillna(0)

        combined_with_person_stats['Avg_D_O'] = combined_with_person_stats['Avg_pay_offer'] - \
                                                combined_with_person_stats['Avg_pay_outside']
        combined_with_person_stats['Avg_D_OS'] = combined_with_person_stats['Avg_pay_offers'] - \
                                                 combined_with_person_stats['Avg_pay_outside']
        combined_with_person_stats['Avg_D_nOS'] = combined_with_person_stats['Avg_net_pay_offers'] - \
                                                 combined_with_person_stats['Avg_pay_outside']

        combined_with_person_stats['Avg_D_nOS']=combined_with_person_stats['Avg_D_nOS'].fillna(0)

    combined_with_person_stats['reward'] = combined_with_person_stats['reward'].fillna(0)
    combined_with_person_stats['Tpay_offers_tot'] = combined_with_person_stats['Tpay_offers_tot'].fillna(0)
    combined_with_person_stats['Tpay_out'] = combined_with_person_stats['Tpay_out'].fillna(0)
    combined_with_person_stats['Npay_out'] = combined_with_person_stats['Npay_out'].fillna(0)
    combined_with_person_stats['Npay_offers_tot'] = combined_with_person_stats['Npay_offers_tot'].fillna(0)
    combined_with_person_stats['Avg_net_pay_offers'] = combined_with_person_stats['Avg_net_pay_offers'].fillna(0)

    combined_with_person_stats['Maxpay_offer'] = combined_with_person_stats['Maxpay_offer'].fillna(0)
    combined_with_person_stats['Minpay_offer'] = combined_with_person_stats['Minpay_offer'].fillna(0)
    combined_with_person_stats['Netpay_offer'] = combined_with_person_stats['Netpay_offer'].fillna(0)
    combined_with_person_stats['Maxpay_offers_tot'] = combined_with_person_stats['Maxpay_offers_tot'].fillna(0)
    combined_with_person_stats['Minpay_offers_tot'] = combined_with_person_stats['Minpay_offers_tot'].fillna(0)
    combined_with_person_stats['Net_pay_offers'] = combined_with_person_stats['Net_pay_offers'].fillna(0)
    combined_with_person_stats['overlaps'] = combined_with_person_stats['overlaps'].fillna(0)

    print(combined_with_person_stats.shape)

    return (combined_with_person_stats)

def add_purchases(combined, profile, transactions_outside_offer, tdo, assign_to='first', calc_net_offer_time=False,
                  add_person_stats=True, debug=False,skip_overlap=False):
    """
    adds the purchase information (during and outside offers) to the main pivot table.
    the information of average spending during and outside of offers is calculated

    :param combined:
    :param profile:
    :param transactions_outside_offer:
    :param tdo:
    :param assign_to:
    :param calc_net_offer_time:
    :param add_person_stats:
    :return:
    """

    if skip_overlap==False:
        # deal with multiple offers
        print('assign_overlapping_purchases:')
        transactions_during_offer = assign_overlapping_purchases(tdo, assign_to=assign_to)

        # this should be done in a separate function
        transactions_during_offer['payments_original'] = 1.0 * transactions_during_offer['payments']
        if assign_to == 'ignore':
            # a payment is received during multiple offers are received, the amount is split equally between the offers
            transactions_during_offer['payments'] = transactions_during_offer['payments_original'] / (
                    transactions_during_offer['overlaps'] + 1.)
    else:
        transactions_during_offer=tdo

    t_spec_offer = transactions_during_offer.groupby(by=['person', 'offer', 'offer_viewed'])['payments'].agg(
        ['sum', 'count', 'max', 'min'])
    t_spec_offer.rename(
        columns={'sum': 'Tpay_offer', 'count': 'Npay_offer', 'max': 'Maxpay_offer', 'min': 'Minpay_offer', },
        inplace=True)

    # adding the total purchases outside of offers and during offer (specific and total)
    if debug:
        print('add_purchases:')
        print(combined.shape)

    combined_with_purchases = combined.merge(t_spec_offer, on=['person', 'offer', 'offer_viewed'], how='left')
    print(combined_with_purchases.shape)

    combined_with_purchases['Tpay_offer'] = combined_with_purchases['Tpay_offer'].fillna(0)
    combined_with_purchases['Npay_offer'] = combined_with_purchases['Npay_offer'].fillna(0)

    combined_with_purchases['Netpay_offer'] = combined_with_purchases['Tpay_offer'] - combined_with_purchases['reward']

    # if the offer is not seen, it should not be considered. It also prevents the division by zero
    # prevents division by zero if there's a payment
    combined_with_purchases['Avg_pay_offer'] = combined_with_purchases['Tpay_offer'] / np.maximum(
        combined_with_purchases['duration_effective'],
        1.0 + 0.0 * combined_with_purchases['duration_effective'])
    combined_with_purchases.loc[(combined_with_purchases['duration_effective'] == 0), 'Avg_pay_offer'] = 0.0
    combined_with_purchases['Avg_pay_offer'] = combined_with_purchases['Avg_pay_offer'].fillna(0)

    combined_with_purchases = combined_with_purchases.merge(profile, on='person', how='left')

    if add_person_stats:
        combined_with_purchases = add_stats_by_person(combined_with_purchases, transactions_outside_offer,
                                                      transactions_during_offer,
                                                      calc_net_offer_time=calc_net_offer_time)

    return (combined_with_purchases)

def create_pivot_table(portfolio, profile_all, offers_all, transactions_all, add_transactions=False,
                       skip_overlap=False,
                       skip_add_purchases=False, calc_net_offer_time=True, assign_to='first'):
    """
    creates the pivot table with all the derived information
    # portfolio, profile_all,offers_all,  transactions_all

    :param portfolio:
    :param profile_all:
    :param offers_all:
    :param transactions_all:
    :param add_transactions:
    :param skip_overlap:
    :param skip_add_purchases:
    :param calc_net_offer_time:
    :param assign_to:
    :return:
    """

    combined = create_person_offer_table(offers_all, transactions_all, portfolio)
    combined = add_offers_df(combined, portfolio);
    combined, trans, transactions_during_offer, transactions_outside_offer = extract_transactions(combined,
                                                                                transactions_all,
                                                                                add_transactions=add_transactions,
                                                                                skip_overlap=skip_overlap);
    if not skip_add_purchases:
        combined = add_purchases(combined, profile_all, transactions_outside_offer, transactions_during_offer,
                                 calc_net_offer_time=calc_net_offer_time, assign_to=assign_to, skip_overlap=skip_overlap)

    return (combined, transactions_during_offer, transactions_outside_offer, trans)

def basic_load_data_cv(  rename_offers=True, location='local'):
    # for debugging

    rename_offers_dict = {'0b1e1539f2cc45b7b9fa7c272da2e1d7': 'a',
                          '2298d6c36e964ae4a3e7e9706d1fb8c2': 'b',
                          '2906b810c7d4411798c6938adc9daaa5': 'c',
                          '3f207df678b143eea3cee63160fa8bed': 'd',
                          '4d5c57ea9a6940dd891ad53e9dbe8da0': 'e',
                          '5a8bc65990b245e5a138643cd4eb9837': 'f',
                          '9b98b8c7a33c4b65b9aebfe6a799e6d9': 'g',
                          'ae264e3637204a6fb9bb56bc8210ddfd': 'h',
                          'f19421c1d4aa40978ebb69ca19b0e20d': 'i',
                          'fafdcd668e3743c1bb461111dcafc2a4': 'j'}

    portfolio = load_portfolio(location=location)
    offers_all, transactions_all = load_transcript(location=location)
    profile_all = load_profile(location=location)

    if rename_offers:
        portfolio['offer'] = portfolio['offer'].replace(rename_offers_dict)
        offers_all['offer'] = offers_all['offer'].replace(rename_offers_dict)
        transactions_all['offer'] = transactions_all['offer'].replace(rename_offers_dict)

    print(""" combined, transactions_during_offer, transactions_outside_offer, trans = create_pivot_table(portfolio, profile_all,offers_all,
        transactions_all,add_transactions=False, skip_overlap=False, skip_add_purchases=False, calc_net_offer_time=False, assign_to='first') """)

    return(portfolio,offers_all,profile_all ,transactions_all)

def load_data_cv(person_split=None, rename_offers=False, time_split_min=None, time_split_max=None,
                 add_transactions=False, skip_overlap=False,
                 skip_add_purchases=False, calc_net_offer_time=False, assign_to='first',location='colab'):
    """
    creates the main table and transaction tables
    can also generate CV splits by persons (by fraction) or by time (before a time in hours)

    :param person_split:
    :param rename_offers:
    :param time_split_min:
    :param time_split_max:
    :param add_transactions:
    :param skip_overlap:
    :param skip_add_purchases:
    :param calc_net_offer_time:
    :param assign_to:
    :return:
    """

    # print(person_split)

    rename_offers_dict = {'0b1e1539f2cc45b7b9fa7c272da2e1d7': 'a',
                          '2298d6c36e964ae4a3e7e9706d1fb8c2': 'b',
                          '2906b810c7d4411798c6938adc9daaa5': 'c',
                          '3f207df678b143eea3cee63160fa8bed': 'd',
                          '4d5c57ea9a6940dd891ad53e9dbe8da0': 'e',
                          '5a8bc65990b245e5a138643cd4eb9837': 'f',
                          '9b98b8c7a33c4b65b9aebfe6a799e6d9': 'g',
                          'ae264e3637204a6fb9bb56bc8210ddfd': 'h',
                          'f19421c1d4aa40978ebb69ca19b0e20d': 'i',
                          'fafdcd668e3743c1bb461111dcafc2a4': 'j'}

    if ((not time_split_min is None) & (not person_split is None)):
        print('CV split must be either by person or by time')
        sys.exit(1)

    portfolio = load_portfolio(location=location)
    offers_all, transactions_all = load_transcript(location=location)
    profile_all = load_profile(location=location)

    if rename_offers:
        portfolio['offer'] = portfolio['offer'].replace(rename_offers_dict)
        offers_all['offer'] = offers_all['offer'].replace(rename_offers_dict)
        transactions_all['offer'] = transactions_all['offer'].replace(rename_offers_dict)

    if ((person_split is None) & (time_split_min is None) & (time_split_max is None)):
        combined, transactions_during_offer, transactions_outside_offer, trans = create_pivot_table(portfolio,
                                                                                profile_all,
                                                                                offers_all,
                                                                                transactions_all,
                                                                                add_transactions=add_transactions,
                                                                                skip_overlap=skip_overlap,
                                                                                skip_add_purchases=skip_add_purchases,
                                                                                calc_net_offer_time=calc_net_offer_time,
                                                                                assign_to=assign_to)

        return (
        combined, profile_all, portfolio, offers_all, trans, transactions_during_offer, transactions_outside_offer)

        # creating CV splitting by persons:
    print()
    if person_split is None:
        print('using full person list')

        if not time_split_min is None:
            print()
            print()
            print('min time split:')
            print("time: ", offers_all.time.min(), offers_all.time.max())

            offers = offers_all.loc[offers_all.time < time_split_min, :]
            transactions = transactions_all.loc[transactions_all.time < time_split_min, :]

            if time_split_max is None:
                print('time splitting without max')
                offers_test = offers_all.loc[offers_all.time >= time_split_min, :]
                transactions_test = transactions_all.loc[transactions_all.time >= time_split_min, :]

            else:
                print('time splitting with max')
                offers_test = offers_all.loc[offers_all.time >= time_split_max, :]
                transactions_test = transactions_all.loc[transactions_all.time >= time_split_max, :]
                print("offers.shape: ", offers.shape)
                print("offers_test.shape: ", offers_test.shape)

    else:
        # person split
        Lprofile, Lprofile_test = train_test_split(profile_all.person, test_size=person_split)
        print('train set %d persons' % Lprofile.shape[0])

        profile = profile_all.loc[profile_all.person.isin(list(Lprofile))]
        profile_test = profile_all.loc[profile_all.person.isin(list(Lprofile_test))]

        offers = offers_all.loc[offers_all.person.isin(list(Lprofile))]
        offers_test = offers_all.loc[offers_all.person.isin(list(Lprofile_test))]

        transactions = transactions_all.loc[transactions_all.person.isin(list(Lprofile))]
        transactions_test = transactions_all.loc[transactions_all.person.isin(list(Lprofile_test))]

    combined, transactions_during_offer, transactions_outside_offer, trans = create_pivot_table(portfolio, profile_all,
                                                                                offers, transactions,
                                                                                add_transactions=add_transactions,
                                                                                skip_overlap=skip_overlap,
                                                                                skip_add_purchases=skip_add_purchases,
                                                                                calc_net_offer_time=calc_net_offer_time,
                                                                                assign_to=assign_to)
    out = {}
    out['train'] = combined
    out['train_tdo'] = transactions_during_offer
    out['train_too'] = transactions_outside_offer
    out['train_offers'] = offers
    out['train_transactions'] = trans

    if person_split is None:
        if not time_split_min is None:
            #       return(out ,profile_all,portfolio,offers_all,transactions_all )
            #     else:
            # time split
            combined_test, transactions_during_offer_test, transactions_outside_offer_test, trans_test = create_pivot_table(
                portfolio, profile_all, offers_test, transactions_test,
                add_transactions=add_transactions, skip_overlap=skip_overlap,
                skip_add_purchases=skip_add_purchases,
                calc_net_offer_time=calc_net_offer_time, assign_to=assign_to)
            out['test'] = combined_test
            out['test_tdo'] = transactions_during_offer_test
            out['test_too'] = transactions_outside_offer_test
            out['test_offers'] = offers_test
            out['test_transactions'] = trans_test

            return (out, profile_all, portfolio, offers_all, transactions_all)

    else:
        # person split
        print('test data')
        combined_test, transactions_during_offer_test, transactions_outside_offer_test, trans_test = create_pivot_table(
            portfolio, profile_test, offers_test, transactions_test,
            add_transactions=add_transactions, skip_overlap=skip_overlap,
            skip_add_purchases=skip_add_purchases,
            calc_net_offer_time=calc_net_offer_time, assign_to=assign_to)
        out['test'] = combined_test
        out['test_tdo'] = transactions_during_offer_test
        out['test_too'] = transactions_outside_offer_test
        out['test_offers'] = offers_test
        out['test_transactions'] = trans_test

        return (out, profile_all, portfolio, offers_all, transactions_all)

def overlapping_offer_intevals(pot_df, final_time=744, return_person_occupacy=True):
    """

    :param pot_df:
    :param final_time:
    :param return_person_occupacy:
    :return:
    """

    # derive the net offer time for a person, counting overlapping offers only once

    # work on subset pot (person, offer, time)
    pot = pot_df[['person', 'offer', 'offer_viewed', 'offer_end']].copy().dropna(subset=['offer_viewed'])
    pot['offer_viewed'] = pot['offer_viewed'].astype(int)
    pot['offer_end'] = pot['offer_end'].astype(int)

    start_time = pot['offer_viewed'].values
    end_time = np.minimum(pot['offer_end'].values, final_time + 0 * pot['offer_end'].values)
    ind = np.arange(0, len(pot))

    # Repeat start position index length times and concatenate
    lengths = end_time - start_time + 1
    cat_start = np.repeat(start_time, lengths)
    y_range = np.repeat(ind, lengths)

    # Create group counter that resets for each start/length
    cat_counter = np.arange(lengths.sum()) - np.repeat(lengths.cumsum() - lengths, lengths)

    # Add group counter to group specific starts
    x_range = cat_start + cat_counter

    # sparse matrix to vectorize computation
    occupancy = coo_matrix((1 + 0 * x_range, (y_range, x_range)))

    occupancy_df = pd.DataFrame(occupancy.toarray(), index=pot.index, columns=range(0, final_time + 1))

    person_occupacy = pd.concat((pot.drop(columns=['offer_viewed', 'offer_end']), occupancy_df), axis=1)

    # aggregate by person
    person_occupacy = person_occupacy.groupby(by='person').agg('sum')
    person_occupacy = person_occupacy.clip(upper=1)

    effective_duration = person_occupacy.sum(axis=1).to_frame().rename(columns={0: 'tot_net_offers_time'})
    if return_person_occupacy:
        return (effective_duration, person_occupacy)
    else:
        return (effective_duration)

def plot_offers_of_customer_time(piv, trans, tdo, too, customer, yc=0, step=0.1, plot_transactions=True,
                                 plot_offer_name=True, overlay_spacing=0.2):
    """
    plots the offers and the transactions for a customer versus time

    :param piv:
    :param trans:
    :param tdo:
    :param too:
    :param customer:
    :param yc:
    :param step:
    :param plot_transactions:
    :param plot_offer_name:
    :param overlay_spacing:
    :return:
    """

    gg = piv.loc[(piv.person == customer), 'offer_received'].drop_duplicates()
    ind = gg.index.values

    viewed_offers = []

    v_off_times = np.zeros((ind.shape[0], 2))
    v_off_set = set([])

    c = 0

    offer_colors = []

    for n in ind:
        hh = plt.plot([piv.loc[n, 'offer_received'], piv.loc[n, 'offer_end']], [yc, yc], 'o-', alpha=0.6)
        color = hh[0].get_color()
        offer_colors.append(color)

        if plot_offer_name:
            plt.text((piv.loc[n, 'offer_received'] + piv.loc[n, 'offer_end']) / 2, yc - .2, piv.loc[n, 'offer']);

        if not np.isnan(piv.loc[n, 'offer_viewed']):
            plt.plot([piv.loc[n, 'offer_viewed'], piv.loc[n, 'offer_end']], [yc, yc], '.-', alpha=1.0, color=color)
            viewed_offers.append(piv.loc[n, 'offer'])
            v_off_times[c, 0] = piv.loc[n, 'offer_viewed']
            v_off_times[c, 1] = piv.loc[n, 'offer_end']
            v_off_set = v_off_set.union(set(range(int(piv.loc[n, 'offer_viewed']), int(piv.loc[n, 'offer_end']) + 0)))
            c += 1

        yc += step
    v_off_times = v_off_times[0:c, :]

    diff = np.diff(np.array(list(v_off_set)))
    diff[diff > 1] = 0
    overlapping_offers_time = diff.sum()

    if plot_transactions:
        df = trans.loc[(trans.person == customer)]
        plt.plot(df['time'], df['payments'], 'o', color='k', alpha=0.3)

        # show transactions attributed to several offers
        n_offer = 0

        for n in ind:
            y_offer = n_offer + 1
            tdoc = tdo.loc[(tdo.person == customer) & (tdo.offer == piv.loc[n, 'offer']) & (
                    tdo.offer_viewed == piv.loc[n, 'offer_viewed'])]
            dy = (tdoc['overlaps']) * y_offer
            if (tdoc['overlaps'] > 0).any():
                y_offer += overlay_spacing

            if n_offer < len(offer_colors):
                #         print(n_offer)

                #         print(offer_colors[n_offer])
                plt.plot(tdoc['time'], tdoc['payments'] + dy, '.', color=offer_colors[n_offer], alpha=0.8)
            else:
                plt.plot(tdoc['time'], tdoc['payments'] + dy, '.', alpha=0.8)
            n_offer += 1

    plt.xlabel('hours');
    plt.ylabel('purchases');

    return (viewed_offers, v_off_times, overlapping_offers_time)

if __name__ == "__main__":
    out, profile_all, portfolio, offers_all, transactions_all, transactions_during_offer, transactions_outside_offer = load_data_cv(
        person_split=None, rename_offers=True, time_split_min=None, time_split_max=None,
        add_transactions=False, skip_overlap=False, skip_add_purchases=False, calc_net_offer_time=True,
        assign_to='ignore', location='local')


import svgwrite
from bootstrap import application

STYLE = {
    "up-to-date": {
        "text": "up-to-date",
        "rect_size": (95, 20),
        "rect_fill": "green",
        "paragraph_fill": "white",
        "text_fill": "white",
    },
    "update-available": {
        "text": "update available",
        "rect_size": (140, 20),
        "rect_fill": "orange",
        "paragraph_fill": "white",
        "text_fill": "white",
    },
    "security-update-available": {
        "text": "security update available",
        "rect_size": (360, 20),
        "rect_fill": "red",
        "paragraph_fill": "white",
        "text_fill": "white",
    },
    "unknown": {
        "text": "unknown",
        "rect_size": (95, 20),
        "rect_fill": "grey",
        "paragraph_fill": "white",
        "text_fill": "white",
    },
}

def simple_text(name, style, text=None):
    """Draw a text in a rectangle."""
    dwg = svgwrite.Drawing(name, style["rect_size"])

    rect = dwg.rect(insert=(0, 0), size=style["rect_size"], fill=style["rect_fill"])
    dwg.add(rect)

    paragraph = dwg.add(
        dwg.g(font_size=14, font_family="DejaVu Sans", fill=style["paragraph_fill"])
    )
    if not text:
        text = style["text"]
    text_elem = dwg.text(
        text, insert=(5, 15), font_weight="bold", fill=style["text_fill"]
    )
    paragraph.add(text_elem)

    file_name = "{}.svg".format(name.replace(" ", "-"))
    file_path = application.config["GENERATED_SVG_FOLDER"] + file_name

    dwg.saveas(file_path)
    return file_name

if __name__ == "__main__":
    for state in ["up-to-date", "update available", "security update available"]:
        simple_text("{}.svg".format(state.replace(" ", "-")), state, STYLE[state])


from typing import List

def remove_neg(num_list: List[float]) -> None:
    """리스트 num_list에서 음수를 삭제한다.

    >>> numbers = [-5, 1, -3, 2]
    >>> remove_neg(numbers)
    >>> numbers
    [1, 2]
    """

    for item in num_list:
        if item < 0:
            num_list.remove(item)


from rest_framework.pagination import PageNumberPagination
from rest_framework.response import Response
from urllib.parse import urlparse

class DefaultPagination(PageNumberPagination):
    page_size = 10
    page_size_query_param = 'page_size'
    max_page_size = 30

    def get_paginated_response(self, data):
        return Response(
            {
                "links": {
                    "next": self.get_next_link(),
                    "previous": self.get_previous_link(),
                    "current": self.request.query_params.get("page", 1),
                },
                "total_items": self.page.paginator.count,
                "total_pages": self.page.paginator.num_pages,
                "results": data,
            }
        )


from django.db import transaction
from django.shortcuts import render, redirect
from django.template import loader

from datetime import datetime

from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from user.models import Attendee
from user.forms import UserAuthForm, UserExtendForm
from food.models import FoodOrder
from event.models import Event, EventAttendee

def register(request):
    if request.method == 'POST':
        user_auth_form = UserAuthForm(request.POST)
        user_extended_form = UserExtendForm(request.POST)
        if user_auth_form.is_valid() and user_extended_form.is_valid():
            user = user_auth_form.save(commit=False)
            user.set_password(user.password)
            user.save()
            user.refresh_from_db()
            user_extended = user_extended_form.save(commit=False)
            user_extended.user_auth = user
            user_extended.save()
            return render(request, 'registration/login.html', {'form': user_auth_form}, status=200)
        return render(request, 'registration/registration.html', {'error_form': user_auth_form}, status=400)

    elif request.method == 'GET':
        loader.get_template('registration/registration.html')
        user_auth_form = UserAuthForm()
        user_extended_form = UserExtendForm()
        return render(request, 'registration/registration.html',
                      {'user_auth_form': user_auth_form, 'user_extended_form': user_extended_form})

class EventAttendeeResources(APIView):
    permission_classes = ()
    authentication_classes = ()

    @transaction.atomic
    def post(self, request, event_date, event_name):
        try:
            event_date = datetime.strptime(event_date, "%Y-%m-%d").date()
        except ValueError:
             return Response({'failue': 'event date is not well formated'},
                         status=status.HTTP_400_BAD_REQUEST)

        events = Event.objects.filter(name = event_name,
            event_date = event_date)
        if not events.exists():
            return Response({'reason': f"there is no event with the name:\
                     {event_name} and the date {str(event_date)} "},
                      status=status.HTTP_404_NOT_FOUND)

        current_event = events[0]

        attendee_first_name = request.data.get('first_name')
        attendee_last_name = request.data.get('last_name')
        number_of_guests = request.data.get('number_of_guests')

        foods  = current_event.food_types.all()
        food_list = {}

        food_types = foods.values_list('food_type', flat=True)

        for key, value in request.data.items():
            if (key  in food_types) and (value is not None) and int(value) > 0:
                food_list[key] = int(value)

        if ((attendee_first_name in [[], '', None, {}])
            or (attendee_last_name in [[], '', None, {}])
            or (food_list in [[], '', None, {}])):
            return Response({'failue': 'some of the inputs are empty'},
                        status=status.HTTP_400_BAD_REQUEST)

        food_orders = []

        for food_type, number in food_list.items():
            food_order = FoodOrder(food = foods.get(food_type=food_type),
                                number = number)
            food_order.save()
            food_orders.append(food_order)
        try:
            attendee, _ = Attendee.objects.get_or_create(
                first_name = attendee_first_name,
                last_name = attendee_last_name)
        except ValueError:
            return Response({"reason": "attendee format is wrong"},
                    status=status.HTTP_400_BAD_REQUEST)
        try:
            event_attendee = EventAttendee(event = current_event,
                attendee = attendee, number_of_guests = number_of_guests)
            event_attendee.save()
        except ValueError:
            return Response({"reason": "event_attendee format is wrong"},
                    status=status.HTTP_400_BAD_REQUEST)

        for food_order in food_orders:
            event_attendee.food_orders.add(food_order)

        return render(request, 'user/attend.html')


# coding:utf-8

from functools import wraps

from Crypto.Cipher import AES

class CustomException(Exception):
    def __init__(self, msg=None, code=None):
        super(BaseException, self).__init__()
        self.msg = msg
        self.code = code

    def __str__(self):
        return u"code:{0}, msg:{1}".format(self.msg, self.code)

class NeedCaptchaError(CustomException):
    def __init__(self, msg=None, code=None):
        super(NeedCaptchaError, self).__init__()
        self.msg = msg or u"需要图片验证码"
        self.code = code or 10001

class CaptchaError(CustomException):
    def __init__(self, msg=None, code=None):
        super(CaptchaError, self).__init__()
        self.msg = msg or u"图片验证码错误"
        self.code = code or 10002

def aes_decrypt(raw_str, key, iv, mode=None):
    try:
        if isinstance(raw_str, unicode):
            raw_str = raw_str.decode("hex")
        if isinstance(key, unicode):
            key = key.encode("utf-8")
        mode = mode or AES.MODE_CBC
        cipher = AES.new(key=key, IV=iv, mode=mode)
        msg = cipher.decrypt(raw_str)
        padding_len = 0 if len(msg) % len(key) == 0 else ord(msg[-1])
        ret = msg[:-padding_len].decode("utf-8")
        return ret
    except Exception as e:
        raise e

def catch_exception(func):
    @wraps(func)
    def _handle(self, response):
        logger = self.logger
        logger.debug("--->func:{0}, status:{1}, url:{2}".format(func.__name__, response.status, response.url))
        try:
            ret = func(self, response)
            return ret
        except Exception as e:
            logger.exception("--->func:{0}, error:{1}".format(func.__name__, str(e)))
            return

    return _handle


import json
import os
from scene import Scene

class ModelController:
    def __init__(self, game_file_directory: str):
        self.game_data = {}
        self.game_file_directory = game_file_directory
        self._load_game_data()     # Populate the game_data dict

    def _is_valid_game_file_directory(self):
        return os.path.exists(self.game_file_directory)

    def _load_game_data(self):
        """
        Populates the game_data dict with a SceneFile object for each
        game file in the game_file directory.
        """
        if not self._is_valid_game_file_directory():
            raise Exception("Invalid game file directory")
        else:
            for file in os.listdir(self.game_file_directory):
                scene = Scene()
                if str(file).endswith("json"):
                    try:
                        with open(self.game_file_directory + file) as f:
                            json_file = json.load(f)
                            scene.scene_id = json_file["scene_id"]
                            scene.presentation_style = json_file[
                                "presentation_style"
                                ]
                            scene.print_slowly = json_file["print_slowly"]
                            scene.options_string = json_file["options_string"]
                            scene.choice_target_dict = json_file[
                                "choice_target_dict"
                                ]
                            try:
                                scene.roll_message = json_file["roll_message"]
                            except Exception:
                                print("no roll message")
                            self.game_data[scene.scene_id] = scene
                    except Exception:
                        raise Exception("Unable to update Scene object from " +
                                        "JSON file.")

            for file in os.listdir(self.game_file_directory):
                if str(file).endswith("txt"):
                    try:
                        with open(self.game_file_directory + file) as f:
                            list_text = f.readlines()
                            scene_string = ""
                            for line in list_text:
                                scene_string += line

                            file_prefix = os.path.splitext(file)[0]
                            if file_prefix in self.game_data.keys():
                                tmp_scene_object = self.game_data[file_prefix]
                                tmp_scene_object.scene_string = scene_string
                                print(tmp_scene_object.scene_string)
                    except Exception:
                        raise Exception("Unable to open game text file.")


import requests
import pandas as pd
import numpy as np
import os

local = '/home/innereye/alarms/'
if os.path.isdir(local):
    os.chdir(local)
    with open('/home/innereye/alarms/oath.txt') as f:
        oath = f.readlines()[0][:-1]
else:
    oath = os.environ['OAuth']
# get_coordinates(city_name)

def get_coordinates(city_name):
    city_name = city_name + ', ישראל'
    geocoder_url = f'https://maps.googleapis.com/maps/api/geocode/json?address={city_name}&key={oath}&language=iw'
    geocoding_result = requests.get(geocoder_url).json()
    if geocoding_result['results'] == []:
        lat = 0
        long = 0
    else:
        lat = geocoding_result['results'][0]['geometry']['location']['lat']
        long = geocoding_result['results'][0]['geometry']['location']['lng']
    return (lat, long)

missing = ['רחובות','חולון']
lt = []
lg = []
for miss in missing:
    lat, long = get_coordinates(miss)
    lt.append(lat)
    lg.append(long)
df = pd.DataFrame(missing, columns=['loc'])
df['lat'] = lt
df['long'] = lg
df.to_csv('test.csv', sep=',', index=False)