######## Evaluation pipeline for perch data on pointrcnn model #########
# Author: Beiming Li

import _init_path
import os
import numpy as np
import torch
from torch.utils.data import DataLoader
from lib.datasets.perch_dataset import PerchDataset
import argparse

from lib.net.point_rcnn import PointRCNN
import tools.train_utils.train_utils as train_utils
from lib.utils.bbox_transform import decode_bbox_target
from tools.kitti_object_eval_python.evaluate import evaluate as kitti_evaluate

from lib.config import cfg, cfg_from_file, save_config_to_file, cfg_from_list
import lib.utils.kitti_utils as kitti_utils
import lib.utils.iou3d.iou3d_utils as iou3d_utils
from datetime import datetime
import logging
import re
import glob
import time
from tensorboardX import SummaryWriter
import tqdm

import matplotlib.pyplot as plt

np.random.seed(1024)  # set the same seed

parser = argparse.ArgumentParser(description="arg parser")

parser = argparse.ArgumentParser(description="arg parser")
parser.add_argument('--cfg_file', type=str, default='cfgs/default.yml', help='specify the config for evaluation')
parser.add_argument("--eval_mode", type=str, default='rpn', required=True, help="specify the evaluation mode")

parser.add_argument('--eval_all', action='store_true', default=False, help='whether to evaluate all checkpoints')
parser.add_argument('--test', action='store_true', default=False, help='evaluate without ground truth')
parser.add_argument("--ckpt", type=str, default=None, help="specify a checkpoint to be evaluated")
parser.add_argument("--rpn_ckpt", type=str, default=None, help="specify the checkpoint of rpn if trained separated")
parser.add_argument("--rcnn_ckpt", type=str, default=None, help="specify the checkpoint of rcnn if trained separated")

parser.add_argument('--batch_size', type=int, default=1, help='batch size for evaluation')
parser.add_argument('--workers', type=int, default=4, help='number of workers for dataloader')
parser.add_argument("--extra_tag", type=str, default='default', help="extra tag for multiple evaluation")
parser.add_argument('--output_dir', type=str, default=None, help='specify an output directory if needed')
parser.add_argument("--ckpt_dir", type=str, default=None, help="specify a ckpt directory to be evaluated if needed")

parser.add_argument('--save_result', action='store_true', default=False, help='save evaluation results to files')
parser.add_argument('--save_rpn_feature', action='store_true', default=False,
                    help='save features for separately rcnn training and evaluation')

parser.add_argument('--random_select', action='store_true', default=True, help='sample to the same number of points')
parser.add_argument('--start_epoch', default=0, type=int, help='ignore the checkpoint smaller than this epoch')
parser.add_argument("--rcnn_eval_roi_dir", type=str, default=None,
                    help='specify the saved rois for rcnn evaluation when using rcnn_offline mode')
parser.add_argument("--rcnn_eval_feature_dir", type=str, default=None,
                    help='specify the saved features for rcnn evaluation when using rcnn_offline mode')
parser.add_argument('--set', dest='set_cfgs', default=None, nargs=argparse.REMAINDER,
                    help='set extra config keys if needed')
args = parser.parse_args()

def create_dataloader():
    mode = 'TEST' if args.test else 'EVAL'
    DATA_PATH = os.path.join('..', 'data')

    # create dataloader
    test_set = PerchDataset(root_dir=DATA_PATH, mode=mode)

    test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, pin_memory=True,
                             num_workers=args.workers, collate_fn=test_set.collate_batch)

    return test_loader

def load_ckpt_based_on_args(model):
    train_utils.load_checkpoint(model, filename=args.ckpt)

def eval_one_epoch(model, dataloader):
    if cfg.RPN.ENABLED and not cfg.RCNN.ENABLED:
        ret_dict = eval_one_epoch_rpn(model, dataloader)
    elif not cfg.RPN.ENABLED and cfg.RCNN.ENABLED:
        ret_dict = eval_one_epoch_rcnn(model, dataloader)
    elif cfg.RPN.ENABLED and cfg.RCNN.ENABLED:
        ret_dict = eval_one_epoch_joint(model, dataloader)
    else:
        raise NotImplementedError
    return ret_dict

def eval_one_epoch_rpn(model, dataloader):
    np.random.seed(1024)
    mode = 'TEST' if args.test else 'EVAL'

    thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9]
    total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0

    for data in dataloader:
        pts_rect, pts_features, pts_input, gt_boxes3d = data['pts_rect'], data['pts_features'], data['pts_input'], data['gt_boxes3d']
        inputs = torch.from_numpy(pts_input).cuda(non_blocking=True).float()
        input_data = {'pts_input': inputs}

        # model inference
        ret_dict = model(input_data)
        rpn_cls, rpn_reg = ret_dict['rpn_cls'], ret_dict['rpn_reg']
        backbone_xyz, backbone_features = ret_dict['backbone_xyz'], ret_dict['backbone_features']

        rpn_scores_raw = rpn_cls[:, :, 0]
        rpn_scores = torch.sigmoid(rpn_scores_raw)
        seg_result = (rpn_scores > cfg.RPN.SCORE_THRESH).long()

        # proposal layer
        rois, roi_scores_raw = model.rpn.proposal_layer(rpn_scores_raw, rpn_reg, backbone_xyz)  # (B, M, 7)
        batch_size = rois.shape[0]

        # calculate recall and save results to file
        for bs_idx in range(batch_size):
            cur_scores_raw = roi_scores_raw[bs_idx]  # (N)
            cur_boxes3d = rois[bs_idx]  # (N, 7)
            cur_seg_result = seg_result[bs_idx]
            cur_pts_rect = pts_rect[bs_idx]
            visualize_data(pts_input[bs_idx], gt_boxes3d[bs_idx], cur_boxes3d[cur_scores_raw > 0].cpu())

    return ret_dict

def eval_one_epoch_joint(model, dataloader):
    np.random.seed(666)
    MEAN_SIZE = torch.from_numpy(cfg.CLS_MEAN_SIZE[0]).cuda()
    mode = 'TEST' if args.test else 'EVAL'

    model.eval()

    thresh_list = [0.1, 0.3, 0.5, 0.7, 0.9]
    total_recalled_bbox_list, total_gt_bbox = [0] * 5, 0
    total_roi_recalled_bbox_list = [0] * 5
    cnt = final_total = total_cls_acc = total_cls_acc_refined = total_rpn_iou = 0

    progress_bar = tqdm.tqdm(total=len(dataloader), leave=True, desc='eval')

    for data in dataloader:
        pts_rect, pts_features, pts_input = data['pts_rect'], data['pts_features'], data['pts_input']
        batch_size = len(pts_input)
        inputs = torch.from_numpy(pts_input).cuda(non_blocking=True).float()
        input_data = {'pts_input': inputs}
        # model inference
        ret_dict = model(input_data)

        roi_scores_raw = ret_dict['roi_scores_raw']  # (B, M)
        roi_boxes3d = ret_dict['rois']  # (B, M, 7)
        seg_result = ret_dict['seg_result'].long()  # (B, N)

        rcnn_cls = ret_dict['rcnn_cls'].view(batch_size, -1, ret_dict['rcnn_cls'].shape[1])
        rcnn_reg = ret_dict['rcnn_reg'].view(batch_size, -1, ret_dict['rcnn_reg'].shape[1])  # (B, M, C)

        # bounding box regression
        anchor_size = MEAN_SIZE
        if cfg.RCNN.SIZE_RES_ON_ROI:
            assert False

        pred_boxes3d = decode_bbox_target(roi_boxes3d.view(-1, 7), rcnn_reg.view(-1, rcnn_reg.shape[-1]),
                                            anchor_size=anchor_size,
                                            loc_scope=cfg.RCNN.LOC_SCOPE,
                                            loc_bin_size=cfg.RCNN.LOC_BIN_SIZE,
                                            num_head_bin=cfg.RCNN.NUM_HEAD_BIN,
                                            get_xz_fine=True, get_y_by_bin=cfg.RCNN.LOC_Y_BY_BIN,
                                            loc_y_scope=cfg.RCNN.LOC_Y_SCOPE, loc_y_bin_size=cfg.RCNN.LOC_Y_BIN_SIZE,
                                            get_ry_fine=True).view(batch_size, -1, 7)
        # print(pred_boxes3d, rcnn_cls)
        # scoring
        if rcnn_cls.shape[2] == 1:
            raw_scores = rcnn_cls  # (B, M, 1)

            norm_scores = torch.sigmoid(raw_scores)
            pred_classes = (norm_scores > cfg.RCNN.SCORE_THRESH).long()

        # evaluation
        recalled_num = gt_num = rpn_iou = 0
        if not args.test:
            if not cfg.RPN.FIXED:
                rpn_cls_label, rpn_reg_label = data['rpn_cls_label'], data['rpn_reg_label']
                rpn_cls_label = torch.from_numpy(rpn_cls_label).cuda(non_blocking=True).long()

            gt_boxes3d = data['gt_boxes3d']

            for k in range(batch_size):
                # calculate recall
                cur_gt_boxes3d = gt_boxes3d[k]
                tmp_idx = cur_gt_boxes3d.__len__() - 1

                while tmp_idx >= 0 and cur_gt_boxes3d[tmp_idx].sum() == 0:
                    tmp_idx -= 1

                if tmp_idx >= 0:
                    cur_gt_boxes3d = cur_gt_boxes3d[:tmp_idx + 1]

                    cur_gt_boxes3d = torch.from_numpy(cur_gt_boxes3d).cuda(non_blocking=True).float()
                    iou3d = iou3d_utils.boxes_iou3d_gpu(pred_boxes3d[k], cur_gt_boxes3d)
                    gt_max_iou, _ = iou3d.max(dim=0)
                    refined_iou, _ = iou3d.max(dim=1)

                    for idx, thresh in enumerate(thresh_list):
                        total_recalled_bbox_list[idx] += (gt_max_iou > thresh).sum().item()
                    recalled_num += (gt_max_iou > 0.7).sum().item()
                    gt_num += cur_gt_boxes3d.shape[0]
                    total_gt_bbox += cur_gt_boxes3d.shape[0]

                    # original recall
                    iou3d_in = iou3d_utils.boxes_iou3d_gpu(roi_boxes3d[k], cur_gt_boxes3d)
                    gt_max_iou_in, _ = iou3d_in.max(dim=0)

                    for idx, thresh in enumerate(thresh_list):
                        total_roi_recalled_bbox_list[idx] += (gt_max_iou_in > thresh).sum().item()

                if not cfg.RPN.FIXED:
                    fg_mask = rpn_cls_label > 0
                    correct = ((seg_result == rpn_cls_label) & fg_mask).sum().float()
                    union = fg_mask.sum().float() + (seg_result > 0).sum().float() - correct
                    rpn_iou = correct / torch.clamp(union, min=1.0)
                    total_rpn_iou += rpn_iou.item()

        disp_dict = {'mode': mode, 'recall': '%d/%d' % (total_recalled_bbox_list[3], total_gt_bbox)}
        progress_bar.set_postfix(disp_dict)
        progress_bar.update()

        # scores thresh
        inds = norm_scores > cfg.RCNN.SCORE_THRESH

        for k in range(batch_size):
            cur_inds = inds[k].view(-1)
            if cur_inds.sum() == 0:
                continue

            pred_boxes3d_selected = pred_boxes3d[k, cur_inds]
            raw_scores_selected = raw_scores[k, cur_inds]
            norm_scores_selected = norm_scores[k, cur_inds]

            # NMS thresh
            # rotated nms
            boxes_bev_selected = kitti_utils.boxes3d_to_bev_torch(pred_boxes3d_selected)
            keep_idx = iou3d_utils.nms_gpu(boxes_bev_selected, raw_scores_selected, cfg.RCNN.NMS_THRESH).view(-1)
            pred_boxes3d_selected = pred_boxes3d_selected[keep_idx]
            scores_selected = raw_scores_selected[keep_idx]
            pred_boxes3d_selected, scores_selected = pred_boxes3d_selected.cpu().numpy(), scores_selected.cpu().numpy()

            if (gt_max_iou > thresh).sum().item() != 0:
                visualize_data(pts_input[k], gt_boxes3d[k], pred_boxes3d_selected)

    progress_bar.close()
    return ret_dict

def visualize_data(pts_input, gt_boxes3d, pred_boxes3d):
    print("visualization")
    fig = plt.figure()
    ax = plt.axes(projection='3d')
    ax.set_xlim(-100, 25)
    ax.set_ylim(-100, 50)
    ax.set_zlim(-100, 100)

    x, y, z = pts_input.T
    ax.scatter3D(x, y, z, s=0.5)

    x_gt, y_gt, z_gt = gt_boxes3d[:, 0], gt_boxes3d[:, 1], gt_boxes3d[:, 2]
    ax.scatter3D(x_gt, y_gt, z_gt, s=50, c='r')

    x_pred, y_pred, z_pred = pred_boxes3d[:, 0], pred_boxes3d[:, 1], pred_boxes3d[:, 2]
    ax.scatter3D(x_pred, y_pred, z_pred, s=100, c='g')
    plt.show()

if __name__ == "__main__":
    # merge config and log to file
    if args.cfg_file is not None:
        cfg_from_file(args.cfg_file)
    if args.set_cfgs is not None:
        cfg_from_list(args.set_cfgs)
    cfg.TAG = os.path.splitext(os.path.basename(args.cfg_file))[0]

    if args.eval_mode == 'rpn':
        cfg.RPN.ENABLED = True
        cfg.RCNN.ENABLED = False
    elif args.eval_mode == 'rcnn':
        cfg.RCNN.ENABLED = True
        cfg.RPN.ENABLED = cfg.RPN.FIXED = True
    else:
        raise NotImplementedError

    with torch.no_grad():
        # create dataloader & network
        test_loader = create_dataloader()
        model = PointRCNN(num_classes=2, use_xyz=True, mode='TEST')
        model.cuda()

        # load checkpoint
        load_ckpt_based_on_args(model)

        # start evaluation
        eval_one_epoch(model, test_loader)


# %%
import pandas as pd
import datetime as dt
# %%
df = pd.read_csv('ProductionReport.csv')
df['END_TIME'] = pd.to_datetime(df['END_TIME'])
# %%
# Check progress -
# INPUT: replace pattern variable with pattern name
pattern = "{P220082-line}"

pattern_df = df[df['AGG_PATTERN_NAME'].str.contains(pattern)]
total_sum =pattern_df[['COMPLETE']].sum()
# OUTPUT:
print(total_sum)

# %%

# INPUT: list of machine () replace list_of_machine with list of machine
field_name = 'MC_ID'
list_of_machine = df[field_name].unique()

# if past 24 hr have completed job means the machine is busy
df_past24hr = df[df['END_TIME']>=(dt.datetime.now()-dt.timedelta(hours=24))]
list_of_occupied_machine = df_past24hr[field_name].unique()
list_if_machine_is_occupied = [(x in list_of_occupied_machine) for x in list_of_machine]
df_occupied_or_not = pd.DataFrame({'list_of_machine': list(list_of_machine), 'list_if_machine_is_occupied': list_if_machine_is_occupied, }, columns=['list_of_machine', 'list_if_machine_is_occupied'])
# OUTPUT:
df_occupied_or_not

# %%


import mygrad as mg
from mygrad.operation_base import _NoValue

from .functools import SmartSignature

def test_init():
    pargs = SmartSignature(1, None, a_kwarg=1, not_set=_NoValue)
    assert (1, None) == pargs.args
    assert dict(a_kwarg=1) == pargs.kwargs

def test_tensor_only():
    x = mg.tensor([1])
    y = mg.tensor(1)
    x1, y1 = SmartSignature(1, x, [1], y, mg.asarray(x), a_kwarg=None).tensors_only()
    assert x is x1
    assert y is y1

def test_as_no_mygrad():
    x = mg.tensor([1])
    a, b, c = SmartSignature(1, x, None).args_as_no_mygrad()
    assert a == 1
    assert b is x.data
    assert c is None

def test_smart_sig_named_unpack():
    assert dict(**SmartSignature(1, 2, _NoValue, a=-1, b=-2, c=_NoValue)) == dict(
        a=-1, b=-2
    )

def test_iter_unpack():
    assert [*SmartSignature(1, 2, _NoValue, 4, a=-1, b=-2, c=_NoValue)] == [1, 2, 4]

def test_setitem_ignores_no_value():
    sig = SmartSignature()
    sig["a"] = _NoValue
    sig["b"] = 1
    assert dict(**sig) == {"b": 1}


import webapp2
import jinja2
import logging
import os
from google.appengine.api import users
from google.appengine.ext import ndb
import json

jinja_environment = jinja2.Environment(loader=jinja2.FileSystemLoader(os.path.dirname(__file__)))

class Quotes(ndb.Model):
    text = ndb.StringProperty(repeated=True)
    person = ndb.StringProperty(repeated=True)

class MainHandler(webapp2.RequestHandler):
    def get(self):
        quotes = {}
        data = Quotes.get_by_id(5135269057527808)

        combined_list = zip(data.text, data.person)
        quotes["zip"] = combined_list

        template = jinja_environment.get_template('templates/homepage.html')
        self.response.write("this it totally the get page");
        self.response.write(template.render(quotes));
    def post(self):
        data = Quotes.get_by_id(5135269057527808)
        data.text.append(self.request.get("quote"));
        poster = self.request.get("name")
        if poster == "":
            poster = "Anonymous"
        data.person.append(poster)
        data.put()
        self.response.write("this it totally the post page");
        self.redirect("/")

app = webapp2.WSGIApplication([
    ('/', MainHandler)
], debug=True)


import streamlit as st
from streamlit_tags import st_tags
import main_
import pyautogui
import time

# st.set_page_config(
#     page_title="Recovery Keyword"
# )
def recovery_keyword_func():
    st.markdown("<h4 style='text-align: left; color: black;'>Recovery Keyword </h4>",
                unsafe_allow_html=True)
    tab1, tab2 = st.tabs(["📝 Input", "📤 Upload"])

    with st.form("my_form"):
        with tab1:
            keywords = st_tags(
                label="keyword :",
                text='Press enter to add more',
                maxtags=100,
                key='1')

        with tab2:
            uploaded_files = st.file_uploader("Import Keyword :")
            if st.button("upload"):
                if not uploaded_files:
                    st.warning("No data uploaded")
                try:
                    if uploaded_files is not None:
                        for upload in uploaded_files:
                            msg = str(upload, "utf-8")
                            main_.list_input.append(msg)
                        st.info("Data Uploaded")
                except:
                    st.warning("Upload Filed")

        since = st.date_input(  # since
            "since :",
            key="since"
        )
        format_since = since.strftime('%Y-%m-%d')

        if 'dummy_data' not in st.session_state.keys():
            dummy_data = ['youtube', 'instagram', 'twitter', 'tiktok', 'helo']
            st.session_state['dummy_data'] = dummy_data
        else:
            dummy_data = st.session_state['dummy_data']

        def checkbox_container(data):
            st.write('select platform :')
            cols = st.columns(5)
            if cols[0].form_submit_button('Select All'):
                for i in data:
                    st.session_state[i + '_dynamic_checkbox'] = True
                st.experimental_rerun()
            if cols[1].form_submit_button('UnSelect All'):
                for i in data:
                    st.session_state[i + '_dynamic_checkbox'] = False
                st.experimental_rerun()
            for i in data:
                st.checkbox(i, key=i + '_dynamic_checkbox')

        def get_selected_checkboxes():
            return [i.replace('_dynamic_checkbox', '\_.*keyword') for i in st.session_state.keys() if
                    i.endswith('_dynamic_checkbox') and st.session_state[i]]

        def in_submit():
            try:
                with st.spinner('Wait for it...'):
                    time.sleep(5)
                st.success('Done!')
                main_.list_input.extend(keywords)
                main_.platforms.extend(get_selected_checkboxes())
                main_.job_keyword(since=format_since)
                pyautogui.hotkey("ctrl", "F5")
            except:
                st.warning('Recovery Filed')
                keywords.clear()
                main_.list_input.clear()
                time.sleep(1)
                pyautogui.hotkey("ctrl", "F5")

        checkbox_container(dummy_data)
        # st.write(get_selected_checkboxes())
        st.markdown("""---""")

        send_submit = st.form_submit_button("Recovery")
        if send_submit:
            if not keywords:
                if not uploaded_files:
                    st.warning("Input / Upload Keyword")

                elif uploaded_files:
                    if not get_selected_checkboxes():
                        st.warning("Choose platform")
                    elif get_selected_checkboxes():
                        in_submit()
            elif keywords:
                if not get_selected_checkboxes():
                    st.warning("Choose platform")
                elif get_selected_checkboxes():
                    in_submit()


import driving
import capture

import cv2
import datetime

import os
import shutil

WIDTH = 480
HEIGHT = 240
folderName = "/media/91C5-7087/image_data"

def main():
    if os.path.isdir(folderName):
        shutil.rmtree(folderName, ignore_errors=True)

    os.mkdir(folderName)

    steering_wheel_thread = driving.SteeringWheelThread()
    capture_image_thread = capture.CaptureImageThread(WIDTH, HEIGHT)

    steering_wheel_thread.start()
    capture_image_thread.start()

    while True:
        try:
            steering_angle, throttle, reverse = steering_wheel_thread.getData()

            image = capture_image_thread.buffer.get()
            time = datetime.datetime.now()

            cv2.imshow("test", image)
            cv2.waitKey(1)

            # filename = str(time).replace(" ", "") + "_" + str(steering_angle)
            # filename = filename.replace(".", "_") + ".jpg"
            # # print(os.path.join(folderName, filename))
            # cv2.imwrite(os.path.join(folderName, filename), image)

        except KeyboardInterrupt:
            print('Closing Program ...')
            steering_wheel_thread.stoppingFlag = True
            capture_image_thread.stoppingFlag = True
            steering_wheel_thread.join()
            capture_image_thread.join()
            exit()

if __name__ == '__main__':
    main()


# 代码3-8 菜品盈利帕累托图

# 菜品盈利数据 帕累托图
import pandas as pd
import matplotlib.pyplot as plt

plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签
plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号

# 初始化参数
dish_profit = '../data/catering_dish_profit.xls'  # 餐饮菜品盈利数据
data = pd.read_excel(dish_profit, index_col='菜品名')

data = data['盈利']
data.sort_values(ascending=False)  # False降序排列
p = 1.0 * data.cumsum() / data.sum()

plt.ylabel('盈利（元）')
data.plot(kind='bar')  # kind='bar'直方图,默认折线图

p.plot(color='r', secondary_y=True, style='-o', linewidth=2)
plt.ylabel('盈利（比例）')
plt.annotate(format(p[6], '.4%'), xy=(6, p[6]), xytext=(6 * 0.9, p[6] * 0.9),
             arrowprops=dict(arrowstyle="->", connectionstyle="arc3,rad=.2"))  # 添加注释，即85%处的标记。这里包括了指定箭头样式。
plt.show()


import cloudscraper
from bs4 import BeautifulSoup

url = 'https://ksa.yallamotor.com/used-cars/search?page={}&sort=updated_desc'
arts = 0

for page in range(1 , 501):
    print("Scrapping page number:", page)

    scraper = cloudscraper.create_scraper()  # returns a CloudScraper instance
    data = scraper.get(url.format(page) , timeout=300)
    soup = BeautifulSoup(data.content, "html.parser")
    links = soup.find_all("a", {"class" : "black-link"}, href=True)

    arts += len(links)
    print(arts)


import mido

from player import Player
from utils import builder
from utils import commander
from effect import soma

midfile = mido.MidiFile("./1.mid")
tracks = midfile.tracks

# 1 beat = 20 gt

ticksPerBeat = midfile.ticks_per_beat
beatsPerMinute = 85

Msglists = []

toFind = []

def findNote(type,msg,ct,notelist):
    global toFind
    global ticksPerBeat
    if type == "on":
        toFind.append(note(msg.note,int(round(ct/ticksPerBeat*20)),msg.velocity))
    if type == "off":
        for i , Note in enumerate(toFind):
            if msg.note == Note.note:
                if ct/ticksPerBeat - Note.tick/20 > 1:

                    Note.p = ""
                    notelist.append(Note)
                else:

                    Note.p = "c"
                    notelist.append(Note)
                del toFind[i]

class note:
    def __init__(self,n,t,v) -> None:
        self.note = n
        self.tick = t
        self.velocity = v / 127
        self.p = ""

class Msglist:
    # 通道 乐器
    def __init__(self,channel,program=1,noteList=None) -> None:
        if noteList is None:
            self.note_list = []
        else:
            self.note_list = noteList
        self.channel = channel
        self.program = program

for i,track in enumerate(tracks):
    currentTick = 0
    msglist = Msglist(i,program=1)
    for msg in track:
        currentTick = currentTick + msg.time
        if msg.is_meta:
            pass
        else:
            if msg.type == "program_change":
                Msglist.program = msg.program + 1
            if msg.type == 'note_on':
                gt = currentTick / ticksPerBeat * 20
                if gt - int(round(gt)) < 0 or gt - int(round(gt)) > 0:
                    print(f'warning gt is {gt}')
                # add to tofind list when it turn off it will write to Msglist
                findNote("on",msg,currentTick,msglist.note_list)
            if msg.type == "note_off":
                findNote("off",msg,currentTick,msglist.note_list)
    if len(msglist.note_list) > 0:
        Msglists.append(msglist)

cPlayer = Player(Msglists)
cEffect = soma(Msglists,1)
b= cEffect.load()
a = cPlayer.load()

finalList = commander.merge(a,b)
build = builder(finalList,-5,64)
build.writeBuildList()

print()


import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
from tensorflow import keras
from keras.utils import plot_model
from keras.preprocessing.sequence import pad_sequences
from sklearn.preprocessing import LabelEncoder
from sklearn.utils.class_weight import compute_class_weight
from sklearn.metrics import roc_auc_score
from scipy.io import loadmat
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import StratifiedKFold
from scipy import optimize
import random

random.seed(86)

STEP = 5000

def load_challenge_data(filename):
    x = loadmat(filename)
    data = np.asarray(x['val'], dtype=np.float64)
    new_file = filename.replace('.mat','.hea')
    input_header_file = os.path.join(new_file)
    with open(input_header_file,'r') as f:
        header_data=f.readlines()
    return data, header_data

def import_key_data(path):
    labels=[]
    ecg_filenames=[]
    for subdir, dirs, files in sorted(os.walk(path)):
        for filename in files:
            filepath = subdir + os.sep + filename
            if filepath.endswith(".mat"):
                data, header_data = load_challenge_data(filepath)
                labels.append(header_data[15][5:-1])
                ecg_filenames.append(filepath)
    return labels, ecg_filenames

def random_mix_12lead(signal):
  """
  SSL Approach 1: Mixing the channels of ECG
  """
  order = np.arange(12)
  np.random.shuffle(order)
  return signal[:, order]

def split_join_12lead(signal, no_split=2):

  new_signal = np.copy(signal)
  order = np.arange(12)
  np.random.shuffle(order)
  # pick how many channels to split and join
  no_channels = np.random.randint(0, 12, size=1)[0]
  for i in order[0:no_channels]:
    new_signal[:,i] = np.hstack(np.split(new_signal[:,i], no_split)[::-1])
  return new_signal

def SSL_batch_generator(ecg_filenames):
  while True:
    random.shuffle(ecg_filenames)
    for i in range(ecg_filenames.shape[0]):
      data, header_data = load_challenge_data(ecg_filenames[i])
      data = pad_sequences(data, maxlen=STEP, truncating='post', padding="post")

      signal_1 = data.T
      signal_2 = random_mix_12lead(signal_1)
      signal_3 = split_join_12lead(signal_1, no_split=2)
      yield np.stack((signal_1,signal_2,signal_3), axis = 0)


"""Tests for concepts data service"""

import pytest
import datetime
from ideabank_webapi.services import ConceptsDataService
from ideabank_webapi.models.artifacts import FuzzyOption

def test_account_has_all_parent_class_attributes():
    service = ConceptsDataService()
    expected = ['_query_buffer', '_query_results', '_session', '_s3_client']
    assert all(attr in service.__dict__ for attr in expected)

def test_concept_creation_query_builds():
    stmt = ConceptsDataService.create_concept(
            title='atitle',
            author='anauthor',
            description='adescription',
            diagram={'key': 'value'}
            )
    assert str(stmt) == 'INSERT INTO concepts ' \
                        '(title, author, description, diagram, created_at, updated_at) VALUES ' \
                        '(:title, :author, :description, :diagram, :created_at, :updated_at) ' \
                        'RETURNING concepts.identifier'

def test_concept_find_query_builds():
    stmt = ConceptsDataService.find_exact_concept('atitle', 'anauthor')
    assert str(stmt) == 'SELECT concepts.author, concepts.title, concepts.description,' \
                        ' concepts.diagram \n' \
                        'FROM concepts \nWHERE concepts.title = :title_1 AND concepts.author = :author_1'

def test_concept_linking_query_builds():
    stmt = ConceptsDataService.link_existing_concept('parentid', 'childid')
    assert str(stmt) == 'INSERT INTO concept_links (ancestor, descendant) ' \
                        'VALUES (:ancestor, :descendant) ' \
                        'RETURNING concept_links.ancestor, concept_links.descendant'

@pytest.mark.parametrize("fuzzy", FuzzyOption)
def test_concept_query_result_builds(fuzzy):
    stmt = ConceptsDataService.query_concepts(
            'atitle',
            'anauthor',
            datetime.datetime.utcnow(),
            datetime.datetime.utcnow(),
            fuzzy
            )
    if fuzzy == FuzzyOption.ALL:
        assert str(stmt) == 'SELECT concepts.identifier \n' \
                            'FROM concepts \n' \
                            'WHERE concepts.author LIKE :author_1 AND ' \
                            'concepts.title LIKE :title_1 AND ' \
                            'concepts.updated_at > :updated_at_1 AND ' \
                            'concepts.updated_at < :updated_at_2'
    elif fuzzy == FuzzyOption.TITLE:
        assert str(stmt) == 'SELECT concepts.identifier \n' \
                            'FROM concepts \n' \
                            'WHERE concepts.author = :author_1 AND ' \
                            'concepts.title LIKE :title_1 AND ' \
                            'concepts.updated_at > :updated_at_1 AND ' \
                            'concepts.updated_at < :updated_at_2'
    elif fuzzy == FuzzyOption.AUTHOR:
        assert str(stmt) == 'SELECT concepts.identifier \n' \
                            'FROM concepts \n' \
                            'WHERE concepts.author LIKE :author_1 AND ' \
                            'concepts.title = :title_1 AND ' \
                            'concepts.updated_at > :updated_at_1 AND ' \
                            'concepts.updated_at < :updated_at_2'
    else:
        assert str(stmt) == 'SELECT concepts.identifier \n' \
                            'FROM concepts \n' \
                            'WHERE concepts.author = :author_1 AND ' \
                            'concepts.title = :title_1 AND ' \
                            'concepts.updated_at > :updated_at_1 AND ' \
                            'concepts.updated_at < :updated_at_2'

def test_concept_find_children_query_build():
    stmt = ConceptsDataService.find_child_ideas('testuser/sample-idea', 5)
    assert str(stmt) == 'WITH RECURSIVE anon_1(descendant, ancestor, depth) AS \n' \
                        '(SELECT concept_links.descendant AS descendant, concept_links.ancestor AS ancestor, :param_1 AS depth \n' \
                        'FROM concept_links \n' \
                        'WHERE concept_links.ancestor = :ancestor_1 ' \
                        'UNION ALL ' \
                        'SELECT concept_links.descendant AS descendant, concept_links.ancestor AS ancestor, anon_1.depth + :depth_1 AS anon_2 \n' \
                        'FROM concept_links, anon_1 \n' \
                        'WHERE concept_links.ancestor = anon_1.descendant)\n ' \
                        'SELECT anon_1.ancestor, anon_1.descendant \n' \
                        'FROM anon_1 \n' \
                        'WHERE anon_1.depth <= :depth_2 ORDER BY anon_1.depth'

def test_concept_find_parents_query_build():
    stmt = ConceptsDataService.find_parent_ideas('testuser/sample-idea', 5)
    assert str(stmt) == 'WITH RECURSIVE anon_1(descendant, ancestor, depth) AS \n' \
                        '(SELECT concept_links.descendant AS descendant, concept_links.ancestor AS ancestor, :param_1 AS depth \n' \
                        'FROM concept_links \n' \
                        'WHERE concept_links.descendant = :descendant_1 ' \
                        'UNION ALL ' \
                        'SELECT concept_links.descendant AS descendant, concept_links.ancestor AS ancestor, anon_1.depth + :depth_1 AS anon_2 \n' \
                        'FROM concept_links, anon_1 \n' \
                        'WHERE concept_links.descendant = anon_1.ancestor)\n ' \
                        'SELECT anon_1.ancestor, anon_1.descendant \n' \
                        'FROM anon_1 \n' \
                        'WHERE anon_1.depth <= :depth_2 ORDER BY anon_1.depth'


import json
import time
import base64
import hashlib
from datetime import datetime
from urllib import request, parse
from urllib.error import HTTPError

from apub import signatures, utils

def get(url):
    url = utils.trim_frag(url)
    req = request.Request(url, headers={
        'Accept': 'application/json',
        'User-Agent': 'sns-to-activitypub/1',
    })
    try:
        response = request.urlopen(req)
        return json.load(response)
    except HTTPError as ex:
        print(ex.headers)
        print(ex.read())
        raise

GET_CACHE = {}

def get_cached(url, max_age=60.0):
    """Get, but with extra cache flavor"""
    url = utils.trim_frag(url)
    if url in GET_CACHE:
        if time.time() < GET_CACHE[url]['requested'] + max_age:
            return GET_CACHE[url]['json']

    response = get(url)
    GET_CACHE[url] = {
        'json': response,
        'requested': time.time()
    }
    return response

def post(url, body):
    print('POST to', url)
    print(json.dumps(body))

    parsed_url = parse.urlparse(url)
    data = json.dumps(body).encode()
    sha = hashlib.sha256(data)

    headers = {
        'User-Agent': 'sns-to-activitypub/1',
        'Accept': 'application/json',
        'Content-Type': 'application/activity+json',
        'Digest': f'SHA-256={base64.b64encode(sha.digest()).decode()}',
        'Date': datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'),
        'Host': parsed_url.hostname,
    }
    headers = signatures.create_signature_header(headers, parsed_url.path)
    print(headers)

    req = request.Request(url, data=data, headers=headers)
    response = request.urlopen(req)
    print(response.status, response.reason)
    if response.status == 200:
        r = json.load(response)
        print('Response:')
        print(json.dumps(r))
        return r
    else:
        return {}


import codecs
import socket
import threading
from base64 import b64decode
from Crypto.Cipher import AES

K3 = 'FE8CBA98765432100123456789ABC7EF'

HOST_B = 'localhost'
PORT_B = 9997

HOST_KM = 'localhost'
PORT_KM = 9991

iv = bytearray(16)

file_to_read = ''

def pad(text):
    if len(text) % 16 == 0:
        return text
    data = text + (16 - (len(text) % 16)) * b'\x00'
    return data

# IMPLEMENTARE ECB
# impartim textul in blocuri de dimensiune 16 si criptam fiecare bloc cu aceeasi cheie
def encrypt_text_ECB(text, key, s):
    text = pad(text)
    cipher = AES.new(key.encode('utf8'), AES.MODE_ECB)
    for bloc in range(len(text) // 16 ):
        block_enc = cipher.encrypt(text[bloc * 16:(bloc + 1) * 16])
        s.sendall(codecs.encode(block_enc, 'base64'))

def XOR(input_bytes, key_input):
    index = 0
    output_bytes = b''
    for byte in input_bytes:
        if index >= len(key_input):
            index = 0
        output_bytes += bytes([byte ^ key_input[index]])
        index += 1
    return output_bytes

# IMPLEMENTARE OFB
def encrypt_text_OFB(text, key, s):
    global iv
    text = pad(text)
    cipher = AES.new(key.encode('utf8'), AES.MODE_ECB)
    iv = '0' * 16
    for b in range(len(text) // 16 ):
        iv = cipher.encrypt(iv)
        enc = XOR(text[b * 16: (b + 1) * 16], iv)
        s.sendall(codecs.encode(enc, 'base64'))

def send_text(s, key_decrypted):
    global file_to_read
    with open(file_to_read, "rb") as fd:
        text = fd.read()
        if operation_mode == "ECB":
            encrypt_text_ECB(text, key_decrypted, s)
        elif operation_mode == "OFB":
            encrypt_text_OFB(text, key_decrypted, s)

# Stim ca avem chei de dimensiune fixa asa ca nu mai impartim pe blocuri doar decriptam(implementare pentru decriptare ECB cu mai multe blocuri in nodul B)
def decrypt_key_ECB(key):
    return AES.new(K3.encode('utf8'), AES.MODE_ECB).decrypt(b64decode(key)).decode('utf8')

# Stim ca avem chei de dimensiune fixa asa ca nu mai impartim pe blocuri doar decripta(implementare pentru decriptare OFB cu mai multe blocuri in nodul B)
def decrypt_key_OFB(key):
    iv = '0' * 16
    cipher = AES.new(K3.encode('utf8'), AES.MODE_ECB)
    iv = cipher.encrypt(iv)
    decrypt = XOR(b64decode(key), iv)
    return decrypt.decode('utf8')

def receive_KM(s_km, s):
    while True:
        received = s_km.recv(1024)
        received = received.decode()
        if not received:
            break
        else:
            # print(received)
            key = received.split()[-1]
            if operation_mode == "ECB":
                key_decrypted = decrypt_key_ECB(key)
                # print("Key Decrypted: {}".format(key_decrypted))
                send_text(s, key_decrypted)
            else:
                key_decrypted = decrypt_key_OFB(key)
                # print("Key Decrypted: {}".format(key_decrypted))
                send_text(s, key_decrypted)

def receive_B(s):
    global start_communication
    while True:
        try:
            r_msg = s.recv(1024).decode()
            if not r_msg:
                break
            else:
                if "Start" in r_msg:
                    start_communication = 1
        except:
            raise
            break

def send_msg(s, s_km):
    global file_to_read
    global operation_mode
    while True:
        operare = input("Introdu modul de operare(ECB sau OFB)")
        file_to_read = input("Introdu numele fisierului pe care incerci sa il criptezi: ")
        if operare == '':
            pass
        else:
            if operare.upper().strip() == "ECB" or operare.upper().strip() == "OFB":
                operation_mode = operare
                try:
                    s.sendall("[A] [MODE]: {}".format(operare).encode())
                    s_km.sendall("[A] [KEY-A]: {}".format(operare).encode())
                except:
                    raise
            else:
                print("Modul de criptare introdu nu exista incearca iar")

if __name__ == '__main__':
    start_communication = 0
    operation_mode = ''

    socket_B = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    socket_KM = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    socket_B.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    socket_KM.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    socket_B.connect((HOST_B, PORT_B))
    socket_KM.connect((HOST_KM, PORT_KM))

    thread1 = threading.Thread(target=receive_B, args=([socket_B]))
    thread2 = threading.Thread(target=receive_KM, args=([socket_KM, socket_B]))
    thread3 = threading.Thread(target=send_msg, args=([socket_B, socket_KM]))
    try:
        thread1.start()
        thread2.start()
        thread3.start()
    except:
        thread1.join()
        thread2.join()
        thread3.join()
        socket_B.close()
        socket_KM.close()


"""
E2 Manager (Control, E2 Setup, indication)
"""
import os
import sys
import logging
import time, threading
import json
from ricxappframe.xapp_frame import RMRXapp
from ricsdl.syncstorage import SyncStorage
from ricsdl.exceptions import RejectedByBackend, NotConnected, BackendError
from ricxappframe.rmr import rmr
from influxdb import InfluxDBClient
import tracemalloc
import gc

os.environ['DBAAS_SERVICE_PORT'] = '6379'
os.environ['DBAAS_SERVICE_HOST'] = '10.0.2.15'#'10.244.0.18' #'140.123.102.112'
os.environ['RMR_SEED_RT'] = 'test_route_thrd.rt'
influxdbIP = '10.98.0.0'
influxdbPort = '8086'

count=0
# Constants used in the examples below.
#MY_NS = 'E2setup'
MY_GRP_NS = 'my_group_ns'
MY_LOCK_NS = 'my_group_ns'

set_list = list()
RAN_fun_id_list = list()
Temporary_storage_E2setup = list()
Temporary_storage_Report = list()
Temporary_storage_Insert = list()

def createTimer():
    t = threading.Timer(2, send_broadcast)
    t.start()

def send_broadcast():
    global q
    createTimer()
    size = q.qsize()
    if size > 0:
        pass

# database
def _try_func_return(func):
    try:
        return func()
    except RejectedByBackend as exp:
        #print(f'SDL function {func.__name__} failed: {str(exp)}')
        #raise
        pass
    except (NotConnected, BackendError) as exp:
        pass
        #print(f'SDL function {func.__name__} failed for a temporal error: {str(exp)}')

# initial E2 Manager
def post_init(_self):
    print("***********************************")
    print("E2 Manager born up!!!\n")
    print("E2 Manager start to work~~YA\n")
    print("***********************************\n")
    #createTimerForInflux()

    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True
    #print("***********************************")
    #print("E2 Manager store route table from DB\n")
    my_ret_dict = _try_func_return(lambda: mysdl.get('Routing', {str('table')}))
    if(len(my_ret_dict)!=0):
        f = open("test_route_thrd.rt", 'w+')
        f.write('newrt|start\n')
        data = eval(my_ret_dict['table'].decode())
        for key in data.keys():
            ip = data[key]['IP']
            port = str(data[key]['port'])
            messagetype = str(data[key]['massage_type'])+port
            f.write('rte|'+messagetype+'|'+ip+':'+port+'\n')
        f.write('newrt|end\n')
        f.seek(0)
        f.close()
        #print("Write in routing table succussful!!!")

# saving logging
def save_list():
    file_name = 'E2data.txt'
    f = open(file_name, "w+")
    for item in set_list:
        f.write(item)
    f.close()

def get_Requestor_ID(jpay):
    try:
        xApp_port = jpay['RIC Request ID']['RIC Requestor ID']
        return xApp_port
    except:
        #print("There's no RIC ID element")
        return None

""" Procedure Function Section """
# ************************************************

def error_empty_tm(self, data, sbuf):
    #print("****Error occurred****\n")
    #print("The payload is", data, "\n")
    val = json.dumps({
        "RAN Function ID": 1,
        "Cause": "Unspecified",
        "Procedure Code": 2
    }).encode()
    #entry(self, sbuf, val, 7404000)
    self.rmr_send(val, int('7404000'))
# Check Error Indication(xApp)
def error_empty_xapp(self, data, xApp_port):
    #print("****Error occurred****\n")
    #print("The payload is empty\n")

    val = json.dumps(data).encode()

    self.rmr_send(val, int("740"+str(xApp_port)))

# E2 Setup Failure
def e2_setup_failure(self, sbuf, SocketFD):
    #print("E2 Setup Failure!!\n")
    #print("Send 204 back to E2 Termination\n")
    val = json.dumps({
        "Cause": "Unspecified",
        "Procedure Code": 1,
        "Type of Message": 2,
        "SocketFD": SocketFD
    }).encode()
    #entry(self, sbuf, val, 2044000)
    self.rmr_send(val, int('2004000'))

# **************************************************

# Get SocketFD
def get_SocketFD(jpay):
    try:
        SocketFD = jpay['SocketFD']
        return SocketFD
    except:
        #print("There's no SocketFD element")
        return None

# Check Failure
def failure_check(self, sbuf, data, procedure, SocketFD):

    if procedure == "setup":
        if data['E2 Node ID'] == "" or data['RAN Function ID'] == "" or data['RAN Function Definition'] == "" or data['RAN Function Revision'] == "":
            e2_setup_failure(self, sbuf, SocketFD)
            return False

        for i in range(len(RAN_fun_id_list)):
            if data['RAN Function ID'] == RAN_fun_id_list[i]:
                e2_setup_failure(self, sbuf, SocketFD)
                return False

        return True

    elif procedure == "control":
        check_SocketFD = -1

        if data['RAN Function ID'] == "" or data['RIC Control Header'] == "" or data['RIC Control Message'] == "":
            error_empty_xapp(self, data, get_Requestor_ID(data))
            return False, check_SocketFD
        """else:
            return True, check_SocketFD"""

        # check SockFD's correctness
        mysdl = _try_func_return(SyncStorage)
        is_active = mysdl.is_active()
        assert is_active is True

        xApp_ID = int(data['RIC Request ID']['RIC Requestor ID'])
        group_index = str((xApp_ID - 8000)//66)
        warn_msg = 'This port is irregularity.'
        white_text = eval(_try_func_return(lambda: mysdl.get('White', {str('text')}))['text'].decode())
        try:
            white_text = white_text[group_index]
            #print('group_index:', group_index, ' white_text: ', white_text)
            if(xApp_ID not in white_text['listport']):
                #print('warn_msg 1: ', warn_msg) # This port is not in the group
                return False, check_SocketFD
            if(data['SocketFD'] != white_text['SocketFD']): # The SocketFD is wrong
                check_SocketFD = white_text['SocketFD']
        except:
            #print('warn_msg 2: ', warn_msg) # This port is not in the white text table
            return False, check_SocketFD
        return True, check_SocketFD

# write in local routing table
def write_routing_table(NameSpace, Key):
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    #print("Write xApp information into routing table!!!\n")

    f = open("test_route_thrd.rt", 'w+')
    f.write('newrt|start\n')
    data = eval(_try_func_return(lambda: mysdl.get(
        NameSpace, {str(Key)}))[Key].decode())
    for key in data.keys():
        ip = data[key]['IP']
        port = str(data[key]['port'])
        messagetype = str(data[key]['massage_type'])+port
        f.write('rte|'+messagetype+'|'+ip+':'+port+'\n')
    f.write('newrt|end\n')
    f.seek(0)
    # txt=f.read()
    # #print(txt)
    f.close()
    #print("Write in routing table succussful!!!")
    #print("Update after 60s.\n")

def Request_Report(self, RIC_ID, SocketFD, xappIP):
    # xApp needs to get report data from E2 manager.
    # connect to influxdb
    client = InfluxDBClient(influxdbIP, influxdbPort, 'root', '', 'reportdb')
    tables = 'report_' + str(RIC_ID) + str(SocketFD)

    # get data from influxdb
    result = client.query('select * from ' + tables)
    ##print("influxdb data:", list(result.get_points()))
    report_data = list(result.get_points())
    # send to xApp
    val = json.dumps(
        {"ACK": RIC_ID, "Report Data": report_data}).encode()

    try:
        #print("870 Time:",time.ctime())
        #print("sdl ip:",RIC_ID)
        hostname = xappIP
        response = os.system("ping -c 1 " + hostname)
        #and then check the response...
        if response == 0:
            #print("UP Time:",time.ctime())
            self.rmr_send(val, int("871"+str(RIC_ID)))
        else:
            pass
            #print('down Time',time.ctime())
    except:
        #print("870 else")
        pass

    #self.rmr_send(val, int('871'+str(RIC_ID)))

# Send to E2 termination
def Send2E2T(self, summary, sbuf):
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    jpay = json.loads(summary['payload'])
    message = str(summary['message type'])
    message = int(message[0:3])
    SocketFD = get_SocketFD(jpay)

    # Control Request
    if message == 610:
        """
        mysdl = _try_func_return(SyncStorage)
        is_active = mysdl.is_active()
        assert is_active is True
        """
        if len(str(SocketFD))<5:
            pass
        else:
            #self.logger.info("E2 Manager 610 handler called!")
            print("Control Request: ", time.time())
            #print("Got the 610 Control Request!!\n")
            check, check_SocketFD = failure_check(self, sbuf, jpay, "control", SocketFD)
            if(check_SocketFD != -1):
                jpay['SocketFD'] = check_SocketFD
            if check:
                jpay['SocketFD']= int(_try_func_return(lambda: mysdl.get(str(jpay['SocketFD']), {str('SocketFD')}))['SocketFD'].decode())
                #print("610 payload:", jpay, "\n")
                val = json.dumps(jpay).encode()
                self.rmr_send(val, int('6004000'))
                #entry(self, sbuf, val, 6004000)
                #print("Send it\n")

    self.rmr_free(sbuf)
"""
def report_thread(self, xappID, xApp_port)
    t = threading.Timer(1, send_broadcast)
    t.start()
"""
"""
def report_thread_function(self, xappID, xApp_port, val):
    try:
        #print("Time:",time.ctime())
        #print("sdl ip:",xappID)
        hostname = xappID
        response = os.system("ping -c 1 " + hostname)

        #and then check the response...
        if response == 0:
            #print("UP Time:",time.ctime())
            self.rmr_send(val, int("700"+str(xApp_port)))
        else:
            pass
            #print('down Time',time.ctime())

    except:
        #print("else")
        pass

    #global q
    #createTimer()
    #size = q.qsize()
    #if size > 0:
    #    new = RMRXapp(default_handler=Routing, rmr_port=4601, post_init=send, use_fake_sdl=False)
    #    new.run()
"""

global eport_time_before
eport_time_before = 0
sum_report_time=0
report_time=0
# Send to xApp
def Send2xApp(self, summary, sbuf):
    global xApp_port,sum_report_time,report_time
    """global count"""
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    #tracemalloc.start()

    jpay = summary['payload']
    message = str(summary['message type'])
    message = int(message[0:3])
    print("700 jpay:",jpay)
    data = jpay.decode('utf-8')

    if data != "":
        #print("Data is OK\n")
        payload = data[data.index("{") + 1:data.rindex("}")]
        payload = "{" + payload + "}"
        #print("data:", payload, "type:", type(payload))
        #print("##message:",message)
        #print("\njpay",jpay)

        dataSize = sys.getsizeof(jpay)
        jpay = json.loads(payload)

        xApp_port = get_Requestor_ID(jpay)

        ##print("jpay['Procedure Code']:",jpay['Procedure Code'])
        ##print("jpay['RIC Indication Message']:",jpay['RIC Indication Message)'])

        #dataSize = sys.getsizeof(jpay)
        #jpay = json.loads(payload)

        #xApp_port = get_Requestor_ID(jpay)

        # Control Response
        if message == 601:
            #self.logger.info("E2 Manager 601 Response!")
            print("Receive Control ACK: ", time.time())
            #print("Got the 601 Control Response (ACK)!!\n")
            #print("601 Payload:", jpay, "\n")
            jpay = jpay['RIC Control Status']
            if jpay == 0:
                jpay='Success'
            else:
                jpay='False'
            """
            xApp_ID = int(jpay['RIC Request ID']['RIC Requestor ID'])
            group_index = str((xApp_ID - 8000)//66)
            white_text = eval(_try_func_return(lambda: mysdl.get('White', {str('text')}))['text'].decode())
            #RequestFD = white_text[group_index]['SocketFD']
            jpay['SocketFD']= white_text[group_index]['SocketFD']
            """
            val = json.dumps(jpay).encode()
            #print("601 Payload val#@:",val)

            messagetype = int("611"+str(xApp_port))
            self.rmr_send(val, messagetype)
            print("Send Control ACK: ", time.time())

        # Control Failure
        if message == 604:
            self.logger.info("E2 Manager 604 Control Failure!")
            #print("Got the 604 Control Failure!!\n")
            #print("604 Received:", jpay, "\n")

            #print("Send 614 Control Failure to xApp")
            val = json.dumps(jpay).encode()
            self.rmr_send(val, int("614"+str(xApp_port)))

        # Indication Report
        if message == 700:
#            global report_time_before
#            try:
#                report_time_after = time.time()
#                #print("report_time_after is OK !!!")
#                report_time = report_time_after - report_time_before
#                #print("report_time is OK !!!")
#                report_time_before = report_time_after
#                #print("report_time_before is OK !!!")
#            except:
#                print("Enter except")
#                report_time_before = time.time()
#                report_time = 0
            #if count==250:
            #os.execl(sys.executable, sys.executable, *sys.argv)
#            self.logger.info("\n\nGot 700 Indication Report! \nTime: " + str(report_time))
#            print("Reporting period: ", report_time)
            #print("Got the 700 Indication Report!!\n")
            #print("Receive report: ", time.time())
            Receive_report = time.time()
            #print("700 Received:", jpay, "\n")
            #print('Size of integer:', dataSize)

            """val = json.dumps({
                "RIC Requestor ID": jpay['RIC Request ID']['RIC Requestor ID'],
                "paysize": dataSize
            }).encode()

            self.rmr_send(val, 67006000)
            Temporary_storage_Report.append(jpay)"""
            #report_influxdb = dbreport(jpay, message)
            store_thread = threading.Thread(target=dbreport, args=(jpay, message,))
            store_thread.start()
            #print("Send 700 Indication Report to xApp")
            InicationMs=jpay['RIC Indication Message']
            #InicationMs.update({'Time':time.time()})
            val = json.dumps(InicationMs).encode()

            #my_ret_dict = _try_func_return(lambda: mysdl.get(str('connecter'), {str(xApp_port)}))
            ##xappID = list(_try_func_return(lambda: mysdl.get_members('connecter', str(jpay['RIC Request ID']['RIC Requestor ID']))))
            #xappID = self.sdl_get('connecter', str(jpay['RIC Request ID']['RIC Requestor ID'])) # '10.0.2.15'

            #report_thread(self, xappID, xApp_port)
            """
            report_thread = threading.Timer(1, report_thread_function)
            report_thread.start()
            """
#            report_thread = threading.Thread(target=report_thread_function, args=(self, xappID, xApp_port, val))
#            report_thread.start()
            self.rmr_send(val, int("700"+str(xApp_port)))

            report_time = report_time + 1
            sum_report_time = sum_report_time+time.time()-Receive_report
            #print("Send report: ", time.time())
            print("average Receive_report->Send report: ", sum_report_time/report_time)
            del val
            gc.collect()
            #collect_stats = gc.get_stats()
            #print("collect stats = ", collect_stats)

            store_thread.join()
            """
            try:
                #print("Time:",time.ctime())
                #print("sdl ip:",xappID)
                hostname = xappID
                response = os.system("ping -c 1 " + hostname)
                #and then check the response...
                if response == 0:
                    #print("UP Time:",time.ctime())
                    self.rmr_send(val, int("700"+str(xApp_port)))
                else:
                    pass
                    #print('down Time',time.ctime())

            except:
                #print("else")
                pass
            """
            #self.rmr_send(val, int("700"+str(xApp_port)))
            #print("@@@count:",count)
            """count +=1"""
        # Indication Insert
        if message == 710:
            self.logger.info("Got 710 Indication Insert!")
            #print("Got 710 Indication Insert!!\n")
            #print("710 Received:", jpay, "\n")

            """val = json.dumps({
                "RIC Requestor ID": jpay['RIC Request ID']['RIC Requestor ID'],
                "paysize": dataSize
            }).encode()

            self.rmr_send(val, 67106000)
            Temporary_storage_Insert.append(jpay)"""
            #report_influxdb = dbreport(jpay, message)

            #print("Send 710 Indication Insert to xApp")
            val = json.dumps(jpay).encode()
            self.rmr_send(val, int("710"+str(xApp_port)))

        # Indication Insert
        if message == 720:
            self.logger.info("Got 720 E2 Node Disconnected!")
            #print("Got 720 E2 Node Disconnected!!\n")
            #print("720 Received:", jpay, "\n")
            val = json.dumps(jpay).encode()
            self.rmr_send(val, int("720"+str(xApp_port)))

        # Error Indication
        if message == 740:
            self.logger.info("Got 740 Error Indication !")
            #print("Got 740 Error Indication !!\n")
            #print("740 Received:", jpay, "\n")

            #print("Send 740 Error Indication to xApp")
            val = json.dumps(jpay).encode()

            self.rmr_send(val, int("710"+str(xApp_port)))

        # Report Query
        if message == 870:
            self.logger.info("Got 870 Report Query!")
            #print("Got 870 Report Query!!\n")
            #print("870 Received:", jpay, "\n")
            #SocketFD = jpay['SocketFD']
            xApp_port_SPC = jpay['RIC Requestor ID']#xApp_ID
            xappIP = self.sdl_get('connecter', str(jpay['RIC Requestor ID']))

            """
            group_index = str((xApp_port_SPC - 8000)//66)
            white_text = eval(_try_func_return(lambda: mysdl.get('White', {str('text')}))['text'].decode())
            SocketFD = white_text[group_index]['SocketFD']
            """

            #query_report_thread = threading.Thread(target=query_report_thread_function, args=(self, xApp_port_SPC, jpay))
            #query_report_thread.start()

            try:
                SocketFD = _try_func_return(lambda: mysdl.get(jpay['SocketFD'], {str('SocketFD')}))['SocketFD'].decode()
                Request_Report(self, xApp_port_SPC, SocketFD, xappIP)
            except:
                pass
                #print("\n")

        self.rmr_free(sbuf)

    else:
        error_empty_tm(self, "empty", sbuf)
    """
    snapshot = tracemalloc.take_snapshot()
    top_stats = snapshot.statistics('lineno')

    for stat in top_stats:
        print(stat)

    tracemalloc.stop()
    """

# Default Setting
def e2_process(self, summary, sbuf):
    jpay = summary['payload']
    message = str(summary['message type'])
    message = int(message[0:3])
    data = jpay.decode('utf-8')

    if data != "":
        #print("Data is OK\n")
        payload = data[data.index("{") + 1:data.rindex("}")]
        payload = "{" + payload + "}"
        dataSize = sys.getsizeof(jpay)
        jpay = json.loads(payload)
        SocketFD = get_SocketFD(jpay)

        # Recieve E2 setup request --> Response 201
        if message == 200:
            mysdl = _try_func_return(SyncStorage)
            is_active = mysdl.is_active()
            assert is_active is True

            #self.logger.info("Got 200 E2 Setup Request!")
            #print("Got 200 E2 Setup Request!!")
            print("200 Received:", jpay, "\n")
            #print("payload length:", len(data))
            #print('size of integer:', sys.getsizeof(jpay))
            dataSize = sys.getsizeof(data)

            #check = failure_check(self, sbuf, jpay, "setup", SocketFD)
            check = True
            if check:
                val = json.dumps({
                    "E2 Node ID": jpay['E2 Node ID'].replace(' ',''),
                    "paysize": dataSize,
                    "SocketFD":jpay['SocketFD']
                }).encode()
                #print("Send 6200 to RIC Redis Manager\n")
                self.rmr_send(val, 62006000)
                Temporary_storage_E2setup.append(jpay)
                #RAN_fun_id_list.append(jpay['RAN Function ID'])

        # Error indication
        elif message == 740:
            self.logger.info("Got 740 Error Indication!")
            #print("Got 740 Error Indication!\n")
            #print("740 Received:", jpay, "\n")

        # Service Update
        elif message == 800:
            self.logger.info("Got 800 Service Update!")
            #print("Got 800 Service Update!\n")
            #print("800 Received:", jpay, "\n")

            try:
                mode = jpay["RAN Functions Added"]
            except:
                try:
                    mode = jpay["RAN Functions Modified"]
                except:
                    mode = jpay["RAN Functions Deleted"]

            #print("Mode", mode, "\n")

            val = json.dumps(jpay).encode()
            #entry(self, sbuf, val, 8014000)
            self.rmr_send(val, int('8014000'))
        #Restart E2 manager
        elif message == 100:
            self.logger.info("Received the 1000 Routing Restart!!")
            #print("Received the 1000 Routing Restart!!\n")
            #print("1000 Received:", jpay, "\n")

            NS = jpay['Routing Table']['Namespace']
            key1 = jpay['Routing Table']['Key']

            write_routing_table(NS, key1)

        else:
            self.logger.info("Default Callback!")
            #print("Default handler received: {0}".format(summary))

        self.rmr_free(sbuf)

    else:
        #print("Received: {0}".format(summary))
        error_empty_tm(self, "empty", sbuf)
        self.rmr_free(sbuf)

#Send to E2 Termination
def entry(self, sbuf, data, message):

    # Init rmr
    mrc = rmr.rmr_init(b"9998", rmr.RMR_MAX_RCV_BYTES, 0x00)
    while rmr.rmr_ready(mrc) == 0:
        time.sleep(1)
        #print("Not yet ready")
    rmr.rmr_set_stimeout(mrc, 2)
    sbuf = rmr.rmr_alloc_msg(mrc, 1024)
    val=data

    try:
        val = data.encode('utf-8')

    except:
        val = data

    #print("val control request:",val)
    rmr.set_payload_and_length(val, sbuf)
    rmr.generate_and_set_transaction_id(sbuf)
    sbuf.contents.state = 0
    sbuf.contents.mtype = message
    #print("Send summary: {}".format(rmr.message_summary(sbuf)))
    sbuf = rmr.rmr_send_msg(mrc, sbuf)
    #print("Post send summary: {}".format(rmr.message_summary(sbuf)))
    #print("\n")

# Store the data into database
def db(self, summary, sbuf):
    mysdl = _try_func_return(SyncStorage)
    is_active = mysdl.is_active()
    assert is_active is True

    jpay = json.loads(summary['payload'])
    message = summary['message type']
    #print("Got the 6201 message!!!!!!!!!\n")
    #print("6201 Received:", jpay)

    # Wrong message
    if((message % 10) == 4):
        if(jpay['NAK'] == -2):
            #print("There is something wrong. Please check the RMR message type.")
            os._exit(0)
        elif(jpay['NAK'] == -1):
            #print("The memoory is full. Please wait.")
            os._exit(0)

    MY_NS = jpay['namespace']
    #print("MY_NS:", MY_NS)
    #_try_func_return(lambda: mysdl.set(str(jpay["E2 Node ID"].encode()), {'SocketFD': str(jpay["SocketFD"].encode())}))

    # Store data in redis
    try:
        tmpdata = Temporary_storage_E2setup.pop(0)
    except:
        #print("The Temporary_storage_E2setup is empty")
        return
    _try_func_return(lambda: mysdl.set(str(tmpdata["E2 Node ID"]).replace(' ','') , {'SocketFD': str(tmpdata["SocketFD"]).encode()}))
    #print("Send 201 back to E2 Termination\n")

    val = json.dumps({
        "Global RIC ID": 1,
        "RAN Function ID": tmpdata['RAN Function ID'],
        "Procedure Code": 1,
        "Type of Message": 1,
        "SocketFD":  tmpdata['SocketFD']
    }).encode()
    #print("to E2_agent val:",val)
    #entry(self, sbuf, val, 2014000)
    self.rmr_send(val, int('2014000'))
    print("to E2_agent val:",val)

    storedata = {}
    for col in jpay["columns"]:
        try:
            storedata.update({str(col): tmpdata[col]})
        except:
            pass
    storedata.update({'xAppID': 99999})
    storedata.update({"SocketFD":storedata['E2 Node ID'].replace(' ','')})
    store_data = str(storedata)
    byte_send_data = store_data.encode()
    _try_func_return(lambda: mysdl.set(
        MY_NS, {str(storedata['E2 Node ID'].replace(' ','')): byte_send_data}))
    #print("Store in Redis\n")

    # Get data in redis
    my_ret_dict = _try_func_return(lambda: mysdl.get(
        MY_NS, {str(storedata['E2 Node ID'].replace(' ',''))}))
    #print(MY_NS, " redis_get:", str(storedata['E2 Node ID'].replace(' ','')), "->", my_ret_dict)

    self.rmr_free(sbuf)

store_time=0
sum_time=0
#Report (database)
def dbreport(jpay, message):
    global client, store_time, sum_time
    start_ = time.time()
    print("**** Enter the dbreport ****")
    if(message == 700):
        #print("700 enter dbreport:", jpay)
        MY_NS = 'report'
        store_columns = ['RIC Request ID', 'RIC Instance ID', 'RAN Function ID', 'RIC Action ID', 'RIC Indication Type', 'RIC Indication Header', 'RIC Indication Message']
    elif(message == 710):
        #print("710 enter dbreport:", jpay)
        MY_NS = 'insert'
        store_columns = ['RIC Request ID', 'RIC Instance ID', 'RAN Function ID', 'RIC Action ID', 'RIC Indication Type', 'RIC Indication Header', 'RIC Indication Message', 'Call process ID']
    #print("MY_NS:", MY_NS)

    storedata = {}
    for col in store_columns:
        try:
            storedata.update({str(col): jpay[col]})
        except:
            pass

    store_data = str(storedata)
    #print("Store Data", storedata)

    # please decision that these still send to xApp? orange words under this line.
    """
    val = json.dumps({"ACK": storedata['RIC Request ID']['RIC Requestor ID'],
                      "RIC Requestor ID": storedata['RIC Request ID']['RIC Requestor ID'],
                      "namespace": MY_NS}).encode()
    self.rmr_send(val, 67006000)
    """

    # store data in influxdb

    #client = InfluxDBClient(influxdbIP, influxdbPort, 'root', '', 'reportdb')
    #print("database: ", client.get_list_database())  # 顯示所有數據庫名稱
    # client.create_database('testdb') # 創建數據庫
    # #print(client.get_list_database()) # 顯示所有數據庫名稱
    """
    retention_policy = 'exist_Time'
    try:
        client.create_retention_policy(retention_policy, '1h', 3, default=True)
    except:
        #print("Error:not successful create new retention policy.")
    """
    FD = jpay['SocketFD']
    tables = 'report_'+str(storedata['RIC Request ID']['RIC Requestor ID'])# + str(FD)

    # This block is for test
    #latency : ms, reliability : percentage, throughput : Gbps
    # procedure": "Indication Report"

    #print("storedata['RIC Request ID']['RIC Requestor ID']:",storedata['RIC Request ID']['RIC Requestor ID'])
    #print("type storedata['RIC Request ID']['RIC Requestor ID']:",storedata['RIC Request ID']['RIC Requestor ID'])
    json_body = []

    json_body.append(str(tables)+",procedure="+str(MY_NS)+" RIC_Requestor_ID="+str(storedata['RIC Request ID']['RIC Requestor ID'])+",RIC_Instance_ID="+str(storedata['RIC Request ID']['RIC Instance ID'])+",RAN_Function_ID="+str(storedata['RAN Function ID'])+",RIC_Action_ID="+str(storedata['RIC Action ID'])+",RIC_Indication_Type="+str(storedata['RIC Indication Type'])+",RIC_Indication_Header="+str(storedata['RIC Indication Header'])+",RIC_Indication_Message="+str(storedata['RIC Indication Message']))
    """
    json_body.append("{measurement},procedure={procedure} RIC_Requestor_ID={RIC_Requestor_ID},RIC_Instance_ID={RIC_Instance_ID},RAN_Function_ID={RAN_Function_ID},RIC_Action_ID={RIC_Action_ID},RIC_Indication_Type={RIC_Indication_Type},RIC_Indication_Header={RIC_Indication_Header},RIC_Indication_Message={RIC_Indication_Message}"
                .format(measurement='tables',
                        procedure='MY_NS',
                        RIC_Requestor_ID=int(storedata['RIC Request ID']['RIC Requestor ID']),
                        RIC_Instance_ID=int(storedata['RIC Request ID']['RIC Instance ID']),
                        RAN_Function_ID=int(storedata['RAN Function ID']),
                        RIC_Action_ID=int(storedata['RIC Action ID']),
                        RIC_Indication_Type=int(storedata['RIC Indication Type']),
                        RIC_Indication_Header=int(storedata['RIC Indication Header']),
                        RIC_Indication_Message=str(storedata['RIC Indication Message'])))
    """
    client.write_points(json_body, batch_size=5000, protocol='line',retention_policy=retention_policy)

    end_ = time.time()
    store_time = store_time + 1
    sum_time = sum_time + end_- start_
    print("average storedb time = ", sum_time/store_time)
    #print("-----------------store in influxdb------------------")
    #result = client.query('select * from ' + tables)
    #print("influxdb data:", list(result.get_points()))
    """
    # please decision that these still send to xApp? orange words under this line.
    if message == 6701:
        #print("Send 700 Indication Report to xApp")
        self.rmr_send(val, int("700"+str(xApp_port)))

    elif message == 6711:
        #print("Send 710 Indication Insert to xApp")
        self.rmr_send(val, int("710"+str(xApp_port)))
    return list(result.get_points())
    """

client = InfluxDBClient(influxdbIP, influxdbPort, 'root', '', 'reportdb')
client.create_database('reportdb')  # 創建數據庫
retention_policy = 'exist_Time'
try:
    client.create_retention_policy(retention_policy, '1h', 3, default=True)
except:
    pass
    #print("Error:not successful create new retention policy.")

xapp = RMRXapp(default_handler=e2_process, rmr_port=4200,post_init=post_init, use_fake_sdl=False)

#--------------------------------------------
xapp.register_callback(Send2E2T, 6104200)  # control request
xapp.register_callback(Send2xApp, 6014200)  # control response
xapp.register_callback(Send2xApp, 6044200)  # control failure
#--------------------------------------------
xapp.register_callback(Send2xApp, 7004200)  # report
xapp.register_callback(Send2xApp, 7104200)  # insert
#--------------------------------------------
xapp.register_callback(Send2xApp, 7004200)  # report
xapp.register_callback(Send2xApp, 7204200)  # disconnected
xapp.register_callback(Send2xApp, 8704200)  # Report Query
#--------------------------------------------
xapp.register_callback(db, 62014200)  # db
xapp.register_callback(db, 62044200)  # db
xapp.register_callback(db, 66014200)  # db
xapp.register_callback(db, 66044200)  # db
#--------------------------------------------
xapp.register_callback(dbreport, 67014200)  # dbreport
xapp.register_callback(dbreport, 67044200)  # dbreport
xapp.register_callback(dbreport, 67114200)  # dbreport
xapp.register_callback(dbreport, 67144200)  # dbreport

xapp.run()  # will not thread by default


#https://note.nkmk.me/python-opencv-face-detection-haar-cascade/
import cv2
import numpy as np
import sys

def main():
	hand_cascade_path = '/Users/dennomaaya/Desktop/py/cascade_directory/cascade/test_data/hand.xml'
	#hand_cascade_path = '/Users/dennomaaya/Desktop/py/cascade_directory/cascade/test_data/Hand.Cascade.1.xml'
	#hand_cascade_path = '/usr/local/opt/opencv@3/share/OpenCV/haarcascades/haarcascade_frontalface_default.xml'

	hand_cascade = cv2.CascadeClassifier(hand_cascade_path)

	img = cv2.imread("%s"%param[1])
	gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

	hands = hand_cascade.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)
	print(hands)

	for (x, y, w, h) in hands:
	    cv2.rectangle(img, (x, y), (x + w, y + h), (0, 0, 255), 5)
	    #roi_gray = gray[y: y + h, x: x + w]
	    #roi_color = img[y: y + h, x: x + w]

	cv2.imwrite('hand_detect.jpg', img)

if __name__ == '__main__':
	param = sys.argv
	main()


import scrapy
import re
import pandas as pd
from .parentClass import parentClass
import os
import shutil
import json
from .tools import exceptions

class parliamentScraper(scrapy.Spider, parentClass):
    def __init__(self, urls: list, chapterScrapers: dict, dataPath: str):
        self.start_urls = urls
        self.chapterScrapers = chapterScrapers
        self.path = dataPath

        self.loadUselessElementRegex()
        self.loadChapterHeaders()

    def loadUselessElementRegex(self):
        path = os.path.join(
            os.path.dirname(__file__), "operatingData", "uselessElements.csv"
        )
        temp = pd.read_csv(path)
        temp = "^" + temp["regex"]
        self.uselessElementRegex = "|".join(temp.tolist())

    def loadChapterHeaders(self):
        path = os.path.join(os.path.dirname(__file__), "operatingData", "headers.csv")
        self.chapterHeaders = pd.read_csv(path)
        self.chapterHeadersRegex = ""
        self.subChapterHeadersRegex = ""
        self.chapterRegexAndName = dict()
        self.subChapterRegexAndName = dict()
        for name in self.chapterHeaders["name"].unique():
            if (
                self.chapterHeaders[self.chapterHeaders["name"] == name].iloc[0]["type"]
                == "header"
            ):
                regexes = self.chapterHeaders[self.chapterHeaders["name"] == name]
                self.chapterRegexAndName[name] = "|".join(
                    regexes["regex"].tolist()
                ).lower()
            elif (
                self.chapterHeaders[self.chapterHeaders["name"] == name].iloc[0]["type"]
                == "subHeader"
            ):
                regexes = self.chapterHeaders[self.chapterHeaders["name"] == name]
                self.subChapterRegexAndName[name] = "|".join(
                    regexes["regex"].tolist()
                ).lower()

        regexes = self.chapterHeaders[self.chapterHeaders["type"] == "header"]
        self.chapterHeadersRegex = ("|".join(regexes["regex"].tolist())).lower()

    def isElementUseless(self, element) -> bool:
        text = self.getInnerText(element)
        return (len(text) == 0) or (
            bool(re.match(self.uselessElementRegex, text, flags=re.IGNORECASE))
        )

    def filterOutUselessElements(self, elementList) -> list:
        out = [None] * len(elementList)
        j = 0
        for i in range(0, len(elementList)):
            if not self.isElementUseless(elementList[i]):
                out[j] = elementList[i]
                j += 1
        (
            self.status["amountOfElements"],
            self.status["removedElements"],
            self.status["originalELements"],
        ) = (len(elementList) - j, j, len(elementList))
        return out[0:j]

    def isChapterHeader(self, element) -> bool:
        if bool(re.search("titre1|titel1", self.getClass(element))):
            return True
        if bool(re.search("titre2|titel2", self.getClass(element))) and bool(
            re.search(
                "\bverklaring van de regering|\bdéclaration du gouvernement",
                self.getInnerText(element),
                flags=re.IGNORECASE,
            )
        ):
            return True
        return False

    def getHeaderType(self, element) -> str:
        text = self.getInnerText(element)
        for key, value in self.chapterRegexAndName.items():
            if bool(re.search(value, text, flags=re.IGNORECASE)):
                return key
        return None

    def getChapters(self, elements) -> dict:
        chapters = list()
        for i in range(0, len(elements)):
            if self.isChapterHeader(elements[i]):
                t = self.getHeaderType(elements[i])
                chapters.append((i, t, self.getInnerText(elements[i])))

        if len(chapters) == 0:
            return {"intro": [(0, len(elements))]}

        new = list()
        i = 0
        while i < len(chapters):
            new.append((chapters[i][1], chapters[i][0], chapters[i + 1][0]))
            i += 2

        chapters = new
        new = list()
        for i in range(0, len(chapters)):
            if i < len(chapters) - 1:
                new.append((chapters[i][0], chapters[i][2] + 1, chapters[i + 1][1]))
            else:
                new.append((chapters[i][0], chapters[i][2] + 1, len(elements)))

        chapters = new
        new = dict()
        for i in range(0, len(chapters)):
            if chapters[i][0] in new:
                continue
            new[chapters[i][0]] = []
            new[chapters[i][0]].append((chapters[i][1], chapters[i][2]))
            for j in range(i + 1, len(chapters)):
                if chapters[j][0] == chapters[i][0]:
                    new[chapters[i][0]].append((chapters[j][1], chapters[j][2]))
        min = 100000
        for key, value in new.items():
            for v in value:
                if v[0] < min:
                    min = v[0]
        new["intro"] = [(0, min)]
        return new

    def makeInputForChapterScraper(self, elements, indices):
        out = []
        for item in indices:
            start, stop = item
            out += elements[start:stop]
        return out

    def parse(self, response):
        self.status = {"error": False, "errorChapters": [], "url": response.request.url}
        totalNodes = list()
        allNodes = response.selector.css("body>*")
        for x in allNodes:
            y = x.css("div>*")
            if len(y) > 0:
                totalNodes += y
        self.totalElements = self.filterOutUselessElements(totalNodes)
        if len(self.totalElements) < 15:
            raise exceptions.tooFewElements(
                len(self.totalElements), response.request.url
            )
        self.chapters = self.getChapters(self.totalElements)

        for cs in self.chapterScrapers:
            if cs in self.chapters:
                inp = self.makeInputForChapterScraper(
                    self.totalElements, self.chapters[cs]
                )
                self.chapterScrapers[cs].parse(inp)
                if self.chapterScrapers[cs].failed():
                    self.status["error"] = True
                    self.status["errorChapters"].append(cs)

        self.status["chapters"] = self.chapters

        temp = response.request.url.split("/")
        if temp[-1][0:2] == "ic":
            org = "commision"
        else:
            org = "plenary"
        i = temp[-1].find("x")
        documentNumber = temp[-1][2:i].lstrip("0")
        self.status["legis"] = temp[-2]
        self.status["orgName"] = org
        self.status["documentNumber"] = documentNumber

        self.makeBlockIdsGlobal()

        self.save(legislature=temp[-2], orgName=org, documentNumber=documentNumber)

    def save(self, legislature: int, orgName: str, documentNumber: int):
        if not os.path.isdir(self.path):
            os.mkdir(self.path)
        if (not os.path.exists(f"{self.path}/{legislature}")) or (
            not os.path.isdir(f"{self.path}/{legislature}")
        ):
            os.mkdir(f"{self.path}/{legislature}")

        if (not os.path.exists(f"{self.path}/{legislature}/{orgName}")) or (
            not os.path.isdir(f"{self.path}/{legislature}/{orgName}")
        ):
            os.mkdir(f"{self.path}/{legislature}/{orgName}")

        if (
            not os.path.exists(f"{self.path}/{legislature}/{orgName}/{documentNumber}")
        ) or (
            not os.path.isdir(f"{self.path}/{legislature}/{orgName}/{documentNumber}")
        ):
            os.mkdir(f"{self.path}/{legislature}/{orgName}/{documentNumber}")

        folder = f"{self.path}/{legislature}/{orgName}/{documentNumber}"
        # clean folder
        for f in os.listdir(folder):
            if os.path.isfile(f"{folder}/{f}"):
                os.remove(f"{folder}/{f}")
            else:
                shutil.rmtree(f"{folder}/{f}")

        for cs in self.chapterScrapers:
            if cs in self.chapters:
                os.mkdir(f"{folder}/{cs}")
                self.chapterScrapers[cs].save(f"{folder}/{cs}")

        with open(folder + "/log.json", "w") as fp:
            fp.write(json.dumps(self.status, indent=4))

    def localToGlobalIndices(self, localIndices: list, ranges: list) -> list:
        # local indices are [[chapter name,blockid , index],[...]]
        # ranges are[[start,stop],[...]]
        lengths = [x[1] - x[0] for x in ranges]
        globalIndices = []
        for i in range(len(lengths)):
            shift = lengths[i]
            localIndices = [[x[0], x[1], x[2] - shift] for x in localIndices]

            # remaining
            globalIndices += [
                [x[0], x[1], x[2] + shift + ranges[i][0]]
                for x in localIndices
                if x[2] < 0
            ]
            localIndices = [[x[0], x[1], x[2]] for x in localIndices if x[2] >= 0]
        return globalIndices

    def makeBlockIdsGlobal(self):
        topics = []
        for cs in self.chapterScrapers:
            if cs in self.chapters:
                blocks = [
                    [cs, x[0], x[1]] for x in self.chapterScrapers[cs].getBlockIndices()
                ]
                if len(blocks) > 0:
                    topics += self.localToGlobalIndices(blocks, self.chapters[cs])
        # topics [[chapterName,blocid,globalIndex],...]
        topics = sorted(topics, key=lambda tup: tup[2])
        conversionDicts = {}
        for i in range(0, len(topics)):
            if topics[i][0] not in conversionDicts:
                conversionDicts[topics[i][0]] = {}  # add a dict for every chapter
            conversionDicts[topics[i][0]][topics[i][1]] = i
        for key, value in conversionDicts.items():
            self.chapterScrapers[key].addToStatus("localBlockIdToGlobal", value)