#!/usr/bin/env python3
# - ids.py - #
# - Snort IDS for offline pcap scanning on Ubuntu 18.04
# - Author: mcb2Eexe
# - Last modified: 16 Oct 19
# - Dependencies: python3, pip3, pip install requests, curl, snort, rules dir, pcap dir, logs dir,
# - ToDo: add alert output, check sig errors, check snort version, check how to use rulepacks best, bash setup script
# - Issues: Rule packs have errors when updated (possibly ubuntu repo snort version), can't delete tar files

import sys
import time
import tarfile
import requests
from os import path
import subprocess as sp

SNORT_PATH = "/etc/snort/"
RULE_PATH = "/etc/snort/community_rules"
SNORT_TAR = "/etc/snort/community_rules/SC.tar.gz"
ET_TAR = "/etc/snort/community_rules/ET.tar.gz"
ET_URL = "https://rules.emergingthreats.net/open/snort-2.9.7.0/emerging.rules.tar.gz"
SNORT_URL = "https://snort.org/downloads/community/community-rules.tar.gz"

# - Clear Terminal - #
def clear():
    sp.call('clear', shell=True)
# --- #

# - Help menu - #
def help(error):
    clear()
    print(error)
    print("\n Usage: $ids.py [pcap] [-h]")
    print("\n     [pcap]  Input pcap file to be scanned")
    print("\n Example: ids file.pcap\n")
# --- #

# - Snort Function - #
def snort(pcap):
    sp.call('snort -A full -c {}snort.conf -l {}logs/ -r {}'.format(SNORT_PATH, SNORT_PATH, pcap), shell=True)
# --- #

# - Emerging Threats Function - #
def et(pcap):
    sp.call('snort -A full -c {}snort_et.conf -l {}logs/ -r {}'.format(SNORT_PATH, SNORT_PATH, pcap), shell=True)
# --- #

# - All Rules Function - #
def combined(pcap):
    sp.call('snort -A full -c {}snort_all.conf -l {}logs/ -r {}'.format(SNORT_PATH, SNORT_PATH, pcap), shell=True)
# --- #

# - Update Rules - #
def update(rules, vendor, tar):
    clear()
    print("Searching for {} updates...".format(vendor))
    response = requests.get(rules)
    if response.status_code == 200:
        print("{} update found! Updating...".format(vendor))
        r = requests.get(rules)
        open('{}'.format(tar), 'wb').write(r.content)
        tar = tarfile.open("{}".format(tar), 'r:*')
        tar.extractall('{}'.format(RULE_PATH))
        tar.close()
        print("Update complete!")
        time.sleep(2)
    else:
        print("{} rule update not found!".format(vendor))
# --- #

# - Check Connections - #
def connection_status(url):
    response = requests.get('{}'.format(url))
    return response.status_code
# --- #

# - Start - #
while True:
    if len(sys.argv) != 2:
        help(" Invalid input! Check and try again...")
        exit(0)
    elif not path.exists(sys.argv[1]):
        help(" Pcap not found! Check and try again...")
        exit(0)
    else:
        argv = sys.argv[1]
    clear()
    sp.call('snort -V', shell=True)
    print(" Menu:\n")
    print("   [1] Snort Community Open Rules")
    print("   [2] Emerging Threats Rules")
    print("   [3] All Rules")
    print("   [4] Update Rules")
    arg = input("\n Select an option: ")
    if arg == "1":
        snort(argv)
        break
    if arg == "2":
        et(argv)
        break
    if arg == "3":
        combined(argv)
        break
    if arg == "4":
        update(ET_URL, "ET", ET_TAR)
        update(SNORT_URL, "SC", SNORT_TAR)
        clear()
        continue
    else:
        clear()
        print("\nUnknown input! Expecting integer...\n")
        continue
# - End - #


from datetime import datetime, date

def formatNumber(numbers):
    value = '('
    for acode in numbers[0:3]:
        value += str(acode)
    value += ') '
    for f3 in numbers[3:6]:
        value += str(f3)
    value += '-'
    for l4 in numbers[6:11]:
        value += l4
    return value

def indexOfLastDigit(number):
    index = 0
    for x in range(0, len(number)):
        if (number[x].isdigit()):
            index = x
    return index

def parseDateTime(fileName):
    for i in [i for i, ltr in enumerate(fileName) if ltr == '_']:
        fileName = fileName[:i] + ':' + fileName[i + 1:]
    try:
        return datetime.fromisoformat(fileName[fileName.rindex(' ') + 1:fileName.rindex('Z')])
    except ValueError as err:
        try:
            return date.fromisoformat(fileName[fileName.rindex(' ') + 1:fileName.rindex('T')])
        except Exception as err2:
            raise Exception

def sortDictionaryByDate(history):
    return sorted(history)


# -*- coding: utf-8 -*-

from tastypie.resources import ModelResource
from exhibition.models import Exhibition, ExhibitionComment, ExhibitionRate
from work.models import Work
from buildengine.models import PrefWebsite
from django.contrib.auth.models import User
from tastypie.serializers import Serializer
import time
from tastypie import fields
from django.utils import simplejson
from django.core.serializers import json
from tastypie.authorization import DjangoAuthorization
from tastypie.resources import ModelResource, ALL, ALL_WITH_RELATIONS

class UserResource(ModelResource):
    class Meta:
        queryset = User.objects.filter(is_active=True)
        resource_name = 'user'
        excludes = ['email', 'password', 'is_superuser', 'date_joined', 'is_staff', 'is_active', 'last_login', 'resource_uri',
                    'first_name', 'last_name']

class ExhibitionResource(ModelResource):
    user = fields.ForeignKey(UserResource, 'user', full=True)
    class Meta:
        queryset = Exhibition.objects.filter(user_id__groups__name="Artist")
        resource_name = 'exhibition'
        excludes = ['rate']
        authorization = DjangoAuthorization()
        filtering = {
            'comment': ALL_WITH_RELATIONS,
            'date_pub': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'date_begin': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'date_end': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'mapLongitude': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'mapLatitude': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
            'zipcode': ['exact', 'range', 'gt', 'gte', 'lt', 'lte'],
        }

    def dehydrate(self, bundle):
        bundle.data['work_preview'] = [str(x.image) for x in Work.objects.filter(user_id=bundle.obj.user.id)[:3]]
        bundle.data['anonymity'] = ''.join([str(x.anonymity) for x in PrefWebsite.objects.filter(user_id=bundle.obj.user.id)])
        arrayRate = [x.rate for x in ExhibitionRate.objects.filter(exhibition = bundle.obj.id)]
        if(len(arrayRate) == 0):
            bundle.data['rateMedian'] = -1
        else:
            bundle.data['rateMedian'] = sum(arrayRate) / len(arrayRate)
        if not bundle.data['anonymity']:
            bundle.data['last_name'] = ''.join([str(x.last_name) for x in User.objects.filter(id=bundle.obj.user.id)])
            bundle.data['first_name'] = ''.join([str(x.first_name) for x in User.objects.filter(id=bundle.obj.user.id)])
        return bundle

class CommentResource(ModelResource):
    exhibition = fields.ForeignKey(ExhibitionResource, 'exhibition', full=True)
    class Meta:
        queryset = ExhibitionComment.objects.all()
        resource_name = 'comment'
        authorization = DjangoAuthorization()
        filtering = {
            'exhibition': ALL_WITH_RELATIONS
        }

    def dehydrate(self, bundle):
        bundle.data['username'] = bundle.obj.user.username
        bundle.data['rate'] = ''.join([str(x.rate) for x in ExhibitionRate.objects.filter(exhibition=bundle.obj.exhibition.id, user=bundle.obj.user.id)])
        return bundle


import time

import openpyxl
from selenium import webdriver
from selenium.webdriver import ActionChains
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

from PageObjects.Elements import Elements
from Utils import XLUtils
from Utils.BaseClass import BaseClass
from Utils.readProperties import ReadConfig

class TestOne(BaseClass):
    log = BaseClass.getLogger()
    def test_datadriven(self):
        readconfig = ReadConfig()  # create object without any argument
        self.baseURL = readconfig.getApplicationURL()
        self.path= readconfig.getinputPath()
        self.driver.get(self.baseURL)
        self.log.info("Opened the URL")
        self.log.info(self.driver.title)
        elements_page=Elements(self.driver)
        # Scroll down to the "Elements" link
        self.driver.execute_script("window.scrollBy(0,300)")
        # elements_link = self.driver.find_element(By.XPATH, "//h5[normalize-space()='Elements']")
        elements_link = elements_page.click_elements_link()
        #To hover the cursor over the elements link and click
        actions = ActionChains(self.driver)
        actions.move_to_element(elements_link).click().perform()
        self.log.info("Clicked on Elements")
        # elements_heading=self.driver.find_element(By.XPATH,"//div[@class ='main-header']").text
        elements_heading = elements_page.verify_elements_heading().text
        self.log.info(elements_heading)
        assert "Elements" in elements_heading

        # self.driver.find_element(By.XPATH, "//span[normalize-space()='Text Box']").click()
        elements_page.click_elements_textbox().click()
        self.log.info("Clicked on Text Box")
        workbook = openpyxl.load_workbook(
            "C:/Users/amuly/PycharmProjects1/WebUIAutomation/Input/Input_TestBox.xlsx")
        worksheet = workbook.active
        self.rows = XLUtils.getRowCount(self.path, 'Sheet1')
        for r in range(2, self.rows+1):
            self.Fullname_value = XLUtils.readData(self.path, 'Sheet1', r, 1)
            self.Email_value = XLUtils.readData(self.path, 'Sheet1', r, 2)
            self.CurrentAddress_value = XLUtils.readData(self.path, 'Sheet1', r, 3)
            self.PermenantAdd_value = XLUtils.readData(self.path, 'Sheet1', r, 4)
            # Fullname_input=self.driver.find_element(By.XPATH,"//input[@id='userName']")
            Fullname_input = elements_page.input_textbox_fullname()
            Fullname_input.clear()
            Fullname_input.send_keys(self.Fullname_value)
            self.log.info("Given Full Name")
            # Email_input = self.driver.find_element(By.XPATH, "//input[@id='userEmail']")
            Email_input = elements_page.input_textbox_email()
            Email_input.clear()
            Email_input.send_keys(self.Email_value)
            self.log.info("Given Email")
            # CurrentAddr_input = self.driver.find_element(By.XPATH, "//textarea[@id='currentAddress']")
            CurrentAddr_input = elements_page.input_textbox_currentaddress()
            CurrentAddr_input.clear()
            CurrentAddr_input.send_keys(self.CurrentAddress_value)
            self.log.info("Given Current Address")
            # PermanantAdd_input = self.driver.find_element(By.XPATH, "//textarea[@id='permanentAddress']")
            PermanantAdd_input = elements_page.input_textbox_permanentaddress()
            PermanantAdd_input.clear()
            PermanantAdd_input.send_keys(self.PermenantAdd_value)
            self.log.info("Given Permenant Address")
            self.driver.execute_script("window.scrollBy(0,300)")
            # self.driver.find_element(By.XPATH, "//button[@id = 'submit']").click()
            elements_page.click_submit_button().click()
            self.log.info("Clicked on Submit button")
            time.sleep(5)
            FullName=self.driver.find_element(By.XPATH, "//p[@id='name']").text
            Email=self.driver.find_element(By.XPATH, "//p[@id='email']").text
            Current_Address=self.driver.find_element(By.XPATH, "//p[@id='currentAddress']").text
            Permanent_Address=self.driver.find_element(By.XPATH, "//p[@id='permanentAddress']").text
            self.log.info(FullName)
            self.log.info(Email)
            self.log.info(Current_Address)
            self.log.info(Permanent_Address)
            self.log.info("Form details displayed on UI as given in text boxes")
            split_name=FullName.split(':')
            name_split=split_name[1]
            if name_split == self.Fullname_value:
                assert True
                self.log.info("Pass")
            else:
                assert False
                self.log.info("Fail")


import os
from datetime import datetime
from data_center.services.settings import IMAGE_COUNT_LIMIT, UNPROCESSED_IMAGES_DIRECTORY, UNPROCESSED_VIDEOS_DIRECTORY
import cv2
from data_center.services.settings import FACE_CASCADE_PATH

def valid_video_capture_devices():
    potential_device_range = 10
    available_capture_devices = []
    for devices_index in range(potential_device_range):
        cap = cv2.VideoCapture(devices_index)
        cap_is_opened = cap.isOpened()
        if cap and cap_is_opened:
            available_capture_devices.append(devices_index)
            cap.release()
    return available_capture_devices

def generate_name(file_type='image'):
    file_date_time_taken = datetime.now().strftime('%Y%m%d%H%M%S')
    file_name = datetime.now().strftime(f'{file_date_time_taken}')
    if file_type == 'image':
        os.makedirs(UNPROCESSED_IMAGES_DIRECTORY, exist_ok=True)
        file_location = os.path.join(UNPROCESSED_IMAGES_DIRECTORY, file_name + '.jpg')
    else:
        os.makedirs(UNPROCESSED_VIDEOS_DIRECTORY, exist_ok=True)
        file_location = os.path.join(UNPROCESSED_VIDEOS_DIRECTORY, file_name + '.avi')
    return file_location

def capture_image(image_frame):
    img_name = generate_name('image')
    imwrite_result = cv2.imwrite(img_name, image_frame)
    print("imwrite_result:", imwrite_result)
    if imwrite_result:
        print("Image {} saved.".format(img_name))
        return img_name
    else:
        print("Image {} failed to save.".format(img_name))
        return imwrite_result

def run_image_capture(valid_image_capture_device, window_name='Image Capture'):
    cam = cv2.VideoCapture(valid_image_capture_device)
    cv2.namedWindow(window_name)

    img_counter = 0

    while True:
        ret, frame = cam.read()
        if not ret:
            print("failed to grab frame")
            break
        cv2.imshow(window_name, frame)

        wait_key = cv2.waitKey(1)
        if wait_key % 256 == 27:
            # ESC pressed
            print("Closing...")
            break
        elif wait_key % 256 == 32:
            # SPACE pressed
            capture_image(frame)
            img_counter += 1

    cam.release()
    cv2.destroyAllWindows()

def run_face_detecting_camera(valid_image_capture_device, window_name='Image Capture', display_grey=False):
    cam = cv2.VideoCapture(valid_image_capture_device)
    cv2.namedWindow(window_name)
    img_counter = 0
    face_cascade = cv2.CascadeClassifier(FACE_CASCADE_PATH)
    while True:
        ret, frame = cam.read()
        if not ret:
            print("failed to grab frame")
            break
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30)
        )
        wait_key = cv2.waitKey(1)
        if display_grey:
            frame = gray
        write_text_on_frame(frame, text=datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
        if wait_key % 256 == 27:
            # ESC pressed
            print("Closing...")
            break
        elif wait_key % 256 == 32:
            # SPACE pressed
            if len(faces) > 0:
                # Draw a rectangle around the faces
                if len(faces) > 1:
                    print("Capturing multiple faces...")
                else:
                    print("Capturing single faces...")
            capture_image(frame)
            img_counter += 1
        for (x, y, w, h) in faces:
            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
        cv2.imshow(window_name, frame)
    cam.release()
    cv2.destroyAllWindows()

def run_automated_face_image_capture(valid_image_capture_device, window_name='Image Capture', display_grey=False):
    cam = cv2.VideoCapture(valid_image_capture_device)
    cv2.namedWindow(window_name)
    img_counter = 0
    face_cascade_path = FACE_CASCADE_PATH
    face_cascade = cv2.CascadeClassifier(face_cascade_path)
    while True:
        ret, frame = cam.read()
        if not ret:
            print("failed to grab frame")
            break
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30)
        )
        if display_grey:
            frame = gray
        write_text_on_frame(frame, text=datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
        k = cv2.waitKey(1)
        if k % 256 == 27:
            # ESC pressed
            print("Closing...")
            break
        if len(faces) > 0:
            # Draw a rectangle around the faces
            if len(faces) > 1:
                print("Multiple faces detected...")
            else:
                print("Single face detected...")
            if img_counter < IMAGE_COUNT_LIMIT:
                capture_image(frame)
                img_counter += 1
            else:
                print("IMAGE COUNT LIMIT:", IMAGE_COUNT_LIMIT)
                break
        for (x, y, w, h) in faces:
            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
        cv2.imshow(window_name, frame)
    cam.release()
    cv2.destroyAllWindows()

def run_face_triggered_video_capture(valid_image_capture_device, window_name='Image Capture', display_grey=True):
    cam = cv2.VideoCapture(valid_image_capture_device)
    cv2.namedWindow(window_name)
    file_locations = []
    face_cascade_path = FACE_CASCADE_PATH
    face_cascade = cv2.CascadeClassifier(face_cascade_path)
    # Define the codec and create VideoWriter object
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    recording = False
    video_output = None
    while True:
        ret, frame = cam.read()
        if not ret:
            print("failed to grab frame")
            break
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(
            gray,
            scaleFactor=1.1,
            minNeighbors=5,
            minSize=(30, 30)
        )
        if display_grey:
            frame = gray
        k = cv2.waitKey(1)
        if k % 256 == 27:
            # ESC pressed
            print("Closing...")
            break
        if len(faces) > 0:
            recording_name = generate_name('video')
            if not recording:
                recording = True
                video_output = cv2.VideoWriter(recording_name, fourcc, 20.0, (640, 480))
            # Draw a rectangle around the faces
            for (x, y, w, h) in faces:
                if len(faces) > 1:
                    print("Multiple faces detected...")
                else:
                    print("Single face detected...")
                cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
            write_text_on_frame(frame, text=datetime.now().strftime("%Y/%m/%d %H:%M:%S"))
            video_output.write(frame)
        else:
            if recording:
                recording = False
                video_output.release()
        cv2.imshow(window_name, frame)
    cam.release()
    cv2.destroyAllWindows()
    return file_locations

def write_text_on_frame(img, text=datetime.now().strftime("%Y/%m/%d %H:%M:%S")):
    font = cv2.FONT_HERSHEY_PLAIN
    cv2.putText(img, text, (10, 450), font, 1, (255, 255, 255), 2, cv2.LINE_AA)


"""

11060 점프 점프

"""
import sys
from collections import deque
input = sys.stdin.readline

n = int(input())
a = list(map(int, input().split()))

if n == 1:
    print(0)
    sys.exit()
visit = [0] * n

q = deque()
q.append((0, a[0]))

while q:
    now, jump = q.popleft()
    for i in range(1, jump + 1):
        if now + i >= n or visit[now + i]:
            continue
        visit[now + i] = visit[now] + 1
        q.append((now + i,a[now + i]))

print(visit[-1] if visit[-1] else -1)


"""
Transaction Models
"""
from django.db import models
from django.db import transaction
from django.core.exceptions import ValidationError
from django.db.models import Q

class Transaction(models.Model):
    """
    Transaction model includes withdrawal, deposit, transfer.
    uses ints to avoid pitfalls of floats.
    e.g:
    >>> 1000.00 - 65.43
    934.5699999999999
    >>> (int(1000.00*100) - int(65.43*100))/100
    934.57
    """
    TRANSACTION_TYPES = ("deposit", "withdrawal", "transfer")
    created_on = models.DateTimeField(auto_now=True, null=False, db_index=True,)
    transaction_type = models.CharField(max_length=20, db_index=True, null=False)
    _amount = models.IntegerField(db_index=True, null=False)
    account_to = models.ForeignKey(
        'accounts.Account',
        on_delete=models.CASCADE,
        null=True,
        blank=True,
        related_name='deposits_to'
    )
    account_from = models.ForeignKey(
        'accounts.Account',
        on_delete=models.CASCADE,
        null=True,
        blank=True,
        related_name='withdrawals_from'
    )
    creator = models.ForeignKey(
        'auth.User',
        on_delete=models.CASCADE,
        null=False,
        related_name='transactions_processed'
    )

    @classmethod
    def get_by_account(cls, account):
        return cls.objects.filter(
            Q(account_from=account) | Q(account_to=account)
        ).order_by('-pk').all()

    @property
    def amount(self):
        return self._amount / 100

    @amount.setter
    def amount(self, value):
        self._amount = int(value * 100)
        return self.amount

    def __str__(self):
        return "{0}:{1:.2f}".format(
            self.transaction_type,
            self.amount
        )

    def clean(self, *args, **kwargs):
        """
        Custom Validatation:
        A transaction cannot be edited.
        A transaction must have an account associated
        A transaction must have a known type
        """
        if self.id:
            raise ValidationError("Cannot edit a transaction")
        if not (self.account_from or self.account_to)  or self.account_from == self.account_to:
            raise ValidationError("Must be associated with an account")
        if self.transaction_type not in self.TRANSACTION_TYPES:
            raise ValidationError("Cannot have unknown transaction type")
        if not (
            (
                self.transaction_type == "deposit" and
                self.account_to and
                not self.account_from
            ) or
            (
                self.transaction_type == "withdrawal" and
                not self.account_to and
                self.account_from
            ) or
            (
                self.transaction_type == "transfer" and
                self.account_to and
                self.account_from
            )
        ):
            raise ValidationError("Incorrect transaction format")
        if not (self._amount and self.amount > 0):
            raise ValidationError("Amount muse be greater than 0")
        super(Transaction, self).clean(*args, **kwargs)

    @transaction.atomic
    def save(self, *args, **kwargs):
        """
        During save ask the accounts to update.
        """
        self.full_clean()
        if self.account_from:
            self.account_from.withdraw(self.amount)
        if self.account_to:
            self.account_to.deposit(self.amount)
        super(Transaction, self).save(*args, **kwargs)


"""TOJA command-line application."""
import asyncio
import click
import logging
import logging.config
import yaml
import os

from cerberus import Validator
from typing import Union

from .actions import actions
from ..background import run_background_app
from ..server import run_application_server
from ..utils import set_config
from ..setup import setup_backend

logger = logging.getLogger(__name__)

CONFIG_SCHEMA = {
    'server': {
        'type': 'dict',
        'schema': {
            'host': {
                'type': 'string',
                'default': '127.0.0.1'
            },
            'port': {
                'type': 'integer',
                'default': 6543
            },
            'base': {
                'type': 'string',
                'default': 'http://127.0.0.1:6543'
            }
        },
        'default': {
            'host': '127.0.0.1',
            'port': 6543,
            'base': 'http://127.0.0.1:6543'
        }
    },
    'database': {
        'type': 'dict',
        'required': True,
        'schema': {
            'server': {
                'type': 'string',
                'default': 'http://localhost:5984'
            },
            'user': {
                'type': 'string'
            },
            'password': {
                'type': 'string'
            }
        }
    },
    'mosquitto': {
        'type': 'dict',
        'required': True,
        'schema': {
            'host': {
                'type': 'string',
                'default': 'localhost'
            },
            'username': {
                'type': 'string',
                'nullable': True,
                'default': None
            },
            'password': {
                'type': 'string',
                'nullable': True,
                'default': None
            }
        }
    },
    'meilisearch': {
        'type': 'dict',
        'required': True,
        'schema': {
            'server': {
                'type': 'string',
                'required': True
            },
            'auth_token': {
                'type': 'string',
                'empty': True,
                'default': None
            }
        }
    },
    'email': {
        'type': 'dict',
        'schema': {
            'server': {
                'type': 'string',
                'required': True,
                'empty': False,
            },
            'secure': {
                'type': 'boolean',
                'default': True,
            },
            'auth': {
                'type': 'dict',
                'schema': {
                    'user': {
                        'type': 'string',
                        'required': True,
                        'empty': False,
                    },
                    'password': {
                        'type': 'string',
                        'required': True,
                        'empty': False,
                    }
                }
            },
            'sender': {
                'type': 'string',
                'default': 'automaton@oldjokearchive.com'
            }
        }
    },
    'debug': {
        'type': 'boolean',
        'default': False,
    },
    'test': {
        'type': 'boolean',
        'default': False,
    },
    'logging': {
        'type': 'dict'
    }
}

def validate_config(config: dict) -> dict:
    """Validate the configuration.

    :param config: The configuration to validate
    :type config: dict
    :return: The validated and normalised configuration
    :rtype: dict
    """
    validator = Validator(CONFIG_SCHEMA)
    if validator.validate(config):
        return validator.normalized(config)
    else:
        error_list = []

        def walk_error_tree(err: Union[dict, list], path: str) -> None:
            if isinstance(err, dict):
                for key, value in err.items():
                    walk_error_tree(value, path + (str(key), ))
            elif isinstance(err, list):
                for sub_err in err:
                    walk_error_tree(sub_err, path)
            else:
                error_list.append(f'{".".join(path)}: {err}')

        walk_error_tree(validator.errors, ())
        error_str = '\n'.join(error_list)
        raise click.ClickException(f'Configuration errors:\n\n{error_str}')

@click.group()
def main() -> None:
    """Run the TOJA applications."""
    config = None
    if os.path.exists('config.yml'):
        with open('config.yml') as in_f:
            config = yaml.load(in_f, Loader=yaml.FullLoader)
    elif os.path.exists('/etc/toja/config.yml'):
        with open('/etc/toja/config.yml') as in_f:
            config = yaml.load(in_f, Loader=yaml.FullLoader)
    if not config:
        raise click.ClickException('No configuration found (./config.yml, /etc/toja/config.yml)')
    normalised = validate_config(config)
    set_config(normalised)
    if 'logging' in normalised:
        logging.config.dictConfig(normalised['logging'])

@click.command()
def server() -> None:
    """Run the TOJA server."""
    run_application_server()

main.add_command(server)

@click.command()
def background() -> None:
    """Run the TOJA background processing application."""
    run_background_app()

main.add_command(background)
main.add_command(actions)

@click.command()
def setup() -> None:
    """Set up the TOJA system."""
    asyncio.run(setup_backend())

main.add_command(setup)


import json
import pickle

student_ID = "student_ID"
student_name = "student_name"
student_age = "student_age"
student_address = "student_address"
total_course_info = "total_course_info"
num_of_course_learned = "num_of_course_learned"
total_course_info = "total_course_info"
learning_course_info = "learning_course_info"
course_code = "course_code"
course_name = "course_name"
course_teacher = "course_teacher"
course_open_date = "course_open_date"
course_close_date = "course_close_date"

json_file_name = "ITT_Student.json"
g_json_bigdata_id_index_file = "ITT_Student_ID_Index.txt"
g_json_bigdata_left_id_list = []
g_json_bigdata = []
student_index_list = []
# temp_total_course_info = []

def add_student_total_course_info(student_index) :
    while 1 :
        order_1 = input("현재 수강 과목 정보를 추가하시겠습니까?(예=1 아니오=1)\n\
입력동작 : ")
        if order_1 == '2':
            break
        elif order_1 == '1':
            course_code_input = input("학생이 수강중인 강의코드를 입력하십시오.(강의코드 알파벳 2자리 + 개걍 년,월,일 6자리)\n\
    (e.g. IB171106) : ")
            course_name_input = input("해당 강의코드의 강의명을 입력하십시오.\n\
    (e.g. IoT 빅데이터 실무반) : ")
            course_teacher_input = input("해당 강의코드의 강사명을 입력하십시오.\n\
    (e.g. 이현구) : ")
            course_open_date_input = input("해당 강의코드의 개강날짜를 입력하십시오.\n\
    (e.g. 2017-11-06) : ")
            course_close_date_input = input("해당 강의코드의 종강날짜를 입력하십시오.\n\
    (e.g. 2018-09-05) : ")
            g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][course_code_input] = \
                {
                    course_name: course_name_input, course_teacher: course_teacher_input,
                    course_open_date: course_open_date_input, course_close_date: course_close_date_input
                }
    return None

def add_student_info(student_index) :
    student_name_input = input("학생 이름을 입력해 주십시오.\n\
(e.g. 김상엽) : ")
    student_age_input = input("학생 나이를 입력해 주십시오.\n\
(e.g. 30) : ")
    student_address_input = input("학생 주소를 입력해 주십시오.\n\
(e.g.  경북 경산시 조영동) : ")
    num_of_course_learned_input = input("과거 수강 횟수를 입력해 주십시오.\n\
(e.g. 0) : ")
    g_json_bigdata[0][student_index] =\
        {
            student_name : student_name_input,
            student_age : student_age_input,
            student_address :student_address_input,
            total_course_info :
                {
                    num_of_course_learned : num_of_course_learned_input,
                    learning_course_info : {}
                }
        }
    add_student_total_course_info(student_index)
    return None

def show_student_info(student_index) :
    print("<학생정보>")
    print("-학생아이디 : %s" %student_index)
    print("-학생이름 : %s" %g_json_bigdata[0][student_index][student_name])
    print("-학생나이 : %s" %g_json_bigdata[0][student_index][student_age])
    print("-학생주소 : %s" %g_json_bigdata[0][student_index][student_address])
    print("-수강정보")
    print("--과거수강횟수 : %s" %g_json_bigdata[0][student_index][total_course_info][num_of_course_learned])
    temp_attending_class_keys_list = list(g_json_bigdata[0][student_index][total_course_info][learning_course_info][0].keys())
    for y in temp_attending_class_keys_list:
        print("--현재수강과목코드 : %s" % y)
        print("--현재수강강의이름 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_name])
        print("--현재수강강의교사 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_teacher])
        print("--현재수강강의개강 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_open_date])
        print("--현재수강강의종강 : %s" %g_json_bigdata[0][student_index][total_course_info][learning_course_info][0][y][course_close_date])
        print()
    return None

def get_student_index_list() :
    global student_index_list
    student_index_list = list(g_json_bigdata[0].keys())
    return None

def show_all() :
    for x in student_index_list :
        print(show_student_info(x))
        print("------------------------------------------------------------------------")
    return None

def check_codition(stored_data, input_data) :
    divide_stored_data = list(str(stored_data))
    divide_inputted_data = list(str(input_data))
    append_or_not = '0'
    starting_index = -1
    for x in range(len(divide_stored_data)) :
        if divide_stored_data[x] == divide_inputted_data[0] :
            starting_index = x
            break
    if starting_index == -1 : return append_or_not
    elif starting_index != -1 :
        for x in range(len(divide_inputted_data)) :
            try :
                if divide_stored_data[starting_index+x] == divide_inputted_data[x] : append_or_not = '1'
                else :
                    append_or_not = '0'
                    return append_or_not
            except :
                append_or_not = '0'
                return append_or_not
        return append_or_not

def show_searched_student_info(searched_student_name_list) :
    if len(searched_student_name_list) == 1:
        show_student_info(searched_student_name_list[0])
    elif len(searched_student_name_list) > 1:
        print("2개 이상의 결과가 검색되어 아이디와 이름만 보여줍니다.")
        for x in searched_student_name_list:
            print("아이디 : %s    이름 : %s " %(x, g_json_bigdata[0][x][student_name]))
    elif searched_student_name_list == []:
        print("수강중인 학생이 없습니다.")
    return None

def search_with_student_primary_key(input_data) :
    searched_student_name_list = []
    for x in student_index_list :
        if check_codition(x, input_data) == '1' :
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_into_one_depth(input_data_category, input_data) :
    searched_student_name_list = []
    for x in student_index_list:
        if check_codition(g_json_bigdata[0][x][input_data_category], input_data) == '1':
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_with_past_registrarion(student_ragistrantion) :
    searched_student_name_list = []
    for x in student_index_list :
        if check_codition(g_json_bigdata[0][x][total_course_info][num_of_course_learned], student_ragistrantion) == '1' :
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_with_now_studying() :
    searched_student_name_list = []
    for x in student_index_list :
        if list(g_json_bigdata[0][x][total_course_info][learning_course_info].keys()) != [] :
            searched_student_name_list.append(x)
    show_searched_student_info(searched_student_name_list)
    return None

def search_into_two_depth(input_data_category, input_data) :
    searched_student_name_list = []
    for x in student_index_list:
        student_class_code_list = list(g_json_bigdata[0][x][total_course_info][learning_course_info].keys())
        for y in student_class_code_list:
            if check_codition(g_json_bigdata[0][x][total_course_info][learning_course_info][y][input_data_category], input_data) == '1':
                searched_student_name_list.append(x)
                break
    show_searched_student_info(searched_student_name_list)
    return None

def delet_with_student_primary_key(student_index) :
    for x in student_index_list :
        if student_index == x :
            del(g_json_bigdata[0][x])
            g_json_bigdata_left_id_list.append(student_index)
            return None

def update_student_info(primary_key) :
    for x in student_index_list :
        if primary_key == x[1] :
            while 1 :
                print("선택하신 아이디의 정보는 아래와 같습니다.")
                print("1:학생이름 : %s" % g_json_bigdata[0][x][student_name])
                print("2:학생나이 : %s" % g_json_bigdata[0][x][student_age])
                print("3:학생주소 : %s" % g_json_bigdata[0][x][student_address])
                print("-수강정보")
                print("4:과거수강횟수 : %s" % g_json_bigdata[0][x][total_course_info][num_of_course_learned])
                temp_attending_class_keys_list = list(g_json_bigdata[0][x][total_course_info][num_of_course_learned].keys())
                print("5:현재수강강의 코드 : ", end = '')
                print(temp_attending_class_keys_list)
                print("6:현재수강강의 추가 : ")
                print("7:수정 끝")
                order_1 = input("바꾸려는 정보를 선택하십시오. : ")
                if order_1 == '7' : break
                elif order_1 == '1' :
                    new_name = input("이름을 입력하십시오.\ne.g)김상엽 : ")
                    g_json_bigdata[0][x][student_name] = new_name
                elif order_1 == '2' :
                    new_age = input("나이를 입력하십시오.\ne.g)30 : ")
                    g_json_bigdata[0][x][student_age] = new_age
                elif order_1 == '3' :
                    new_address = input("주소를 입력하십시오.\ne.g)경북 경산시 조영동 : ")
                    g_json_bigdata[0][x][student_address] = new_address
                elif order_1 == '4' :
                    new_p_registration = input("과거수강횟수를 입력하십시오.\ne.g)0 : ")
                    g_json_bigdata[0][x][total_course_info][num_of_course_learned] = new_p_registration
                elif order_1 == '5' :
                    c_code_from = input("바꾸려는 강의를 코드를 입력하십시오. : ")
                    print(">>현재수강강의이름 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_name])
                    print(">>현재수강강의교사 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_teacher])
                    print(">>현재수강강의개강 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_open_date])
                    print(">>현재수강강의종강 : %s" %g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from][course_close_date])
                    order_2 = input("현재 수강 과목 정보를 수정하시겠습니까?(1:예 2:아니오 3:삭제) : ")
                    if order_2 == '2' : pass
                    elif order_2 == '1' :
                        print()
                        del(g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from])
                        add_student_total_course_info(primary_key)
                    elif order_2 == '3' : del(g_json_bigdata[0][x][total_course_info][num_of_course_learned][c_code_from])
                elif order_1 == '6' :
                    print()
                    add_student_total_course_info(primary_key)
    return None

def main_function() :
    global student_index_list
    print("학생 정보 관리 프로그램입니다.".center(30))
    while 1 :
        order_1 = input("\n동작을 선택해 주십시오.\n\
1:학생 정보입력\n2:학생 정보조회\n3:학생 정보수정\n4:학생 정보삭제\n5:프로그램 종료\n동작입력 : ")
        if order_1 == '5' : break
        elif order_1 == '1' :
            print()
            while 1 :
                print()
                g_json_bigdata.append(add_student_info(student_index_list.pop()))
                student_index_list = get_student_index_list()
                order_2 = input("계속 하시겠습니까?(1:예 2:아니오) : ")
                if order_2 == '2' : break
                elif order_2 == '1' : continue
        elif order_1 == '2' :
            print()
            while 1 :
                order_2 = input("\n1:전체정보출력\n2:ID로조회\n3:이름으로조회\n4:나이로조회\n5:주소로조회\n6:과거수강횟수로조회\n\
7:현재강의를수강하는학생으로조회\n8:현재수강과목의강의명으로조회\n9:현재수강과목의강사로조회\n10:상위메뉴로\n동작입력 : ")
                if order_2 == '10' : break
                elif order_2 == '1' :
                    print()
                    show_all()
                elif order_2 == '2' :
                    print()
                    input_data = input("아이디를 입력하세요. : ")
                    search_with_student_primary_key(input_data)
                elif order_2 == '3' :
                    print()
                    input_data = input("이름을 입력하세요. : ")
                    search_into_one_depth(student_name, input_data)
                elif order_2 == '4' :
                    print()
                    input_data = input("나이를 입력하세요. : ")
                    search_into_one_depth(student_age, input_data)
                elif order_2 == '5' :
                    print()
                    input_data = input("주소를 입력하세요. : ")
                    search_into_one_depth(student_address, input_data)
                elif order_2 == '6' :
                    print()
                    input_data = input("과거수강횟수를 입력하세요. : ")
                    search_with_past_registrarion(input_data)
                elif order_2 == '7' :
                    print()
                    search_with_now_studying()
                elif order_2 == '8' :
                    print()
                    input_data = input("현재수강과목의강의명을 입력하세요. : ")
                    search_into_two_depth(course_name, input_data)
                elif order_2 == '9' :
                    print()
                    input_data = input("현재수강과목의강사명을 입력하세요. : ")
                    search_into_two_depth(course_teacher, input_data)
        elif order_1 == '3' :
            print()
            input_data = input("수정하려는 아이디를 입력하십시오. : ")
            update_student_info(input_data)
            student_index_list = get_student_index_list()
        elif order_1 == '4' :
            print()
            input_data = input("삭제하려는 아이디를 입력하십시오. : ")
            delet_with_student_primary_key(input_data)
            student_index_list = get_student_index_list()
    return None

while 1 :
    try :
        with open(json_file_name, encoding='UTF8') as json_file :
            json_object = json.load(json_file)
            json_string = json.dumps(json_object)
            g_json_bigdata = json.loads(json_string)
        with open(g_json_bigdata_id_index_file, 'rb') as json_index:
            student_index_list = pickle.load(json_index)
        main_function()
        with open('ITT_Student.json', 'w', encoding='utf8') as outfile:
            jsonResult = g_json_bigdata
            readable_result = json.dumps(jsonResult, indent=4, sort_keys=True, ensure_ascii=False)
            outfile.write(readable_result)
        with open(g_json_bigdata_id_index_file, 'wb') as json_index:
            pickle.dump(student_index_list, json_index)
        break

    except FileNotFoundError :
        file_not_found = input("%s파일을 찾을 수 없습니다. 다음 동작을 선택해 주십시오.\n1:신규생성 2:경로설정" %json_file_name)
        if file_not_found == '2' :
            path = input("경로를 입력해 주십시오.\ne.g)D:\\Pycharm_projects\\Emigration\\Jason_Practice\\ : ")
            try:
                with open(path + json_file_name, encoding='UTF8') as json_file:
                    json_object = json.load(json_file)
                    json_string = json.dumps(json_object)
                    g_json_big_data = json.loads(json_string)
                with open(path + g_json_bigdata_id_index_file, 'rb') as index_file:
                    left_primary_key_index_list = pickle.load(index_file)
                student_primary_key_list = get_student_index_list()
                main_function()
                with open(path + json_file_name, 'w', encoding='utf8') as outfile:
                    jsonResult = g_json_big_data
                    readable_result = json.dumps(jsonResult, indent=4, sort_keys=True, ensure_ascii=False)
                    outfile.write(readable_result)
                with open(path + g_json_bigdata_id_index_file, 'wb') as json_index:
                    pickle.dump(left_primary_key_index_list, json_index)
                break
            except:
                print("해당 경로에도 파일이 없습니다. 기본 경로에서 신규생성을 합니다.")
                file_not_found = '1'
        if file_not_found == '1' :
            temp_id_index_str = ""
            for x in range(1,1000) :
                temp_id_index_str += "ITT" + "{0:0>3}".format(str(x)) + '\n'
            temp_id_index_list = list(reversed(temp_id_index_str[:-1].split('\n')))
            with open('ITT_Student.json', 'w', encoding='utf8') as outfile:
                json_big_dict = []
                while 1:
                    order_1 = input("학생정보를 추가하시겠습니까?(1:예 2:아니오) : ")
                    if order_1 == '2':
                        break
                    elif order_1 == '1':
                        g_json_bigdata.append(add_student_info(temp_id_index_list.pop()))
                jsonResult = g_json_bigdata
                readable_result = json.dumps(jsonResult, indent=4, sort_keys=True, ensure_ascii=False)
                outfile.write(readable_result)
                print('ITT_Student.json SAVED')
            with open(g_json_bigdata_id_index_file, 'wb') as index_file:
                pickle.dump(student_index_list, index_file)
                print("%s파일 생성 완료" %g_json_bigdata_id_index_file)
        else : print("동작 선택 좀 똑디 하소.")

김상엽_빅데이터저장_코딩테스트.py


"""
Views Module
------------
"""
from pyramid.response import Response
from pyramid.httpexceptions import HTTPNotFound, HTTPFound
from ..lib.auth import __acl__

__all__ = ['View', 'forbidden', 'Root', 'Users', 'AdminRoot', 'APIRoot',
           'APIUsers']

class View(object):
    """
    The master View class for the app.

    .. note:: Pagination size is handled via the ``items_per_page`` attribute.
    """
    def __init__(self, request):
        self.request = request
        self.items_per_page = 25
        self.__acl__ = __acl__

def forbidden(request):
    """
    The custom ``forbidden`` view.

    By customizing the ``forbidden`` view, we can do things like redirect
    unauthenticated users to the login page, or return a proper JSON forbidden
    response.
    """
    flash = request.session.flash
    next = request.current_route_path()

    # If accepting JSON, just respond with a 401 or 403
    if 'application/json' in str(request.accept):
        # Prepare the JSON response object
        response = Response(content_type='application/json')
        # This is a publicly accessible API
        response.headers.update({'Access-Control-Allow-Origin': '*'})

        # Set the appropriate response status
        if request.current_user:
            response.status = '403 Forbidden'
        else:
            response.status = '401 Unauthorized'
            # Token authorization is expected
            response.headers.update({'WWW-Authenticate': 'Token'})

        return response
    # Otherwise, return a redirect
    elif request.current_user:
        flash(('You are not authorized to access that location.', 'warning'))
        return HTTPFound(location=request.route_path('root_index'))
    else:
        flash(('Please log in before continuing.', 'warning'))
        return HTTPFound(
            location=request.route_path('users', action='login',
                                        _query=dict(next=next))
        )

# Load view classes
from .root import Root
from .users import Users
from .admin.root import AdminRoot
from .api.root import APIRoot
from .api.users import APIUsers


import ctypes
import numpy as np
import os
import sys
import torch
import pdb

class _gnn_lib(object):

    def __init__(self, args):
        dir_path = os.path.dirname(os.path.realpath(__file__))
        self.lib = ctypes.CDLL('%s/build/dll/libgnn.so' % dir_path)

        self.lib.GetGraphStruct.restype = ctypes.c_void_p
        self.lib.PrepareBatchGraph.restype = ctypes.c_int
        self.lib.PrepareSparseMatrices.restype = ctypes.c_int
        self.lib.NumEdgePairs.restype = ctypes.c_int

        if sys.version_info[0] > 2:
            args = [arg.encode() for arg in args]  # str -> bytes for each element in args
        arr = (ctypes.c_char_p * len(args))()
        arr[:] = args
        self.lib.Init(len(args), arr)

        self.batch_graph_handle = ctypes.c_void_p(self.lib.GetGraphStruct())

    def _prepare_graph(self, graph_list, is_directed=0):
        edgepair_list = (ctypes.c_void_p * len(graph_list))()
        list_num_nodes = np.zeros((len(graph_list), ), dtype=np.int32)
        list_num_edges = np.zeros((len(graph_list), ), dtype=np.int32)
        for i in range(len(graph_list)):
            if type(graph_list[i].edge_pairs) is ctypes.c_void_p:
                edgepair_list[i] = graph_list[i].edge_pairs
            elif type(graph_list[i].edge_pairs) is np.ndarray:
                edgepair_list[i] = ctypes.c_void_p(graph_list[i].edge_pairs.ctypes.data)
            else:
                raise NotImplementedError

            list_num_nodes[i] = graph_list[i].num_nodes
            list_num_edges[i] = graph_list[i].num_edges
        total_num_nodes = np.sum(list_num_nodes)
        total_num_edges = np.sum(list_num_edges)

        self.lib.PrepareBatchGraph(self.batch_graph_handle,
                                len(graph_list),
                                ctypes.c_void_p(list_num_nodes.ctypes.data),
                                ctypes.c_void_p(list_num_edges.ctypes.data),
                                ctypes.cast(edgepair_list, ctypes.c_void_p),
                                is_directed)

        return total_num_nodes, total_num_edges

    def PrepareSparseMatrices(self, graph_list, is_directed=0):
        assert not is_directed
        total_num_nodes, total_num_edges = self._prepare_graph(graph_list, is_directed)

        n2n_idxes = torch.LongTensor(2, total_num_edges * 2)
        n2n_vals = torch.FloatTensor(total_num_edges * 2)

        e2n_idxes = torch.LongTensor(2, total_num_edges * 2)
        e2n_vals = torch.FloatTensor(total_num_edges * 2)

        subg_idxes = torch.LongTensor(2, total_num_nodes)
        subg_vals = torch.FloatTensor(total_num_nodes)

        idx_list = (ctypes.c_void_p * 3)()
        idx_list[0] = n2n_idxes.numpy().ctypes.data
        idx_list[1] = e2n_idxes.numpy().ctypes.data
        idx_list[2] = subg_idxes.numpy().ctypes.data

        val_list = (ctypes.c_void_p * 3)()
        val_list[0] = n2n_vals.numpy().ctypes.data
        val_list[1] = e2n_vals.numpy().ctypes.data
        val_list[2] = subg_vals.numpy().ctypes.data

        self.lib.PrepareSparseMatrices(self.batch_graph_handle,
                                ctypes.cast(idx_list, ctypes.c_void_p),
                                ctypes.cast(val_list, ctypes.c_void_p))

        n2n_sp = torch.sparse.FloatTensor(n2n_idxes, n2n_vals, torch.Size([total_num_nodes, total_num_nodes]))
        e2n_sp = torch.sparse.FloatTensor(e2n_idxes, e2n_vals, torch.Size([total_num_nodes, total_num_edges * 2]))
        subg_sp = torch.sparse.FloatTensor(subg_idxes, subg_vals, torch.Size([len(graph_list), total_num_nodes]))
        return n2n_sp, e2n_sp, subg_sp

dll_path = '%s/build/dll/libgnn.so' % os.path.dirname(os.path.realpath(__file__))
if os.path.exists(dll_path):
    GNNLIB = _gnn_lib(sys.argv)
else:
    GNNLIB = None


import os
import pickle
from pathlib import Path
from typing import Tuple

import numpy as np
from tqdm import tqdm

from processing.bayesian_inference import BayesianInference
from processing.utils.constants import PHASE_OUTPUT_PATH

window_size = 57
overlap = 1
sampling_step = 1 / 100

def load_phases(file_path: Path) -> Tuple[np.ndarray, np.ndarray]:
    infile = open(str(file_path), "rb")
    patient = pickle.load(infile, encoding="utf-8")
    phases = tuple(phase.reshape(1, -1) for _, phase in patient.items())
    return phases[0], phases[1]

def construct_name(file_path: Path) -> Path:
    file_path = file_path.with_suffix("")
    _, file_name = os.path.split(file_path)
    return Path(file_name)

def patient_bayes_main() -> None:
    FILE_PATHS = [path for path in PHASE_OUTPUT_PATH.rglob("*.pickle")]

    bayesian = BayesianInference(window_size, sampling_step, overlap)
    bayesian.check_folder()

    for file_path in tqdm(FILE_PATHS):
        file_name = construct_name(file_path)
        first_phase, second_phase = load_phases(file_path)
        bayesian(first_phase, second_phase)
        bayesian.save_inferred_data(file_name)

    print(bayesian.empty_report)

if __name__ == "__main__":
    patient_bayes_main()


"""Command-line interface for scheduling checks and alerts.

Run the check once unless a frequency (in minutes) is provided.
"""

import click

from .alert import main as alert
from .utils import start_log

@click.group()
def main():
    """Run or schedule exposure site checks."""
    pass

@click.command()
def once():
    """Run a check now."""
    alert()

@click.command()
@click.option('--freq', type=click.IntRange(min=1), default=60, show_default=True, help="Frequency (in minutes) of scheduled checks.")
@click.option('--end', type=str, default="23:00", show_default=True, help="When scheduled checks will end.")
def every(freq, end):
    """Schedule checks in the future."""
    alert(freq, end)

main.add_command(once)
main.add_command(every)

if __name__ == '__main__':
    cli_logger = start_log('cli', 7)
    cli_logger.debug('Called from command line')
    main()


from ardb.User import User
from datetime import datetime

class ScDbUtil:

    @staticmethod
    def create_user_to_db(scUser):
        u = User()
        u.convert(soundcloudUser=scUser)
        u.dt_bd_action=datetime.today()
        return u

    @staticmethod
    def scUsersToJsonDbUsers(usersList):
        dbUsers = []
        for user in usersList :
           dbUsers.append(ScDbUtil.create_user_to_db(user).to_dict())
        return dbUsers


import unittest
import json
import requests
from app import app
from threading import Thread
from time import sleep

# NOTE: Make sure you run 'pip3 install requests' in your virtualenv

# URL pointing to your local dev host
LOCAL_URL = "http://localhost:5000"

# Sample testing data
SAMPLE_USER = {"name": "Cornell AppDev", "username": "cornellappdev"}

# Request endpoint generators
def gen_users_path(user_id=None):
    base_path = f"{LOCAL_URL}/api/user"
    return base_path + "s/" if user_id is None else f"{base_path}/{str(user_id)}/"

def gen_send_path():
    return f"{LOCAL_URL}/api/send/"

def unwrap_response(response, body={}):
    try:
        return response.json()
    except:
        req = response.request
        raise Exception(f"""
            Error encountered on the following request:

            request path: {req.url}
            request method: {req.method}
            request body: {str(body)}

            There is an uncaught-exception being thrown in your
            method handler for this route!
            """)

class TestRoutes(unittest.TestCase):

#-- USERS ---------------------------------------------

    def test_get_initial_users(self):
        res = requests.get(gen_users_path())
        body = unwrap_response(res)
        assert body["success"]

    def test_create_user(self):
        res = requests.post(gen_users_path(), data=json.dumps(SAMPLE_USER))
        body = unwrap_response(res, SAMPLE_USER)
        user = body["data"]
        assert body["success"]
        assert user["name"] == SAMPLE_USER["name"]
        assert user["username"] == SAMPLE_USER["username"]
        assert user["balance"] == 0

    def test_get_user(self):
        res = requests.post(gen_users_path(), data=json.dumps(SAMPLE_USER))
        body = unwrap_response(res, SAMPLE_USER)
        user = body["data"]

        res = requests.get(gen_users_path(user["id"]))
        body = unwrap_response(res)
        user = body["data"]
        assert body["success"]
        assert user.get('id') is not None
        assert user["name"] == SAMPLE_USER["name"]
        assert user["username"] == SAMPLE_USER["username"]
        assert user["balance"] == 0

    def test_delete_user(self):
        res = requests.post(gen_users_path(), data=json.dumps(SAMPLE_USER))
        body = unwrap_response(res, SAMPLE_USER)
        user_id = body["data"]["id"]

        res = requests.delete(gen_users_path(user_id))
        body = unwrap_response(res)
        assert body["success"]

        res = requests.get(gen_users_path(user_id))
        body = unwrap_response(res)
        assert not body["success"]

    def test_send_money(self):
        user_with_balance = {**SAMPLE_USER, "balance": 10}
        res1 = requests.post(gen_users_path(), data=json.dumps(user_with_balance))
        body = unwrap_response(res1, user_with_balance)
        user_id1 = body["data"]["id"]
        bal1 = body["data"]["balance"]
        assert bal1 == 10

        res2 = requests.post(gen_users_path(), data=json.dumps(user_with_balance))
        body = unwrap_response(res2, user_with_balance)
        user_id2 = body["data"]["id"]
        bal2 = body["data"]["balance"]
        assert bal1 == 10

        send_body = {"sender_id": user_id1, "receiver_id": user_id2, "amount": 6}

        res = requests.post(gen_send_path(), data=json.dumps(send_body))
        body = unwrap_response(res, send_body)
        assert body["success"]
        # balances are now (user1: 4, user2: 16)

        res1 = requests.get(gen_users_path(user_id1))
        body1 = unwrap_response(res1)
        res2 = requests.get(gen_users_path(user_id2))
        body2 = unwrap_response(res2)
        assert body1["data"]["balance"] == 4
        assert body2["data"]["balance"] == 16

        res = requests.post(gen_send_path(), data=json.dumps(send_body))
        body = unwrap_response(res, send_body)
        assert not body["success"]  # cannot overdraw user1's balance
        # balances are still (user1: 4, user2: 16)

        res1 = requests.get(gen_users_path(user_id1))
        body1 = unwrap_response(res1)
        res2 = requests.get(gen_users_path(user_id2))
        body2 = unwrap_response(res2)
        assert body1["data"]["balance"] == 4
        assert body2["data"]["balance"] == 16

    def test_get_invalid_user(self):
        res = requests.get(gen_users_path(1000))
        body = unwrap_response(res)
        assert not body["success"]

    def test_delete_invalid_user(self):
        res = requests.delete(gen_users_path(1000))
        body = unwrap_response(res)
        assert not body["success"]

    def test_user_id_increments(self):
        res = requests.post(gen_users_path(), data=json.dumps(SAMPLE_USER))
        body = unwrap_response(res, SAMPLE_USER)
        user_id1 = body["data"]["id"]

        res2 = requests.post(gen_users_path(), data=json.dumps(SAMPLE_USER))
        body = unwrap_response(res2, SAMPLE_USER)
        user_id2 = body["data"]["id"]

        assert user_id1 + 1 == user_id2

def run_tests():
    sleep(1.5)
    unittest.main()

if __name__ == "__main__":
    thread = Thread(target=run_tests)
    thread.start()
    app.run(host="0.0.0.0", port=5000, debug=False)


import xml.etree.ElementTree as ET
from Doc import Doc
import pandas as pd

def extract_xml(filename):
    tree = ET.parse(filename)
    root = tree.getroot()
    docs = {}
    for page in root.findall("{http://www.mediawiki.org/xml/export-0.10/}page"):
        id = int(page.find("{http://www.mediawiki.org/xml/export-0.10/}id").text)
        text = page.find("{http://www.mediawiki.org/xml/export-0.10/}revision") \
            .find("{http://www.mediawiki.org/xml/export-0.10/}text").text
        docs[id] = Doc(id, text)
    return docs

def extract_csv(filename):
    docs = {}
    df = pd.read_csv(filename)
    for i in range(df.shape[0]):
        text = df['Title'].values[i] + " " + df['Text'].values[i]
        if "Tag" in df.columns:
            tag = df['Tag'].values[i]
        else:
            tag = None
        docs[i] = Doc(i, text, tag)
    return docs

def read_docs(filename):
    if filename.endswith('xml'):
        return extract_xml(filename)
    return extract_csv(filename)

if __name__ == "__main__":
    # docs = read_docs("../data/Persian.xml")
    docs = read_docs("../data/phase2_train.csv")

    print(docs[0].tag, docs[0].id, docs[0].text)


import os
import stat
import sys
import tempfile

import pytest
import requests
from git import Repo
from mockito import verify, patch, spy2, mock, ANY, when

import opengrok_tools.mirror
from opengrok_tools.scm import Repository
from opengrok_tools.scm.git import GitRepository
from opengrok_tools.scm.repofactory import get_repository
from opengrok_tools.scm.repository import RepositoryException
from opengrok_tools.utils.command import Command
from opengrok_tools.utils.exitvals import (
    CONTINUE_EXITVAL, FAILURE_EXITVAL
)
from opengrok_tools.utils.mirror import check_project_configuration, \
    check_configuration, mirror_project, run_command, get_repos_for_project, \
    HOOKS_PROPERTY, PROXY_PROPERTY, IGNORED_REPOS_PROPERTY, \
    PROJECTS_PROPERTY, DISABLED_CMD_PROPERTY, DISABLED_PROPERTY, \
    CMD_TIMEOUT_PROPERTY, HOOK_TIMEOUT_PROPERTY, DISABLED_REASON_PROPERTY
from opengrok_tools.utils.patterns import COMMAND_PROPERTY, PROJECT_SUBST

def test_empty_project_configuration():
    config = {}
    assert check_project_configuration(config)

def test_none_project_configuration():
    assert check_project_configuration(None)

def test_invalid_project_config_proxy():
    config = {"foo": {PROXY_PROPERTY: True}}
    assert not check_project_configuration(config, proxy=False)

def test_invalid_project_config_option():
    config = {"foo": {"totally_unknown_option": "value"}}
    assert not check_project_configuration(config, proxy=False)

def test_invalid_project_config_regex():
    config = {"[]": {PROXY_PROPERTY: True}}
    assert not check_project_configuration(config, proxy=False)

def test_invalid_project_config_hookdir():
    config = {"foo": {HOOKS_PROPERTY: {"pre": "value"}}}
    assert not check_project_configuration(config, hookdir="/nonexistentdir")

def test_invalid_project_config_ignoredrepos():
    config = {"foo": {IGNORED_REPOS_PROPERTY: {"foo": "bar"}}}
    assert not check_project_configuration(config)

def test_invalid_project_config_hooks():
    with tempfile.TemporaryDirectory() as tmpdir:
        config = {"foo": {HOOKS_PROPERTY: {"pre": "nonexistentfile.sh"}}}
        assert not check_project_configuration(config, hookdir=tmpdir)

def test_invalid_project_config_hooknames():
    with tempfile.TemporaryDirectory() as tmpdir:
        config = {"foo": {HOOKS_PROPERTY: {"blah": "value"}}}
        assert not check_project_configuration(config, hookdir=tmpdir)

@pytest.mark.skipif(not os.name.startswith("posix"), reason="requires posix")
def test_invalid_project_config_nonexec_hook():
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, "foo.sh"), 'w+') as tmpfile:
            tmpfile.write("foo\n")
            config = {"foo": {HOOKS_PROPERTY: {"pre": "foo.sh"}}}
            assert not check_project_configuration(config, hookdir=tmpdir)

@pytest.mark.skipif(not os.name.startswith("posix"), reason="requires posix")
def test_valid_project_config_hook():
    with tempfile.TemporaryDirectory() as tmpdir:
        with open(os.path.join(tmpdir, "foo.sh"), 'w+') as tmpfile:
            tmpfile.write("foo\n")
            st = os.stat(tmpfile.name)
            os.chmod(tmpfile.name, st.st_mode | stat.S_IEXEC)
            config = {"foo": {HOOKS_PROPERTY: {"pre": "foo.sh"}}}
            assert check_project_configuration(config, hookdir=tmpdir)

def test_invalid_config_option():
    assert not check_configuration({"nonexistent": True})

def test_valid_config():
    assert check_configuration({
        PROJECTS_PROPERTY: {"foo": {PROXY_PROPERTY: True}},
        PROXY_PROPERTY: "proxy"
    })

@pytest.mark.skipif(not os.name.startswith("posix"), reason="requires posix")
def test_incoming_retval(monkeypatch):
    """
    Test that the special CONTINUE_EXITVAL value bubbles all the way up to
    the mirror.py return value.
    """

    class MockResponse:

        # mock json() method always returns a specific testing dictionary
        @staticmethod
        def json():
            return "true"

        @staticmethod
        def raise_for_status():
            pass

    with tempfile.TemporaryDirectory() as source_root:
        repo_name = "parent_repo"
        repo_path = os.path.join(source_root, repo_name)
        cloned_repo_name = "cloned_repo"
        cloned_repo_path = os.path.join(source_root, cloned_repo_name)
        project_name = "foo"  # does not matter for this test

        os.mkdir(repo_path)

        def mock_get_repos(*args, **kwargs):
            return [get_repository(cloned_repo_path,
                                   "git", project_name)]

        def mock_get(*args, **kwargs):
            return MockResponse()

        # Clone a Git repository so that it can pull.
        repo = Repo.init(repo_path)
        with repo.config_writer() as git_config:
            git_config.set_value('user', 'email', 'someone@example.com')
            git_config.set_value('user', 'name', 'John Doe')

        new_file_path = os.path.join(repo_path, 'foo')
        with open(new_file_path, 'w'):
            pass
        assert os.path.isfile(new_file_path)
        index = repo.index
        index.add([new_file_path])
        index.commit("add file")
        repo.clone(cloned_repo_path)

        with monkeypatch.context() as m:
            m.setattr(sys, 'argv', ['prog', "-I", project_name])

            # With mocking done via pytest it is necessary to patch
            # the call-site rather than the absolute object path.
            m.setattr("opengrok_tools.mirror.get_config_value",
                      lambda x, y, z: source_root)
            m.setattr("opengrok_tools.utils.mirror.get_repos_for_project",
                      mock_get_repos)
            m.setattr("opengrok_tools.utils.mirror.do_api_call", mock_get)

            assert opengrok_tools.mirror.main() == CONTINUE_EXITVAL

def test_empty_project_config():
    assert check_project_configuration({'foo': None})

def test_disabled_command_api():
    """
    Test that mirror_project() calls call_rest_api() if API
    call is specified in the configuration for disabled project.
    """
    with patch(opengrok_tools.utils.mirror.call_rest_api,
               lambda a, b, c: mock(spec=requests.Response)):
        project_name = "foo"
        config = {
            DISABLED_CMD_PROPERTY: {
                COMMAND_PROPERTY: [
                    "http://localhost:8080/source/api/v1/foo",
                    "POST", "data"]
            },
            PROJECTS_PROPERTY: {
                project_name: {DISABLED_PROPERTY: True}
            }
        }

        assert mirror_project(config, project_name, False,
                              None, None) == CONTINUE_EXITVAL
        verify(opengrok_tools.utils.mirror). \
            call_rest_api(config.get(DISABLED_CMD_PROPERTY),
                          PROJECT_SUBST, project_name)

def test_disabled_command_api_text_append(monkeypatch):
    """
    Test that message text is appended if DISABLED_PROPERTY is a string.
    """

    text_to_append = "foo bar"

    def mock_call_rest_api(command, b, c):
        disabled_command = config.get(DISABLED_CMD_PROPERTY)
        assert disabled_command
        command_args = disabled_command.get(COMMAND_PROPERTY)
        assert command_args
        data = command_args[2]
        assert data
        text = data.get("text")
        assert text
        assert text.find(text_to_append)

        return mock(spec=requests.Response)

    with monkeypatch.context() as m:
        m.setattr("opengrok_tools.utils.mirror.call_rest_api",
                  mock_call_rest_api)

        project_name = "foo"
        data = {'messageLevel': 'info', 'duration': 'PT5M',
                'tags': ['%PROJECT%'],
                'text': 'disabled project'}
        config = {
            DISABLED_CMD_PROPERTY: {
                COMMAND_PROPERTY: [
                    "http://localhost:8080/source/api/v1/foo",
                    "POST", data]
            },
            PROJECTS_PROPERTY: {
                project_name: {
                    DISABLED_REASON_PROPERTY: text_to_append,
                    DISABLED_PROPERTY: True
                }
            }
        }

        mirror_project(config, project_name, False, None, None)

def test_disabled_command_run():
    """
    Make sure that mirror_project() results in calling run_command().
    """
    spy2(opengrok_tools.utils.mirror.run_command)
    project_name = "foo"
    config = {
        DISABLED_CMD_PROPERTY: {
            COMMAND_PROPERTY: ["cat"]
        },
        PROJECTS_PROPERTY: {
            project_name: {DISABLED_PROPERTY: True}
        }
    }

    assert mirror_project(config, project_name, False,
                          None, None) == CONTINUE_EXITVAL
    verify(opengrok_tools.utils.mirror).run_command(ANY, project_name)

def test_disabled_command_run_args():
    """
    Make sure that run_command() calls Command.execute().
    """
    cmd = mock(spec=Command)
    project_name = "foo"
    run_command(cmd, project_name)
    verify(cmd).execute()

def test_mirror_project_timeout(monkeypatch):
    """
    Test mirror_project() timeout inheritance/override from global
    configuration to get_repos_for_project(). The test merely verifies
    that the timeout values are passed between the expected functions,
    not whether it actually affects the execution.
    """
    cmd_timeout = 3
    hook_timeout = 4

    def mock_get_repos(*args, **kwargs):
        mock_get_repos.called = True

        assert kwargs[CMD_TIMEOUT_PROPERTY] == cmd_timeout

        # Technically this function should return list of Repository objects
        # however for this test this is not necessary.
        return ['foo']

    def mock_process_hook(hook_ident, hook, source_root, project_name_arg,
                          proxy, hook_timeout_arg):
        assert hook_timeout_arg == hook_timeout

        # We want to terminate mirror_project() once this function runs.
        # This way mirror_project() will return FAILURE_EXITVAL
        return False

    def test_mirror_project(config):
        retval = mirror_project(config, project_name, False,
                                "http://localhost:8080/source", "srcroot")
        assert retval == FAILURE_EXITVAL

        # TODO: is there better way to ensure that get_repos_for_project()
        #       was actually called ?
        assert mock_get_repos.called

    with monkeypatch.context() as m:
        mock_get_repos.called = False
        m.setattr("opengrok_tools.utils.mirror.get_repos_for_project",
                  mock_get_repos)
        m.setattr("opengrok_tools.utils.mirror.process_hook",
                  mock_process_hook)

        project_name = "foo"
        # override testing
        global_config_1 = {
            PROJECTS_PROPERTY: {
                project_name: {
                    CMD_TIMEOUT_PROPERTY: cmd_timeout,
                    HOOK_TIMEOUT_PROPERTY: hook_timeout
                }
            },
            CMD_TIMEOUT_PROPERTY: cmd_timeout * 2,
            HOOK_TIMEOUT_PROPERTY: hook_timeout * 2
        }
        # inheritance testing
        global_config_2 = {
            CMD_TIMEOUT_PROPERTY: cmd_timeout,
            HOOK_TIMEOUT_PROPERTY: hook_timeout
        }

        test_mirror_project(global_config_1)
        test_mirror_project(global_config_2)

def test_get_repos_for_project(monkeypatch):
    """
    Test argument passing between get_repos_for_project() and get_repository()
    """
    project_name = 'foo'
    proxy_dict = {"http_proxy": "http://foo.bar:80",
                  "https_proxy": "http://foo.bar:80"}
    git_cmd_path = "/foo/git"
    commands = {"git": git_cmd_path}
    timeout = 314159
    test_repo = "/" + project_name

    def mock_get_repos(*args):
        return [test_repo]

    def mock_get_repo_type(*args):
        return "Git"

    with tempfile.TemporaryDirectory() as source_root:
        # Note that it is actually not necessary to create real
        # Git repository for the test to work. This is due to
        # the way how Repository objects are created.
        with monkeypatch.context() as m:
            m.setattr("opengrok_tools.utils.mirror.get_repos",
                      mock_get_repos)
            m.setattr("opengrok_tools.utils.mirror.get_repo_type",
                      mock_get_repo_type)

            repos = get_repos_for_project(project_name, None, source_root,
                                          commands=commands,
                                          proxy=proxy_dict,
                                          command_timeout=timeout)
            assert len(repos) == 1
            assert isinstance(repos[0], GitRepository)
            git_repo = repos[0]
            assert git_repo.timeout == timeout
            assert git_repo.command == git_cmd_path
            assert git_repo.env.items() >= proxy_dict.items()

            # Now ignore the repository
            repos = get_repos_for_project(project_name, None, source_root,
                                          ignored_repos=['.'])
            assert len(repos) == 0

DEFAULT_COMMAND = 'default-command'

@pytest.mark.parametrize(['expected_command', 'config'], [
    (DEFAULT_COMMAND, None),
    ('/usr/bin/git', '/usr/bin/git'),
    (DEFAULT_COMMAND, {}),
    (DEFAULT_COMMAND, {'incoming': '/bin/false'}),
    (DEFAULT_COMMAND, []),
    ('/usr/local/bin/git', {'command': '/usr/local/bin/git'}),
    (
            '/usr/local/bin/git',
            {'command': '/usr/local/bin/git', 'incoming': '/bin/false'}
    )
])
def test_mirroring_custom_repository_command(expected_command, config):
    assert expected_command == Repository._repository_command(config, lambda: DEFAULT_COMMAND)

@pytest.mark.parametrize(
    ('touch_binary'), [
        pytest.param('/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/touch'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/touch'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_invoke_command(touch_binary):
    checking_file = 'incoming.txt'
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'incoming': [touch_binary, checking_file]
        }, None, None, None)
        assert repository.incoming() is False
        assert checking_file in os.listdir(repository_root)

@pytest.mark.parametrize(
    ('echo_binary'), [
        pytest.param('/bin/echo',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/echo'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/echo',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/echo'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_changes(echo_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'incoming': [echo_binary, 'new incoming changes!']
        }, None, None, None)
        assert repository.incoming() is True

@pytest.mark.parametrize(
    ('true_binary'), [
        pytest.param('/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/true'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/true'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_no_changes(true_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'incoming': true_binary
        }, None, None, None)
        assert repository.incoming() is False

@pytest.mark.parametrize(
    ('false_binary'), [
        pytest.param('/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/false'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/false'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_incoming_error(false_binary):
    with pytest.raises(RepositoryException):
        with tempfile.TemporaryDirectory() as repository_root:
            repository = GitRepository(mock(), repository_root, 'test-1', {
                'incoming': false_binary
            }, None, None, None)
            repository.incoming()

def test_mirroring_incoming_invoke_original_command():
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1',
                                   None, None, None, None)
        with when(repository).incoming_check().thenReturn(0):
            repository.incoming()
            verify(repository).incoming_check()

@pytest.mark.parametrize(
    ('touch_binary'), [
        pytest.param('/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/touch'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/touch',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/touch'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_sync_invoke_command(touch_binary):
    checking_file = 'sync.txt'
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'sync': [touch_binary, checking_file]
        }, None, None, None)
        assert repository.sync() == 0
        assert checking_file in os.listdir(repository_root)

@pytest.mark.parametrize(
    ('true_binary'), [
        pytest.param('/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/true'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/true',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/true'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_sync_success(true_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'sync': true_binary
        }, None, None, None)
        assert 0 == repository.sync()

@pytest.mark.parametrize(
    ('false_binary'), [
        pytest.param('/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/bin/false'),
                         reason="requires /bin binaries")),
        pytest.param('/usr/bin/false',
                     marks=pytest.mark.skipif(
                         not os.path.exists('/usr/bin/false'),
                         reason="requires /usr/bin binaries")),
    ]
)
def test_mirroring_custom_sync_error(false_binary):
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1', {
            'sync': false_binary
        }, None, None, None)
        assert 1 == repository.sync()

def test_mirroring_sync_invoke_original_command():
    with tempfile.TemporaryDirectory() as repository_root:
        repository = GitRepository(mock(), repository_root, 'test-1',
                                   None, None, None, None)
        with when(repository).reposync().thenReturn(0):
            repository.sync()
            verify(repository).reposync()


from django import forms
from betterforms.forms import Fieldset, BetterModelForm
from assistenten.models import FesteSchicht
from assistenten.widgets import XDSoftTimePickerInput

class FesteSchichtenForm(BetterModelForm):
    class Meta:
        fields = ['wochentag', 'beginn', 'ende']
        model = FesteSchicht
        liste = ['auauauaua', 2, 3]
        fieldsets = (
            Fieldset('feste_schicht', fields=('wochentag',
                                              'beginn',
                                              'ende'),
                     legend='Feste Schicht hinzufügen',
                     template_name="assistenten/fieldset_feste_schichten.html",
                     liste=liste),
        )

    CHOICES = [
        ('-1', 'keine neue feste Schicht'),
        ('1', 'Montag'),
        ('2', 'Dienstag'),
        ('3', 'Mittwoch'),
        ('4', 'Donnerstag'),
        ('5', 'Freitag'),
        ('6', 'Samstag'),
        ('7', 'Sonntag')]
    wochentag = forms.CharField(label='Wochentag', widget=forms.Select(choices=CHOICES))
    beginn = forms.TimeField(
        input_formats=['%H:%M'],
        widget=XDSoftTimePickerInput(attrs={'autocomplete': 'off'}),
        required=False
    )
    ende = forms.TimeField(
        input_formats=['%H:%M'],
        widget=XDSoftTimePickerInput(attrs={'autocomplete': 'off'}),
        required=False
    )


"""
question:
https://leetcode.cn/problems/robot-in-a-grid-lcci/
"""
from typing import List

class Solution:
    def pathWithObstacles(self, obstacleGrid: List[List[int]]) -> List[List[int]]:
        m, n = len(obstacleGrid), len(obstacleGrid[0])
        if obstacleGrid[0][0] == 1 or obstacleGrid[-1][-1] == 1:
            return []

        path = []
        end = False
        visited = [[0] * n for _ in range(m)]

        def dfs(row, col):
            nonlocal end
            if visited[row][col] == 1 or obstacleGrid[row][col] == 1:
                return
            visited[row][col] = 1
            path.append([row, col])
            if row == m - 1 and col == n - 1:
                end = True
                return
            if row + 1 < m:
                dfs(row + 1, col)
                if end:
                    return
            if col + 1 < n:
                dfs(row, col + 1)
                if end:
                    return
            path.pop()

        dfs(0, 0)
        # print(path)
        return path

    def pathWithObstacles2(self, obstacleGrid: List[List[int]]) -> List[List[int]]:
        m, n = len(obstacleGrid), len(obstacleGrid[0])
        if obstacleGrid[0][0] == 1 or obstacleGrid[-1][-1] == 1:
            return []

        path = []

        def dfs(row, col):
            if obstacleGrid[row][col] == 1:
                return
            obstacleGrid[row][col] = 1
            path.append([row, col])
            if row == m - 1 and col == n - 1:
                return True
            if row + 1 < m and dfs(row + 1, col):
                return True
            if col + 1 < n and dfs(row, col + 1):
                return True
            path.pop()

        dfs(0, 0)
        # print(path)
        return path

    def pathWithObstacles3(self, obstacleGrid: List[List[int]]) -> List[List[int]]:
        m, n = len(obstacleGrid), len(obstacleGrid[0])
        if obstacleGrid[0][0] == 1 or obstacleGrid[-1][-1] == 1:
            return []

        dp = [[-1] * n for _ in range(m)]
        dp[0][0] = 0

        for i in range(m):
            for j in range(n):
                if obstacleGrid[i][j] == 1:
                    continue
                if i > 0 and dp[i - 1][j] != -1:
                    dp[i][j] = (i - 1) * n + j
                elif j > 0 and dp[i][j - 1] != -1:
                    dp[i][j] = i * n + j - 1

        i, j = m - 1, n - 1
        if dp[i][j] == -1:
            return []
        path = [[i, j]]
        while i != 0 or j != 0:
            i, j = dp[i][j] // n, dp[i][j] % n
            path.append([i, j])

        # print(path[::-1])
        return path[::-1]

if __name__ == '__main__':
    assert Solution().pathWithObstacles3([
        [0, 0, 0],
        [0, 1, 0],
        [0, 0, 0]
    ]) == [[0, 0], [0, 1], [0, 2], [1, 2], [2, 2]]
    assert Solution().pathWithObstacles3([[1]]) == []
    assert Solution().pathWithObstacles3([[0]]) == [[0, 0]]
    assert Solution().pathWithObstacles3([[0, 0], [1, 0]]) == [[0, 0], [0, 1], [1, 1]]


import copy
from collections import OrderedDict
from typing import List, Tuple

import torch
from torch import Tensor, nn
from torch.optim import Adam, Optimizer
from torch.utils.data import DataLoader

from agent.agent_value_based import ValueBasedAgent
from env.env import Env
from model.model import Model
from system.dqn.dataset import RLDataset
from system.dqn.replay_buffer import ReplayBuffer

from ..system import System

class DqnSystem(System):
    def __init__(
        self,
        env: Env,
        model: Model,
        agent: ValueBasedAgent,
        *args,
        batch_size: int = 16,
        lr: float = 1e-2,
        gamma: float = 0.99,
        sync_rate: int = 10,
        replay_size: int = 1000,
        warm_start_size: int = 1000,
        eps_last_frame: int = 1000,
        eps_start: float = 1.0,
        eps_end: float = 0.01,
        episode_length: int = 200,
        warm_start_steps: int = 1000,
        **kwargs,
    ) -> None:

        super().__init__(env, model, agent, *args, **kwargs)
        self.save_hyperparameters(ignore=["env", "model", "agent"])

        self.env = env

        self.net = model
        self.target_net = copy.deepcopy(model)

        self.buffer = ReplayBuffer(self.hparams.replay_size)
        self.agent = agent

        self.total_reward = 0
        self.episode_reward = 0

        self.populate(self.hparams.warm_start_steps)

    def populate(self, steps: int = 1000) -> None:
        for _ in range(steps):
            self.agent.play_step(self.net, epsilon=1.0, replay_buffer=self.buffer)

    def forward(self, x: Tensor) -> Tensor:
        output = self.net(x)
        return output

    def dqn_mse_loss(self, batch: Tuple[Tensor, Tensor]) -> Tensor:
        states, actions, rewards, dones, next_states = batch

        state_action_values = (
            self.net(states).gather(1, actions.long().unsqueeze(-1)).squeeze(-1)
        )

        with torch.no_grad():
            next_state_values = self.target_net(next_states).max(1)[0]
            next_state_values[dones] = 0.0
            next_state_values = next_state_values.detach()

        expected_state_action_values = next_state_values * self.hparams.gamma + rewards

        return nn.MSELoss()(state_action_values, expected_state_action_values)

    def get_epsilon(self, start: int, end: int, frames: int) -> float:
        if self.global_step > frames:
            return end
        return start - (self.global_step / frames) * (start - end)

    def training_step(self, batch: Tuple[Tensor, Tensor], nb_batch) -> OrderedDict:
        device = self.get_device(batch)
        epsilon = self.get_epsilon(
            self.hparams.eps_start, self.hparams.eps_end, self.hparams.eps_last_frame
        )
        self.log("epsilon", epsilon)

        # step through environment with agent
        reward, done = self.agent.play_step(
            self.net, epsilon, device, replay_buffer=self.buffer
        )
        self.episode_reward += reward
        self.log("episode reward", self.episode_reward)

        # calculates training loss
        loss = self.dqn_mse_loss(batch)

        if done:
            self.total_reward = self.episode_reward
            self.episode_reward = 0

        # Soft update of target network
        if self.global_step % self.hparams.sync_rate == 0:
            self.target_net.load_state_dict(self.net.state_dict())

        self.log_dict(
            {
                "reward": reward,
                "train_loss": loss,
            }
        )
        self.log("total_reward", self.total_reward, prog_bar=True)
        self.log("steps", self.global_step, logger=False, prog_bar=True)

        return loss

    def configure_optimizers(self) -> List[Optimizer]:
        """Initialize Adam optimizer."""
        optimizer = Adam(self.net.parameters(), lr=self.hparams.lr)
        return optimizer

    def __dataloader(self) -> DataLoader:
        """Initialize the Replay Buffer dataset used for retrieving experiences."""
        dataset = RLDataset(self.buffer, self.hparams.episode_length)
        dataloader = DataLoader(dataset=dataset, batch_size=self.hparams.batch_size)
        return dataloader

    def train_dataloader(self) -> DataLoader:
        """Get train loader."""
        return self.__dataloader()

    def get_device(self, batch) -> str:
        """Retrieve device currently being used by minibatch."""
        return batch[0].device.index if self.on_gpu else "cpu"


import os
import sys
import json
import cv2
import gc
import sqlite3
import io
import numpy as np

from imutils.video import FileVideoStream
from multiprocessing import Pool, Lock

CLIPS_DIRECTORY="./dataset_dmd/clips"
ALLOWED_THREADS = 32
lock = Lock()
class_num_iter = 0
vid_num = 0

db = sqlite3.connect('dmd.db',check_same_thread=False)

cur = db.cursor()

cur.execute('''CREATE TABLE IF NOT EXISTS Data(
                    ID INTEGER PRIMARY KEY AUTOINCREMENT,
                    label TEXT, vid_num INT, frame INT,
                    bytes ARRAY, age TEXT,
                    gender TEXT, exp TEXT,
                    freq TEXT
                    )''')

def adapt_array(arr):
    """
    http://stackoverflow.com/a/31312102/190597 (SoulNibbler)
    """
    out = io.BytesIO()
    np.save(out, arr)
    out.seek(0)
    return sqlite3.Binary(out.read())

def convert_array(text):
    import io
    out = io.BytesIO(text)
    out.seek(0)
    return np.load(out)

def save_image_to_db(img,metadata,label,frame_num,vid_num):
    blob = adapt_array(img)
    age = metadata["age"]
    gender = metadata["gender"]
    exp = metadata["exp"]
    freq = metadata["d_freq"]

    sql = ''' INSERT INTO Data(label,vid_num,frame,bytes,age,gender,exp,freq)
    VALUES(?,?,?,?,?,?,?,?) '''
    db.execute(sql, (label,vid_num,frame_num,blob,age,gender,exp,freq))
    db.commit()

sqlite3.register_adapter(np.ndarray, adapt_array)
sqlite3.register_converter("ARRAY", convert_array)

def parse_json_file(path):
    l = open(path)
    j = json.load(l)
    valid_mdata = False
    valid_labels = False
    mdata = {}
    #Boolean array to verify the clip is a valid clip
    for k in j.keys():
        age = 0
        gender = ''
        exp = ''
        d_freq = ''

        events = []
        frame_data = []
        c = 0

        print(j[k].keys())
        metadata_str = 'objects'

        #Grabs meta
        if metadata_str in j[k].keys():
            m_data = j[k][metadata_str]
            for d in m_data.keys():
                if 'type' in m_data[d].keys():

                    if m_data[d]['type'] == 'driver':
                        if 'object_data' in m_data[d].keys():
                            driver_info = m_data[d]['object_data']
                            for category in driver_info['num']:
                                if category['name'] == 'age':
                                    age = category['val']
                                    c+=1
                            for attribute in m_data[d]['object_data']['text']:
                                if attribute['name'] == "gender":
                                    gender = attribute['val']
                                    c+=1
                                elif attribute['name'] == 'experience':
                                    exp = attribute['val']
                                    c+=1
                                elif attribute['name'] == 'drive_freq':
                                    d_freq = attribute['val']
                                    c+=1
                                else:
                                    print(f"DIFFERENT ATTRIBUTE THAN ACCOUNTED FOR {attribute}")
        else:
            print("NO METADATA TO EXTRACT")

        framedata_str = 'actions'

        if framedata_str in j[k].keys():
            actions = j[k][framedata_str]
            for t in actions.keys():
                event = actions[t]
                label = ''
                if 'type' in event.keys():
                    label = event['type']
                    if event['type'] not in events:
                        events.append(label)

                    if "frame_intervals" in event.keys():
                        entries = event["frame_intervals"]
                        for entry in entries:
                            st = entry["frame_start"]
                            en = entry["frame_end"]
                            if label == '':
                                print("bad")
                                break
                            else:
                                valid_labels = True
                                frame_data.append( (st,en,label) )
        #Valid clip
        if c == 4:
            print("CLIP VALID")
            mdata = {'age':age, 'gender':gender, 'exp':exp, 'd_freq':d_freq}
            valid_mdata = True

    if valid_mdata and valid_labels:
        return mdata,frame_data,events
    else:
        return None

def return_start_frame_number(entry):
    return entry[0]

def get_clip_object(match_str,l,st,frame_size):
    fourcc = cv2.VideoWriter_fourcc('M','J','P','G')
    fname = CLIPS_DIRECTORY+"/"+match_str+";"+l.replace("/","-")+f";{st}.mp4"
    if os.path.exists(fname):
        return None
    else:
        lock.acquire()
        w = cv2.VideoWriter(CLIPS_DIRECTORY+"/"+match_str+";"+l.replace("/","-")+f";{st}.mp4", fourcc, 20, frame_size)
        lock.release()
        return w

def write_clip(path,st,en,l,match_str,frame_size,vid_num,class_num,metadata):
    label = l.replace("/","-")
    fname = CLIPS_DIRECTORY+"/"+match_str+";"+l.replace("/","-")+f";{st}.mp4"

    print(path)
    lock.acquire()
    capture = FileVideoStream(path).start()
    lock.release()

    c = 0

    use_db = False

    print(vid_num)

    frames_directory = f"./dataset_dmd/clips/{label}/{vid_num}/"
    if not os.path.exists(frames_directory):
        os.makedirs(frames_directory)

    while True:
        if c == st:

            for i in range(0,en-st):
                frame = capture.read()
                good = type(frame) != type(None)

                if good:
                    print(st,en,l,i)
                    image_name = "img_{:07d}.jpg".format(i)
                    if use_db:
                        lock.acquire()
                        save_image_to_db(frame,metadata,label,i,vid_num)
                        lock.release()
                    else:
                        cv2.imwrite(frames_directory+image_name,frame)

                else:
                    capture.stop()
                    print(st,en,l,i)
                    print("STREAM BAD")
                    gc.collect()
                    return None
                c+=1
            break
        elif c < st:
            frame = capture.read()
            good = type(frame) != type(None)
            if not good:
                capture.stop()
                print(st,en,l,)
                print("STREAM BAD")
                gc.collect()
                return None
            c+=1

    lock.acquire()
    capture.stop()

    with open(CLIPS_DIRECTORY+"/annotations.txt","a+") as f:
        f.write(f"{label}/{vid_num} {en-st} {class_num} {metadata}\n")
    lock.release()

    gc.collect()
    return None

def clip_segmenter(path,match_str,segments,global_labels,metadata):

    class_counters = {}
    pool = Pool(ALLOWED_THREADS)

    segments = sorted(segments,key=return_start_frame_number)
    threads = [0 for i in range(ALLOWED_THREADS)]

    capture2 = cv2.VideoCapture(path)

    frame_width = int(capture2.get(3))
    frame_height = int(capture2.get(4))
    frame_size = (frame_width,frame_height)

    capture2.release()

    c = 0
    for st,en,l in segments:
        # write_clip(path,st,en,l,match_str)
        label = l
        if label not in class_counters.keys():
            class_counters[label] = 1
            vid_num = 1
        else:
            class_counters[label] += 1
            vid_num = class_counters[label]

        if c < ALLOWED_THREADS:
            p = pool.apply_async(write_clip, (path,st,en,l,match_str,frame_size,vid_num,global_labels[l],metadata,))
            threads[c] = (p,st,en,l)
            gc.collect()
            c+=1
        else:
            for i in range(ALLOWED_THREADS):
                t,st1,en1,l1 = threads[i]
                print(t)
                if t != 0:
                    t.get()
                    gc.collect()

                threads[i] = 0
            c = 0
    a = 0
    for i in range(ALLOWED_THREADS):
        if threads[i] != 0:
            t,st1,en1,l1 = threads[i]
            if t != 0:
                t.get()
                gc.collect()

        a+=1

def iterate_data(pairs,global_labels):
    global class_num_iter
    total_labels = []
    features = {}
    for key in pairs.keys():
        json_path,vid_path = pairs[key]

        #If no video was found
        if vid_path == 0:
            continue

        check = parse_json_file(json_path)

        if check != None:
            metadata,segs,labels_set = check
            for label in labels_set:
                if label not in global_labels.keys():
                    global_labels[label] = class_num_iter
                    class_num_iter+=1

            clip_segmenter(vid_path,key,segs,global_labels,metadata)

            features[key] = metadata

#Connect json file to mp4 files
def create_file_pairs(path):
    file_pairs = {}
    for root, dirs, files in os.walk(path, topdown=False):
        for f in files:
            if "clips" in root:
                continue

            if ".mp4" in f:
                match_string = f.split(";")[0]
                if match_string not in file_pairs.keys():
                    file_pairs[match_string] = [0,root+"/"+f]
                else:
                    tmp = file_pairs[match_string]
                    tmp[1] = root+"/"+f
                    file_pairs[match_string]= tmp
            if ".json" in f:
                match_string = f.split(";")[0]
                if match_string not in file_pairs.keys():
                    file_pairs[match_string] = [root+"/"+f,0]
                else:
                    tmp = file_pairs[match_string]
                    tmp[0] = root+"/"+f
                    file_pairs[match_string]= tmp

    return file_pairs

if __name__ == "__main__":
    clear_data = True

    if clear_data and os.path.exists(CLIPS_DIRECTORY):
        import shutil
        shutil.rmtree(CLIPS_DIRECTORY)

    if not os.path.exists(CLIPS_DIRECTORY):
        os.mkdir(CLIPS_DIRECTORY)

    all_labels = {}
    file_pairs = create_file_pairs("./dataset_dmd")
    iterate_data(file_pairs,all_labels)