import copy
from concurrent.futures import ThreadPoolExecutor

from django.http import JsonResponse
from django.views.decorators.cache import never_cache
from django.views.generic import TemplateView

from health_check.conf import HEALTH_CHECK
from health_check.exceptions import ServiceWarning
from health_check.plugins import plugin_dir

class MainView(TemplateView):
    template_name = 'health_check/index.html'

    @never_cache
    def get(self, request, *args, **kwargs):
        errors = []

        plugins = sorted((
            plugin_class(**copy.deepcopy(options))
            for plugin_class, options in plugin_dir._registry
        ), key=lambda plugin: plugin.identifier())

        def _run(plugin):
            plugin.run_check()
            try:
                return plugin
            finally:
                from django.db import connection
                connection.close()

        with ThreadPoolExecutor(max_workers=len(plugins) or 1) as executor:
            for plugin in executor.map(_run, plugins):
                if plugin.critical_service:
                    if not HEALTH_CHECK['WARNINGS_AS_ERRORS']:
                        errors.extend(
                            e for e in plugin.errors
                            if not isinstance(e, ServiceWarning)
                        )
                    else:
                        errors.extend(plugin.errors)

        status_code = 500 if errors else 200

        if 'application/json' in request.META.get('HTTP_ACCEPT', ''):
            return self.render_to_response_json(plugins, status_code)

        context = {'plugins': plugins, 'status_code': status_code}

        return self.render_to_response(context, status=status_code)

    def render_to_response_json(self, plugins, status):
        return JsonResponse(
            {str(p.identifier()): str(p.pretty_status()) for p in plugins},
            status=status
        )


from itertools import product
from unittest import mock, TestCase, main

if __package__:
    from . import solution
else:
    import solution

path = f'{__package__}{"." if __package__ else ""}solution'

@mock.patch(path + '.print', create=True)
@mock.patch(path + '.input', create=True)
class FizzBuzzSolutionTests(TestCase):
    def setUp(self):
        self.versions = ('1', '2', '3')

        expected_results = [
            (1, ), (2, ), ('Fizz', ), (4, ), ('Buzz', ), ('Fizz', ),
            (7, ), (8, ), ('Fizz', ), ('Buzz', ), (11, ), ('Fizz', ),
            (13, ), (14, ), ('FizzBuzz', )
        ]
        self.expected = [
            (s, expected_results[:i]) for i, s in enumerate(('1', '5', '15'))
        ]

        self.errors_str = ('qwe', repr(''), '-1', '0', '000')
        self.expected_error = ('Введено некорректное значение', )

    def test_main(self, mocked_input, mocked_print):
        for (user_input, result), v in product(self.expected, self.versions):
            with self.subTest(user_input=user_input, version=v, result=result):
                mocked_input.side_effect = [user_input, v]
                solution.main()
                for call, res in zip(mocked_print.call_args_list, result):
                    self.assertEqual(call.args, res)
                mocked_input.reset_mock()
                mocked_print.reset_mock()

    def test_main_errors(self, mocked_input, mocked_print):
        for user_input in self.errors_str:
            with self.subTest(user_input=user_input):
                mocked_input.side_effect = user_input
                solution.main()
                self.assertEqual(mocked_print.call_args.args,
                                 self.expected_error)
                mocked_input.reset_mock()
                mocked_print.reset_mock()

if __name__ == '__main__':
    main()


import torch.nn as nn
import torch.nn.functional as F
import resnet as resnet_mod
import pytorch_cifar.models.resnet as resnet
import classifier_models.preact_resnet as preact_resnet
from classifier_models import PreActResNet18, ResNet18
import collections

class BasicBlockNoReLU(nn.Module):
    expansion = 1

    def __init__(self, module):
        super().__init__()
        self.conv1 = module.conv1
        self.bn1 = module.bn1
        self.conv2 = module.conv2
        self.bn2 = module.bn2
        self.shortcut = module.shortcut

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        return out

class SequentialImageNetwork(nn.Sequential):
    def __init__(self, net=resnet.ResNet18()):
        if isinstance(net, collections.OrderedDict):
            super().__init__(net)
            return

        self.net_holder = (net,)
        i = 1
        layers = []
        name = f"layer{i}"
        while hasattr(net, name):
            print(name)
            layers.extend(list(getattr(net, name)))
            i += 1
            name = f"layer{i}"

        layers2 = []
        for layer in layers:
            if isinstance(layer, resnet.BasicBlock):
                layers2.append(BasicBlockNoReLU(layer))
                layers2.append(nn.ReLU())
            else:
                layers2.append(layer)

        super().__init__(
            net.conv1,
            net.bn1,
            nn.ReLU(),
            net.maxpool,
            *layers2,
            net.avgpool,
            nn.Flatten(),
            net.fc
        )

    @property
    def net(self):
        return self.net_holder[0]

class BasicBlockNoReLUpre(nn.Module):
    expansion = 1

    def __init__(self, module):
        super().__init__()
        self.conv1 = module.conv1
        self.bn1 = module.bn1
        self.conv2 = module.conv2
        self.bn2 = module.bn2
        if hasattr(module, "shortcut"):
            self.shortcut = module.shortcut

    def forward(self, x):
        out = F.relu(self.bn1(x))
        shortcut = self.shortcut(out) if hasattr(self, "shortcut") else x
        out = self.conv1(out)
        out = self.conv2(F.relu(self.bn2(out)))
        out += shortcut
        return out

class SequentialImageNetwork_pre(nn.Sequential):
    def __init__(self, net=PreActResNet18(10)):
        if isinstance(net, collections.OrderedDict):
            super().__init__(net)
            return

        self.net_holder = (net,)
        i = 1
        layers = []
        name = f"layer{i}"
        while hasattr(net, name):
            layers.extend(list(getattr(net, name)))
            i += 1
            name = f"layer{i}"

        layers2 = []
        for layer in layers:
            if isinstance(layer, preact_resnet.PreActBlock):
                layers2.append(BasicBlockNoReLUpre(layer))
            else:
                layers2.append(layer)

        super().__init__(
            net.conv1,
            *layers2,
            nn.AvgPool2d(8 if net.in_planes == 64 else 4),
            nn.Flatten(),
            net.linear,
        )

    @property
    def net(self):
        return self.net_holder[0]

class SequentialImageNetwork_imgnet(nn.Sequential):
    def __init__(self, net=PreActResNet18(10)):
        if isinstance(net, collections.OrderedDict):
            super().__init__(net)
            return

        self.net_holder = (net,)
        i = 1
        layers = []
        name = f"layer{i}"
        while hasattr(net, name):
            layers.extend(list(getattr(net, name)))
            i += 1
            name = f"layer{i}"

        layers2 = []
        for layer in layers:
            if isinstance(layer, preact_resnet.PreActBlock):
                layers2.append(BasicBlockNoReLUpre(layer))
            else:
                layers2.append(layer)

        super().__init__(
            net.conv1,
            net.bn1,
            net.relu,
            net.maxpool,
            *layers2,
            net.avgpool,
            nn.Flatten(),
            net.fc,
        )

    @property
    def net(self):
        return self.net_holder[0]

class BasicBlockSplitter(nn.Module):
    def __init__(self, block: resnet.BasicBlock, step="add"):
        super().__init__()
        self.block = block
        self.step = step

    def forward(self, x):
        if self.step == "identity":
            return x
        shortcut = self.block.shortcut(x)
        if self.step == "shortcut":
            return shortcut
        x = self.block.conv1(x)
        if self.step == "conv1":
            return x
        x = self.block.bn1(x)
        if self.step == "bn1":
            return x
        x = F.relu(x)
        if self.step == "relu1":
            return x
        x = self.block.conv2(x)
        if self.step == "conv2":
            return x
        x = self.block.bn2(x)
        if self.step == "bn2":
            return x
        x += shortcut
        if self.step == "add":
            return x
        x = F.relu(x)
        if self.step == "relu2":
            return x
        return x

class SequentialImageNetworkMod(nn.Sequential):
    def __init__(self, net=resnet_mod.resnet32()):
        if isinstance(net, collections.OrderedDict):
            super().__init__(net)
            return

        self.net_holder = (net,)
        i = 1
        layers = []
        name = f"layer{i}"
        while hasattr(net, name):
            layers.extend(list(getattr(net, name)))
            i += 1
            name = f"layer{i}"

        super().__init__(
            net.conv1,
            *layers,
            net.final_bn,
            nn.LeakyReLU(0.1),
            nn.AvgPool2d(8),
            nn.Flatten(),
            net.linear,
        )

    @property
    def net(self):
        return self.net_holder[0]


"""
To run this app, after you've the data, execute the following:

pip install -r requirements.txt
cd src
streamlit run app.py
"""

from collections import Counter

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import spacy
import streamlit as st
from textacy import Corpus, extract
from textacy.extract.basics import entities, ngrams, noun_chunks
from textacy.extract.keyterms.yake import yake
from pathlib import Path
from typing import List

DEBUG = True

# Define Text
st.title("Text Exploration App")
st.write(
    "Depending on your hard disk and CPU, this can take a few minutes to run. Be patient!"
)
# File Paths
data_dir = Path("../data/raw")
assert data_dir.exists()
clients = {"Uber": data_dir / "Uber_us_app_store_reviews.json"}
file_path = clients["Uber"]
assert file_path.exists()

st.sidebar.title("Options")
word_count = st.sidebar.checkbox("Word Counts")
ngrams_count = st.sidebar.checkbox("n-grams")
show_ncs = st.sidebar.checkbox("Noun", value=False)
show_ents = st.sidebar.checkbox("Entities", value=False)
show_yake = st.sidebar.checkbox("YAKE", value=True)

@st.cache
def load_data(file_path: Path) -> List[str]:
    """Load data as a list of reviews"""
    with file_path.open("r") as f:
        df = pd.read_json(f)
    reviews = df.review.to_list()

    return reviews

def make_corpus(reviews: List[str]):
    corpus = Corpus("en_core_web_sm", data=reviews)
    return corpus

# Reduce Corpus Load
reviews = load_data(file_path)
count = st.number_input(
    label="Number of Reviews you want to analyze", value=50, max_value=len(reviews)
)

if DEBUG:
    reviews = reviews[:count]

corpus = make_corpus(reviews)
st.write(
    f"Total Reviews: {corpus.n_docs},\nNumber of Sentences: {corpus.n_sents},\nNumber of words/tokens: {corpus.n_tokens}"
)

def get_word_counts(
    corpus=corpus, by="lemma_", filter_stops=True, filter_nums=True, filter_punct=True
):
    word_counts = corpus.word_counts(
        by="lemma_", filter_stops=True, filter_nums=True, filter_punct=True
    )
    most_common = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)[:25]
    most_common_df = pd.DataFrame(most_common, columns=["Phrase", "Count"])
    return most_common_df

def plot_count(most_common_df: pd.DataFrame, n:int=10):
    fig, ax = plt.subplots()
    sns.histplot(data=most_common_df[:10], x="Count", y="Phrase")
    st.pyplot(fig)

if word_count:
    st.write("## Word Counts")

    most_common_df = get_word_counts(corpus)
    if DEBUG:
        st.dataframe(most_common_df[:10])
    plot_count(most_common_df)

def spacy_span_hash(input_span):
    return str(input_span)

def flatten_list(l:List)->List:
    return [val for sublist in l for val in sublist]

@st.cache(hash_funcs={spacy.tokens.span.Span: spacy_span_hash})
def get_count(list_of_doc_terms: List[List], lower=True) -> pd.DataFrame:
    terms_list = flatten_list(list_of_doc_terms)
    if lower:
        terms_list = [str(term).lower() for term in terms_list]
    else:
        terms_list = [str(term) for term in terms_list]
    terms_freq = dict(Counter(terms_list))
    df = pd.DataFrame.from_dict(terms_freq, orient="index")
    df.reset_index(inplace=True)
    df.rename(columns={"index": "Phrase", 0: "Count"}, inplace=True)
    # st.write(df.columns)
    df.sort_values(by=["Count", "Phrase"], inplace=True, ascending=False)
    return df

def get_linguistic_counts(corpus=corpus, by:str="noun_chunks")->pd.DataFrame:
    """
    Extract the linguistic content needed

    Args:
        corpus ([type], optional): [description]. Defaults to corpus.
        by (str, optional): [description]. Defaults to "noun_chunks".

    Returns:
        df (pd.DataFrame):
    """
    if by == "noun_chunks":
        list_of_doc_terms = [
            list(noun_chunks(doc, drop_determiners=True)) for doc in corpus
        ]
        return get_count(list_of_doc_terms)
    elif by == "ents":
        list_of_doc_terms = [
            list(entities(doc, drop_determiners=True, exclude_types="NUMERIC"))
            for doc in corpus
        ]
        return get_count(list_of_doc_terms)
    elif by == "yake":
        list_of_doc_terms = [yake(doc, topn=5) for doc in corpus]
        terms_list = flatten_list(list_of_doc_terms)
        df = pd.DataFrame(terms_list, columns=["Word", "Score"])
        df.sort_values(by=["Score", "Word"], inplace=True,ascending=False)
        return df

if show_ncs:
    st.write("## Noun Phrases")
    df = get_linguistic_counts(corpus)
    plot_count(df)
    st.write(df)

if show_ents:
    st.write("## Entities")
    st.write(get_linguistic_counts(corpus, by="ents"))

if show_yake:
    st.write("## Keywords by YAKE")
    st.write(get_linguistic_counts(corpus, by="yake"))


import os
import sys
from utils import submit

cache_dir = sys.argv[1]
host_name = sys.argv[2]
patient_dimension = sys.argv[3]
time_series = sys.argv[4]
year = sys.argv[5]
output_dir = sys.argv[6]

submit(host_name, cache_dir, "datatrans.PreprocPerPatSeriesRowCol",
           "--patient_dimension={0}".format(patient_dimension),
           "--time_series=" + time_series,
           "--output_file=" + output_dir,
           "--year=" + year, *sys.argv[7:])


from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras import backend as K
K.set_image_dim_ordering('th')

model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(3, 300, 300)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(13))
model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

train_datagen = ImageDataGenerator(
        rescale=1./255,
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True)

test_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(
        'images_dataset/Train Directory',
        target_size=(300, 300),
        batch_size = 16

        )

validation_generator = test_datagen.flow_from_directory(
        'images_dataset/validation',
        target_size=(300, 300),
        batch_size= 16
        )

model.fit_generator(
        train_generator,
        steps_per_epoch=320 // 8,
        epochs=1,
        validation_data = validation_generator,
        validation_steps = 80 // 8
        )
model.save_weights('first_try.h5')


from tabnanny import check
import cv2
from matplotlib import ticker

#import video or open webcam
cap = cv2.VideoCapture("Mark.mp4")# Change the video name to number for webcam (0 for default webcam, 1,2,3,... for external webcam)
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml') #import the haarcascade_frontalface_default.xml
#? haarcascade_frontalface_default.xml is face detection file.

#show video
while (cap.isOpened()): #while video is open
    check, frame = cap.read() #read video
    if(check == True):#check if video has next frame.
        gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) #convert to grayscale
        face_detect  = face_cascade.detectMultiScale(gray_frame, scaleFactor=1.3, minNeighbors=5) #detect face, scaleFactor is the scale of the image, minNeighbors is the number of neighbors
        for (x, y, w, h) in face_detect: #for each face detected
            #cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), thickness=3) #draw rectangle around face
            frame[y:y+h+30, x:x+w+30] = cv2.blur(frame[y:y+h+30, x:x+w+30], (50,50)) #blur the face
            cv2.imshow("Video", frame) #show video
        if cv2.waitKey(1) & 0xFF == ord('q'): #if q is pressed, break
            break

    else:
        break

cap.release() #release video
cv2.destroyAllWindows() #close all windows and give memory back to computer


import cv2
import sys
import time
import multiprocessing as mp
from gelsight import gsdevice

class Device:
    def __init__(self, cam_id=None) -> None:
        cam_id = gsdevice.get_camera_id("GelSight Mini") if cam_id is None else cam_id
        self.dev = gsdevice.Camera(gsdevice.Finger.MINI, cam_id)
        self.dev.connect()

    def get_frame(self, *frame_queues):
        # use self.dev.get_image((320, 240)) to get a 320x240 image if you want
        try:
            for _ in range(10):
                self.dev.get_image((320, 240))
            while True:
                frame = self.dev.get_image((320, 240))
                for frame_queue in frame_queues:
                    frame_queue.put(frame)
        finally:
            self.dev.stop_video()

    def run(self, *frame_queues):
        """Starts the device and the process that gets the frame from the device
        frame_queue is a multiprocessing.Queue()
        that is used to pass the frame to the other processes"""
        self.processes = []
        self.processes.append(mp.Process(target=self.get_frame, args=(*frame_queues,)))

        [p.start() for p in self.processes]

    def stop(self):
        [p.terminate() for p in self.processes]

if __name__ == "__main__":
    gsmini = Device()
    frame_queue = mp.Queue()
    frame_queue_2 = mp.Queue()
    gsmini.run(frame_queue, frame_queue_2)
    try:
        while True:
            tic = time.time()
            frame = frame_queue.get()
            cv2.imshow("gsmini", frame)
            getKey = cv2.waitKey(1)
            if getKey == 27 or getKey == ord("q"):  # ESC or q
                break
            sys.stdout.write("\rFPS: {:.2f}".format(1 / (time.time() - tic)))
    finally:
        gsmini.stop()
        gsmini.dev.stop_video()
        cv2.destroyAllWindows()


from datetime import datetime, timedelta

from httpx import AsyncClient
import pytest

from app.database.models import Base
from app.main import app
from app.routers import telegram
from app.routers.event import create_event
from tests.client_fixture import get_test_placeholder_user
from tests.conftest import get_test_db, test_engine

@pytest.fixture
async def telegram_client():
    Base.metadata.create_all(bind=test_engine)
    app.dependency_overrides[telegram.get_db] = get_test_db
    async with AsyncClient(app=app, base_url="http://test") as ac:
        yield ac
    app.dependency_overrides = {}
    Base.metadata.drop_all(bind=test_engine)

today_date = datetime.today().replace(hour=0, minute=0, second=0)

@pytest.fixture
def fake_user_events(session):
    Base.metadata.create_all(bind=test_engine)
    user = get_test_placeholder_user()
    session.add(user)
    session.commit()
    create_event(
        db=session,
        title='Cool today event',
        start=today_date,
        end=today_date + timedelta(days=2),
        all_day=False,
        content='test event',
        owner_id=user.id,
        location="Here",
        is_google_event=False,
    )
    create_event(
        db=session,
        title='Cool (somewhen in two days) event',
        start=today_date + timedelta(days=1),
        end=today_date + timedelta(days=3),
        all_day=False,
        content='this week test event',
        owner_id=user.id,
        location="Here",
        is_google_event=False,
    )
    yield user
    Base.metadata.drop_all(bind=test_engine)


from django.http import HttpResponse, JsonResponse
from booksapp.models import Books, Cached_books

def api_downloadbook_controller(helpers, sessions, request, book_id):

    # Ответ
    response = JsonResponse

    # Пользователь
    user_dict = sessions.check_if_authorized(request, True)

    # Базовые проверки
    base_checks = helpers.base_auth_checks(user_dict, response)
    if base_checks is not None:
        return base_checks

    # Если все нормально, получаем данные книги
    try:
        book_object = Books.objects.get(book_id=book_id)
        cached_book_id = book_object.cached_book_id_id
        book_name = book_object.book_name
    except Books.DoesNotExist:
        cached_book_id = 0
        book_name = 'Not_found'
    except Exception:
        return response({'success': False, 'message': 'Произошла непредвиденная ошибка'})

    # Получаем контент книги
    try:
        cached_book_object = Cached_books.objects.get(cached_book_id=cached_book_id)
        book_content = cached_book_object.book_content
    except Cached_books.DoesNotExist:
        book_content = 'Not_found'
    except Exception:
        return response({'success': False, 'message': 'Произошла непредвиденная ошибка'})
    print(helpers.translate(book_name))
    # Присваиваем http-ответу необходимые заголовки
    http_response = HttpResponse(book_content, content_type='application/octet-stream; charset=utf-8')
    http_response['Content-Disposition'] = 'attachment; filename="Book-for-reading.fb2"'

    # Возврат, если все нормально
    return http_response


import logging
import os
import configparser

from vsc.filesystem.gpfs import GpfsOperations
from vsc.config.base import VscStorage

QUOTA_CONF_FILE = '/etc/quota_check.conf'

def set_up_apps(gpfs, storage_settings, storage, filesystem_info, filesystem_name):
    """Set up the apps fileset."""
    logging.info("Setting up the apps fileset on storage %s", storage)
    fileset_name = storage_settings.path_templates[storage]['apps'][0]
    fileset_path = os.path.join(filesystem_info['defaultMountPoint'], fileset_name)
    if fileset_name not in [f['filesetName'] for f in gpfs.gpfslocalfilesets[filesystem_name].values()]:
        gpfs.make_fileset(fileset_path, fileset_name)
        gpfs.chmod(0o755, fileset_path)
        logging.info("Fileset %s created and linked at %s", fileset_name, fileset_path)

def set_up_filesystem(gpfs, storage_settings, storage, filesystem_info, filesystem_name, vo_support=False):
    """Set up the filesets and directories such that user, vo directories and friends can be created."""

    # Create the basic gent fileset
    logging.info("Setting up for storage %s", storage)
    fileset_name = storage_settings.path_templates[storage]['user'][0]
    fileset_path = os.path.join(filesystem_info['defaultMountPoint'], fileset_name)
    if fileset_name not in [f['filesetName'] for f in gpfs.gpfslocalfilesets[filesystem_name].values()]:
        gpfs.make_fileset(fileset_path, fileset_name)
        gpfs.chmod(0o755, fileset_path)
        logging.info("Fileset %s created and linked at %s", fileset_name, fileset_path)

    if vo_support:
        # Create the basic vo fileset
        fileset_name = storage_settings.path_templates[storage]['vo'][0]
        vo_fileset_path = os.path.join(filesystem_info['defaultMountPoint'], fileset_name)
        if fileset_name not in [f['filesetName'] for f in gpfs.gpfslocalfilesets[filesystem_name].values()]:
            gpfs.make_fileset(vo_fileset_path, fileset_name)
            gpfs.chmod(0o755, vo_fileset_path)
            logging.info("Fileset %s created and linked at %s", fileset_name, vo_fileset_path)

def main():

    storage_settings = VscStorage()

    local_storage_conf = configparser.SafeConfigParser()
    local_storage_conf.read(QUOTA_CONF_FILE)

    gpfs = GpfsOperations()
    gpfs.list_filesystems()
    gpfs.list_filesets()

    for storage_name in local_storage_conf.get('MAIN', 'storage').split(','):

        filesystem_name = storage_settings[storage_name].filesystem
        filesystem_info = gpfs.get_filesystem_info(filesystem_name)

        if storage_name in ('VSC_HOME'):
            set_up_filesystem(gpfs, storage_settings, storage_name, filesystem_info, filesystem_name)
            set_up_apps(gpfs, storage_settings, storage_name, filesystem_info, filesystem_name)
        else:
            set_up_filesystem(gpfs, storage_settings, storage_name, filesystem_info, filesystem_name, vo_support=True)

if __name__ == '__main__':
    main()


import cv2
from flask import Flask, render_template, Response, stream_with_context
import mediapipe as mp
import numpy as np
import time
import logging
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)

POSE_TASK = None
TASK_COMPLETED = False
TASK_COMPLETION_TIME = None

app = Flask(__name__)

def generate_head_pose_estimation():
    global TASK_COMPLETED, TASK_COMPLETION_TIME, POSE_TASK

    cap = cv2.VideoCapture(1)

    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)  # Set frame width
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)  # Set frame height
    mp_face_mesh = mp.solutions.face_mesh
    face_mesh = mp_face_mesh.FaceMesh(
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5,
        max_num_faces = 1
    )

    update_pose_task()

    while cap.isOpened():
        ret, frame = cap.read()

        mp_drawing = mp.solutions.drawing_utils
        drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)

        frame = cv2.cvtColor(cv2.flip(frame, 1), cv2.COLOR_BGR2RGB)
        frame.flags.writeable = False

        result = face_mesh.process(frame)

        frame.flags.writeable = True
        frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)

        frame_h, frame_w, frame_c = frame.shape
        mesh_idx = [33, 263, 1, 61, 291, 199]
        face_3d = []
        face_2d = []

        if result.multi_face_landmarks:
            for face_landmarks in result.multi_face_landmarks:
                for idx, lm in enumerate(face_landmarks.landmark):
                    if idx in mesh_idx:
                        if idx == 1:
                            nose_2d = (lm.x * frame_w, lm.y * frame_h)
                            nose_3d = (lm.x * frame_w, lm.y * frame_h, lm.z * 3000)
                        x, y = int(lm.x * frame_w), int(lm.y * frame_h)

                        face_2d.append([x, y])
                        face_3d.append([x, y, lm.z])

                face_2d = np.array(face_2d, dtype=np.float64)
                face_3d = np.array(face_3d, dtype=np.float64)

                focal_length = 1 * frame_w # Not zoomed in

                # called the camera intrinsic matrix or the camera matrix.
                # It encodes the properties of a camera that relate the 3D world to the 2D image captured by the camera.
                # This matrix is fundamental in computer vision and photogrammetry.
                cam_matrix = np.array([
                    [focal_length, 0, frame_w / 2],
                    [0, focal_length, frame_h / 2],
                    [0, 0, 1]
                ], dtype=np.float64)

                dist_matrix = np.zeros((4, 1), dtype=np.float64) # Assuming no lens distortion
                success, rot_vec, trans_vec = cv2.solvePnP(face_3d, face_2d, cam_matrix, dist_matrix)

                if not success:
                    continue

                rmat, jac = cv2.Rodrigues(rot_vec)

                angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)

                x = angles[0] * 360
                y = angles[1] * 360
                z = angles[2] * 360

                if y < -10:
                    detected_pose = "Looking: Left"
                elif y > 10:
                    detected_pose = "Looking: Right"
                elif x < -10:
                    detected_pose = "Looking: Down"
                elif x > 10:
                    detected_pose = "Looking: Up"
                else:
                    detected_pose = "Looking: Forward"

                if detected_pose == f"Looking: {POSE_TASK}":
                    cv2.putText(frame, "CORRECT", (20,100), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 2)
                    if not TASK_COMPLETED:
                        TASK_COMPLETED = True
                        TASK_COMPLETION_TIME = time.time()

                    if time.time() - TASK_COMPLETION_TIME > 3:
                        update_pose_task()
                else:
                    cv2.putText(frame, "WRONG", (20,100), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 2)

                cv2.putText(frame, f"Task: Look {POSE_TASK}", (20,150), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 2)
                cv2.putText(frame, detected_pose, (20,50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 2)

                nose_3d_projection, jacobian = cv2.projectPoints(nose_3d, rot_vec, trans_vec, cam_matrix, dist_matrix)

                # p1 = (int(nose_2d[0]), int(nose_2d[1]))
                # p2 = (int(nose_2d[0] + y * 10), int(nose_2d[1] - x * 10))

                # cv2.line(frame, p1, p2, (0, 255, 0), 2)

                cv2.putText(frame, detected_pose, (20,50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 2)

            mp_drawing.draw_landmarks(
                image=frame,
                landmark_list=face_landmarks,
                connections=mp_face_mesh.FACEMESH_CONTOURS,
                landmark_drawing_spec=drawing_spec,
                connection_drawing_spec=drawing_spec
            )

        ret, jpeg = cv2.imencode('.jpg', frame)
        if not ret:
            continue

        frame = jpeg.tobytes()
        yield (b'--frame\r\n'
                b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n\r\n')

    cap.release()

def update_pose_task():
    global POSE_TASK, TASK_COMPLETED
    poses = ["Right", "Left", "Up", "Down", "Forward"]
    POSE_TASK = np.random.choice(poses)
    TASK_COMPLETED = False

@app.route('/video_feed')
def video_feed():
    return Response(stream_with_context(generate_head_pose_estimation()),
                    mimetype='multipart/x-mixed-replace; boundary=frame')

@app.route('/give_task')
def endow_ALD_task():
    update_pose_task()
    return f"Task: Turn your head {POSE_TASK}"

@app.route('/')
def index():
    return render_template('ALD.html')

if __name__ == '__main__':
    app.debug = True
    app.run(port=8000)


from tornado import web
from tornado import gen, escape
import rethinkdb as r
from myslice.web.controllers import BaseController
from myslice.db.activity import Event
from myslice.db import dispatch

class Index(BaseController):

    @gen.coroutine
    def get(self, id):
        msg = ""
        try:
            ev = yield r.table('activity').get(id).run(self.application.dbconnection)
            if ev is None:
                raise ValueError("event id is not valid")
            event = Event(ev)
            event.setPending()
            event.logInfo("Event is pending, a manager will validate your request")
            msg = "Your email is confirmed. "
            msg += "A request has been sent to a manager. "
            msg += "You will be informed as soon as your account will be validated."
            # dispatch the updated event
            dispatch(self.application.dbconnection, event)
        except Exception as e:
            import traceback
            traceback.print_exc()
            msg = "This link is not valid"
        self.render(self.application.templates + "/confirm.html", message=msg)


from kafka import KafkaProducer
import thread
import inspect
import time

def lineno():
    """Returns the current line number in our program."""
    return inspect.currentframe().f_back.f_lineno

def pro1(producer):
    msg = 'Hello world at '
    while  1:
        msg2 = msg +  str(time.time())
        producer.send('TutorialTopic', (msg2))
        #print ("sleep for 1 second", lineno() )
        #time.sleep(1)

producer1 = KafkaProducer(bootstrap_servers=['localhost:9092'])

thread.start_new_thread(pro1, (producer1,))
while 1 :
    data = raw_input("Press q to exit\n")
    if data == 'q':
        break;


#!/usr/bin/env python3

import datetime
import time
import json
import os
import argparse

from generator import Generator

JSON_FILE = "src/static/activities.js"
SQL_FILE = "scripts/data.db"

def run(client_id, client_secret, refresh_token):
    generator = Generator(SQL_FILE, client_id, client_secret, refresh_token)
    # if you want to update data change False to True
    generator.sync(False)

    athlete, activities_list = generator.load()
    with open(JSON_FILE, "w") as f:

        f.write("const strava_athlete = ")
        json.dump(athlete, f, indent=2)
        f.write(";\n")

        f.write("const activities = ")
        json.dump(activities_list, f, indent=2)
        f.write(";\n")
        f.write("\n")
        f.write("export {activities, strava_athlete};\n")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("client_id", help="strava client id")
    parser.add_argument("client_secret", help="strava client secret")
    parser.add_argument("refresh_token", help="strava refresh token")
    options = parser.parse_args()
    # maybe upload not done just tricky
    time.sleep(5)  # 5 seconds seems enough
    run(options.client_id, options.client_secret, options.refresh_token)


import time
import logging

logging.basicConfig(filename='/demo/hello.log', level=logging.INFO)
logger = logging.getLogger(__name__)

count = 0
while True:
    logger.info(f"Script says: {count} hello, systemd!")
    time.sleep(1)
    count += 1


import cv2
import math
import os
import sys
from PIL import Image
from PIL import ImageChops
import numpy

from target import *

cascPath = "haarcascade_frontalface_default.xml"

def shift_resize_images(target_frames, name, output_scale):

	#find the largest face height/width and scale all other faces

	largest_target_width = 0
	largest_target_height = 0
	for target in target_frames:
		if(target.w > largest_target_width):
			largest_target_width = target.w
		if(target.h > largest_target_height):
			largest_target_height = target.h

	# scale up images to match largest face (pixel-pixel wise)
	# (this is so all targets are the same size, and we dont lose resolution from downsizing larger images)
	for target in target_frames:
		#set new image height and width so we can match face's pixel height/width
		scaleW = largest_target_width / target.w
		scaleH = largest_target_height / target.h
		newImageW = int(scaleW*target.imageW)
		newImageH = int(scaleH*target.imageH)
		target.scale_image(newImageW, newImageH)
		#resize image
		target.PILimage = target.PILimage.resize((target.imageW, target.imageH))

		"""
		# for debugging: use cv2 with face rectangles so we can see
		img = face.PILimage
		open_cv_image = numpy.array(img)
		open_cv_image = open_cv_image[:, :, ::-1].copy()
		#rectangle marking face
		cv2.rectangle(open_cv_image, (face.x, face.y), (face.x+face.w, face.y+face.h), (0, 255, 0), 2)
		#line marking error
		cv2.line(open_cv_image, (int(face.imageW/2), int(face.imageH/2)),
			(int((face.x+face.x+face.w)/2),int((face.y+face.y+face.h)/2)), (0, 255, 0), 4)
		cv2.imshow("Face found", open_cv_image)
		cv2.waitKey(0)
		"""

	#scale outwards by specified amount so we can see more of the subject
	final_width = target_frames[0].w*output_scale[0]
	final_height = target_frames[0].h*output_scale[1]

	#position faces to the center of image and then
	#paste images onto black background
	for target in target_frames:
		background = Image.new('RGB', (final_width, final_height), (0, 0, 0))

		#offset the image to have the face centered in new, larger image
		offset = (int((final_width-target.imageW)/2+target.xErr),
			int((final_height-target.imageH)/2+target.yErr))

		background.paste(target.PILimage, offset)
		target.PILimage = background

		"""
		#convert to cv image so we can see
		open_cv_image = numpy.array(background)
		open_cv_image = open_cv_image[:, :, ::-1].copy()
		cv2.imshow("Face found", open_cv_image)
		cv2.waitKey(0)
		"""

	video=cv2.VideoWriter(f'video_{name}.avi',0,30,(final_width,final_height))

	for target in target_frames:
		open_cv_image = numpy.array(target.PILimage)
		open_cv_image = open_cv_image[:, :, ::-1].copy()
		# cv2.imshow("Face found", open_cv_image)
		# cv2.waitKey(0)
		video.write(open_cv_image)

	cv2.destroyAllWindows()
	video.release()

def parse_video(path, entity_movement_threshold):
	# files = os.listdir(imagesPath)
	# files = [file for file in files if "jpg" in file]
	# files.sort(key=lambda x:int(x[5:][:-4])) #"frame###.jpg" sorts by number
	# print(files)

	frames = []

	vidcap = cv2.VideoCapture(path)
	success,image = vidcap.read()
	count = 1
	while success:
		frames.append(image)
		success,image = vidcap.read()
		count += 1

	# Create the haar cascade
	cascade = cv2.CascadeClassifier(cascPath)
	detected_entities = []

	frame = 0

	for image in frames:

		# Read the image
		# image = cv2.imread(file)
		gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

		# Detect targets in the image
		entities = cascade.detectMultiScale(
		    gray,
		    scaleFactor=1.1,
		    minNeighbors=5,
		    minSize=(30, 30),
		)

		print(f"Found {len(entities)} entitiy(s) in video {path} frame {frame}!")

		# Draw a rectangle around the targets
		height, width, channel = image.shape

		#try to match target to entities
		for (x, y, w, h) in entities:
			if(w !=0 and h!=0):
				newTarget = Target(x,y,w,h,width,height,image)
				found = False
				for entry in detected_entities:
					if(abs(entry.frames[-1].x - x) < entity_movement_threshold
						and abs(entry.frames[-1].y - y) < entity_movement_threshold):
						entry.frames.append(newTarget)
						found=True
				if not found:
					newEntity = Entity(newTarget, len(detected_entities))
					detected_entities.append(newEntity)
				# Uncomment to see each target per frame
				# cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 2)
				# cv2.imshow("Target found", image)
				# cv2.waitKey(0)

		frame+=1

	return detected_entities, frame

if __name__ == "__main__":

	if(len(sys.argv)!=2):
		print("Wrong arguments")
		print("Usage: face_track.py <input mp4>")
		exit()

	videoPath = sys.argv[1]

	#amount of pixel movement between frames for an entity to still be considered
	#that entity
	entity_movement_threshold = 40
	#percent of frames that an entity must be in for an entity to count as a real entity
	entity_frame_threshold = 0.50
	#amount of outside of image from face (2,3) referes to 2x larger than detection width, 3x taller than detection height
	output_scale = (2,2)

	detected_entities, frames = parse_video(videoPath, entity_movement_threshold)

	for entity in detected_entities:
		if(len(entity.frames)>=(frames*entity_frame_threshold)):
			print(f"Entity '{entity.name}' found in {len(entity.frames)} frames. Saving result")
			shift_resize_images(entity.frames, entity.name, output_scale)


import csv
import math
import logging
from pprint import pformat
from datetime import datetime
from cStringIO import StringIO
from mako.template import Template

import ssbench
import ssbench.util
from ssbench.ordered_dict import OrderedDict

REPORT_TIME_FORMAT = '%F %T UTC'

if hasattr(csv.DictWriter, "writeheader"):
    DictWriter = csv.DictWriter
else:
    class DictWriter(csv.DictWriter):

        def writeheader(self):
            header = dict(zip(self.fieldnames, self.fieldnames))
            self.writerow(header)

class Reporter(object):
    def __init__(self, run_results):
        self.run_results = run_results

    def read_results(self, nth_pctile=95, format_numbers=True):
        self.scenario, self.unpacker = self.run_results.read_results()
        self.stats = self.calculate_scenario_stats(nth_pctile, format_numbers)

    def write_rps_histogram(self, target_file):
        target_file.write('"Seconds Since Start","Requests Completed"\n')
        for i, req_count in enumerate(self.stats['time_series']['data'], 1):
            target_file.write('%d,%d\n' % (i, req_count))

    def scenario_template(self):
        return """
${scenario.name}  (generated with ssbench version ${scenario.version})
Worker count: ${'%3d' % agg_stats['worker_count']}   Concurrency: ${'%3d' % scenario.user_count}  Ran ${start_time} to ${stop_time} (${'%.0f' % round(duration)}s)
Object expiration (X-Delete-After): ${scenario.delete_after} (sec)

%% Ops    C   R   U   D       Size Range       Size Name
% for size_datum in size_data:
${size_datum['pct_total_ops']}   % ${size_datum['crud_pcts']}      ${size_datum['size_range']}  ${size_datum['size_name']}
% endfor
---------------------------------------------------------------------
        ${'%3.0f' % weighted_c} ${'%3.0f' % weighted_r} ${'%3.0f' % weighted_u} ${'%3.0f' % weighted_d}      CRUD weighted average

% for label, stats, sstats in stat_list:
% if stats['req_count']:
${label}
       Count: ${'%5d' % stats['req_count']} (${'%5d' % stats['errors']} error; ${'%5d' % stats['retries']} retries: ${'%5.2f' % stats['retry_rate']}%)  Average requests per second: ${'%5.1f' % stats['avg_req_per_sec']}
                            min       max      avg      std_dev  ${'%02d' % nth_pctile}%-ile  ${'%15s' % ''}  Worst latency TX ID
       First-byte latency: ${stats['first_byte_latency']['min']} - ${stats['first_byte_latency']['max']}  ${stats['first_byte_latency']['avg']}  (${stats['first_byte_latency']['std_dev']})  ${stats['first_byte_latency']['pctile']}  (all obj sizes)  ${stats['worst_first_byte_latency'][1] if 'worst_first_byte_latency' in stats else ''}
       Last-byte  latency: ${stats['last_byte_latency']['min']} - ${stats['last_byte_latency']['max']}  ${stats['last_byte_latency']['avg']}  (${stats['last_byte_latency']['std_dev']})  ${stats['last_byte_latency']['pctile']}  (all obj sizes)  ${stats['worst_last_byte_latency'][1] if 'worst_last_byte_latency' in stats else ''}
% for size_str, per_size_stats in sstats.iteritems():
% if per_size_stats:
       First-byte latency: ${per_size_stats['first_byte_latency']['min']} - ${per_size_stats['first_byte_latency']['max']}  ${per_size_stats['first_byte_latency']['avg']}  (${per_size_stats['first_byte_latency']['std_dev']})  ${per_size_stats['first_byte_latency']['pctile']}  ${'(%8s objs)' % size_str}  ${per_size_stats['worst_first_byte_latency'][1] if 'worst_first_byte_latency' in per_size_stats else ''}
       Last-byte  latency: ${per_size_stats['last_byte_latency']['min']} - ${per_size_stats['last_byte_latency']['max']}  ${per_size_stats['last_byte_latency']['avg']}  (${per_size_stats['last_byte_latency']['std_dev']})  ${per_size_stats['last_byte_latency']['pctile']}  ${'(%8s objs)' % size_str}  ${per_size_stats['worst_last_byte_latency'][1] if 'worst_last_byte_latency' in per_size_stats else ''}
% endif
% endfor

% endif
% endfor
Distribution of requests per worker-ID: ${jobs_per_worker_stats['min']} - ${jobs_per_worker_stats['max']} (avg: ${jobs_per_worker_stats['avg']}; stddev: ${jobs_per_worker_stats['std_dev']})
"""

    def generate_default_report(self, output_csv=False):
        """Format a default summary report based on calculated statistics for
        an executed scenario.

        :returns: A report (string) suitable for printing, emailing, etc.
        """

        stats = self.stats
        template = Template(self.scenario_template())
        tmpl_vars = {
            'size_data': [],
            'stat_list': [
                ('TOTAL', stats['agg_stats'], stats['size_stats']),
                ('CREATE', stats['op_stats'][ssbench.CREATE_OBJECT],
                 stats['op_stats'][ssbench.CREATE_OBJECT]['size_stats']),
                ('READ', stats['op_stats'][ssbench.READ_OBJECT],
                 stats['op_stats'][ssbench.READ_OBJECT]['size_stats']),
                ('UPDATE', stats['op_stats'][ssbench.UPDATE_OBJECT],
                 stats['op_stats'][ssbench.UPDATE_OBJECT]['size_stats']),
                ('DELETE', stats['op_stats'][ssbench.DELETE_OBJECT],
                 stats['op_stats'][ssbench.DELETE_OBJECT]['size_stats']),
            ],
            'agg_stats': stats['agg_stats'],
            'nth_pctile': stats['nth_pctile'],
            'start_time': datetime.utcfromtimestamp(
                stats['time_series']['start_time']
            ).strftime(REPORT_TIME_FORMAT),
            'stop_time': datetime.utcfromtimestamp(
                stats['time_series']['stop']).strftime(REPORT_TIME_FORMAT),
            'duration': stats['time_series']['stop']
            - stats['time_series']['start_time'],
            'jobs_per_worker_stats': stats['jobs_per_worker_stats'],
            'weighted_c': 0.0,
            'weighted_r': 0.0,
            'weighted_u': 0.0,
            'weighted_d': 0.0,
        }
        for size_data in self.scenario.sizes_by_name.values():
            if size_data['size_min'] == size_data['size_max']:
                size_range = '%-15s' % (
                    self._format_bytes(size_data['size_min']),)
            else:
                size_range = '%s - %s' % (
                    self._format_bytes(size_data['size_min']),
                    self._format_bytes(size_data['size_max']))
            initial_files = self.scenario._scenario_data['initial_files']
            initial_total = sum(initial_files.values())
            pct_total = (initial_files.get(size_data['name'], 0)
                         / float(initial_total) * 100.0)
            tmpl_vars['size_data'].append({
                'crud_pcts': '  '.join(map(lambda p: '%2.0f' % p,
                                           size_data['crud_pcts'])),
                'size_range': size_range,
                'size_name': size_data['name'],
                'pct_total_ops': '%3.0f%%' % pct_total,
            })
            tmpl_vars['weighted_c'] += \
                pct_total * size_data['crud_pcts'][0] / 100.0
            tmpl_vars['weighted_r'] += \
                pct_total * size_data['crud_pcts'][1] / 100.0
            tmpl_vars['weighted_u'] += \
                pct_total * size_data['crud_pcts'][2] / 100.0
            tmpl_vars['weighted_d'] += \
                pct_total * size_data['crud_pcts'][3] / 100.0
        if output_csv:
            csv_fields = [
                'scenario_name', 'ssbench_version', 'worker_count',
                'concurrency', 'start_time', 'stop_time', 'duration',
                'delete_after']
            csv_data = {
                'scenario_name': self.scenario.name,
                'ssbench_version': self.scenario.version,
                'worker_count': tmpl_vars['agg_stats']['worker_count'],
                'concurrency': self.scenario.user_count,
                'start_time': tmpl_vars['start_time'],
                'stop_time': tmpl_vars['stop_time'],
                'duration': tmpl_vars['duration'],
                'delete_after': str(self.scenario.delete_after),
            }
            for label, stats, sstats in tmpl_vars['stat_list']:
                label_lc = label.lower()
                if stats.get('req_count', 0):
                    self._add_csv_kv(csv_fields, csv_data,
                                     '%s_count' % label_lc, stats['req_count'])
                    self._add_csv_kv(csv_fields, csv_data,
                                     '%s_errors' % label_lc,
                                     stats['errors'])
                    self._add_csv_kv(csv_fields, csv_data,
                                     '%s_retries' % label_lc,
                                     stats['retries'])
                    self._add_csv_kv(csv_fields, csv_data,
                                     '%s_retry_rate' % label_lc,
                                     '%f' % stats['retry_rate'])
                    self._add_csv_kv(csv_fields, csv_data,
                                     '%s_avg_req_per_s' % label_lc,
                                     stats['avg_req_per_sec'])
                    self._add_stats_for(csv_fields, csv_data, label, 'all',
                                        stats, tmpl_vars['nth_pctile'])
                    for size_str, per_size_stats in sstats.iteritems():
                        if per_size_stats:
                            self._add_stats_for(csv_fields, csv_data, label,
                                                size_str, per_size_stats,
                                                tmpl_vars['nth_pctile'])
            csv_file = StringIO()
            csv_writer = DictWriter(csv_file, csv_fields,
                                    lineterminator='\n',
                                    quoting=csv.QUOTE_NONNUMERIC)
            csv_writer.writeheader()
            csv_writer.writerow(csv_data)
            return csv_file.getvalue()
        else:
            return template.render(scenario=self.scenario, **tmpl_vars)

    def _add_csv_kv(self, csv_fields, csv_data, key, value):
        csv_fields.append(key)
        csv_data[key] = value

    def _add_stats_for(self, csv_fields, csv_data, label, size_str, stats,
                       nth_pctile):
        for latency_type in ('first', 'last'):
            latency_stats = stats['%s_byte_latency' % latency_type]
            key_base = '%s_%s_%s_' % (label.lower(), latency_type, size_str)
            self._add_csv_kv(csv_fields, csv_data, key_base + 'min',
                             latency_stats['min'])
            self._add_csv_kv(csv_fields, csv_data, key_base + 'max',
                             latency_stats['max'])
            self._add_csv_kv(csv_fields, csv_data, key_base + 'avg',
                             latency_stats['avg'])
            self._add_csv_kv(csv_fields, csv_data, key_base + 'std_dev',
                             latency_stats['std_dev'])
            self._add_csv_kv(csv_fields, csv_data,
                             key_base + '%d_pctile' % nth_pctile,
                             latency_stats['pctile'])
            worst_key = 'worst_%s_byte_latency' % latency_type
            self._add_csv_kv(
                csv_fields, csv_data, key_base + 'worst_txid',
                stats[worst_key][1] if worst_key in stats else '')

    def _format_bytes(self, byte_count):
        units = [' B', 'kB', 'MB', 'GB']
        i = 0
        while round(byte_count / 1000.0, 3) >= 1.0:
            byte_count = byte_count / 1000.0
            i += 1
        return '%3.0f %s' % (round(byte_count), units[i])

    def calculate_scenario_stats(self, nth_pctile=95, format_numbers=True):
        """Compute various statistics from worker job result dicts.

        :param nth_pctile: Use this percentile when calculating the stats
        :param format_numbers: Should various floating-point numbers be
        formatted as strings or left full-precision floats
        :returns: A stats python dict which looks something like:
            SERIES_STATS = {
                'min': 1.1,
                'max': 1.1,
                'avg': 1.1,
                'std_dev': 1.1,
                'median': 1.1,
            }
            {
                'agg_stats': {
                    'worker_count': 1,
                    'start': 1.1,
                    'stop': 1.1,
                    'req_count': 1,
                    'retries': 0,
                    'errors' : 0,
                    'avg_req_per_sec': 1.1, # req_count / (stop - start)?
                    'retry_rate': 0.0,
                    'first_byte_latency': SERIES_STATS,
                    'last_byte_latency': SERIES_STATS,
                },
                'worker_stats': {
                    1: {  # keys are worker_ids
                        'start': 1.1,
                        'stop': 1.1,
                        'req_count': 1,
                        'retries': 0,
                        'retry_rate': 0.0,
                        'errors': 0,
                        'avg_req_per_sec': 1.1, # req_count / (stop - start)?
                        'first_byte_latency': SERIES_STATS,
                        'last_byte_latency': SERIES_STATS,
                    },
                    # ...
                },
                'op_stats': {
                    CREATE_OBJECT: { # keys are CRUD constants: CREATE_OBJECT, READ_OBJECT, etc.
                        'req_count': 1, # num requests of this CRUD type
                        'avg_req_per_sec': 1.1, # total_requests / sum(last_byte_latencies)
                        'first_byte_latency': SERIES_STATS,
                        'last_byte_latency': SERIES_STATS,
                        'size_stats': {
                            'small': { # keys are size_str values
                                'req_count': 1, # num requests of this type and size
                                'retries': 0, # num of retries
                                'avg_req_per_sec': 1.1, # total_requests / sum(last_byte_latencies)
                                'errors': 0,
                                'retry_rate': 0.0,
                                'first_byte_latency': SERIES_STATS,
                                'last_byte_latency': SERIES_STATS,
                            },
                            # ...
                        },
                    },
                    # ...
                },
                'size_stats': {
                    'small': { # keys are size_str values
                        'req_count': 1, # num requests of this size (for all CRUD types)
                        'retries': 0, # num of retries
                        'acutual_request_count': 1, # num requests includes retries
                        'avg_req_per_sec': 1.1, # total_requests / sum(last_byte_latencies)
                        'errors': 0,
                        'retry_rate': 0.0,
                        'first_byte_latency': SERIES_STATS,
                        'last_byte_latency': SERIES_STATS,
                    },
                    # ...
                },
                'time_series': {
                    'start': 1, # epoch time of first data point
                    'data': [
                        1, # number of requests finishing during this second
                        # ...
                    ],
                },
            }
        """
        # Each result looks like:
        # {
        #   'worker_id': 1,
        #   'type': 'get_object',
        #   'size': 4900000,
        #   'size_str': 'large',
        #   'first_byte_latency': 0.9137639999389648,
        #   'last_byte_latency': 0.913769006729126,
        #   'retries': 1
        #   'completed_at': 1324372892.360802,
        # }
        # OR
        # {
        #   'worker_id': 1,
        #   'type': 'get_object',
        #   'size_str': 'large'
        #   'completed_at': 1324372892.360802,
        #   'retries': 1
        #   'exception': '...',
        # }
        logging.info('Calculating statistics...')
        agg_stats = dict(start=2 ** 32, stop=0, req_count=0)
        op_stats = {}
        for crud_type in [ssbench.CREATE_OBJECT, ssbench.READ_OBJECT,
                          ssbench.UPDATE_OBJECT, ssbench.DELETE_OBJECT]:
            op_stats[crud_type] = dict(
                req_count=0, avg_req_per_sec=0,
                size_stats=OrderedDict.fromkeys(
                    self.scenario.sizes_by_name.keys()))

        req_completion_seconds = {}
        start_time = 0
        completion_time_max = 0
        completion_time_min = 2 ** 32
        stats = dict(
            nth_pctile=nth_pctile,
            agg_stats=agg_stats,
            worker_stats={},
            op_stats=op_stats,
            size_stats=OrderedDict.fromkeys(
                self.scenario.sizes_by_name.keys()))
        for results in self.unpacker:
            skipped = 0
            for result in results:
                try:
                    res_completed_at = result['completed_at']
                    res_completion_time = int(res_completed_at)
                    res_worker_id = result['worker_id']
                    res_type = result['type']
                    res_size_str = result['size_str']
                except KeyError as err:
                    logging.info('Skipped result with missing keys (%r): %r',
                                 err, result)
                    skipped += 1
                    continue

                try:
                    res_exception = result['exception']
                except KeyError:
                    try:
                        res_last_byte_latency = result['last_byte_latency']
                    except KeyError:
                        logging.info('Skipped result with missing'
                                     ' last_byte_latency key: %r',
                                     result)
                        skipped += 1
                        continue
                    if res_completion_time < completion_time_min:
                        completion_time_min = res_completion_time
                        start_time = (
                            res_completion_time - res_last_byte_latency)
                    if res_completion_time > completion_time_max:
                        completion_time_max = res_completion_time
                    req_completion_seconds[res_completion_time] = \
                        1 + req_completion_seconds.get(res_completion_time, 0)
                    result['start'] = res_completed_at - res_last_byte_latency
                else:
                    # report log exceptions
                    logging.warn('calculate_scenario_stats: exception from '
                                 'worker %d: %s',
                                 res_worker_id, res_exception)
                    try:
                        res_traceback = result['traceback']
                    except KeyError:
                        logging.warn('traceback missing')
                    else:
                        logging.info(res_traceback)

                # Stats per-worker
                if res_worker_id not in stats['worker_stats']:
                    stats['worker_stats'][res_worker_id] = {}
                self._add_result_to(stats['worker_stats'][res_worker_id],
                                    result)

                # Stats per-file-size
                try:
                    val = stats['size_stats'][res_size_str]
                except KeyError:
                    stats['size_stats'][res_size_str] = {}
                else:
                    if not val:
                        stats['size_stats'][res_size_str] = {}
                self._add_result_to(stats['size_stats'][res_size_str],
                                    result)

                self._add_result_to(agg_stats, result)

                type_stats = op_stats[res_type]
                self._add_result_to(type_stats, result)

                # Stats per-operation-per-file-size
                try:
                    val = type_stats['size_stats'][res_size_str]
                except KeyError:
                    type_stats['size_stats'][res_size_str] = {}
                else:
                    if not val:
                        type_stats['size_stats'][res_size_str] = {}
                self._add_result_to(
                    type_stats['size_stats'][res_size_str], result)
            if skipped > 0:
                logging.warn("Total number of results skipped: %d", skipped)

        agg_stats['worker_count'] = len(stats['worker_stats'].keys())
        self._compute_req_per_sec(agg_stats)
        self._compute_retry_rate(agg_stats)
        self._compute_latency_stats(agg_stats, nth_pctile, format_numbers)

        jobs_per_worker = []
        for worker_stats in stats['worker_stats'].values():
            jobs_per_worker.append(worker_stats['req_count'])
            self._compute_req_per_sec(worker_stats)
            self._compute_retry_rate(worker_stats)
            self._compute_latency_stats(worker_stats, nth_pctile,
                                        format_numbers)
        stats['jobs_per_worker_stats'] = self._series_stats(jobs_per_worker,
                                                            nth_pctile,
                                                            format_numbers)
        logging.debug('Jobs per worker stats:\n' +
                      pformat(stats['jobs_per_worker_stats']))

        for op_stats_dict in op_stats.itervalues():
            if op_stats_dict['req_count']:
                self._compute_req_per_sec(op_stats_dict)
                self._compute_retry_rate(op_stats_dict)
                self._compute_latency_stats(op_stats_dict, nth_pctile,
                                            format_numbers)
                for size_str, size_stats in \
                        op_stats_dict['size_stats'].iteritems():
                    if size_stats:
                        self._compute_req_per_sec(size_stats)
                        self._compute_retry_rate(size_stats)
                        self._compute_latency_stats(size_stats, nth_pctile,
                                                    format_numbers)
                    else:
                        op_stats_dict['size_stats'].pop(size_str)
        for size_str, size_stats in stats['size_stats'].iteritems():
            if size_stats:
                self._compute_req_per_sec(size_stats)
                self._compute_retry_rate(size_stats)
                self._compute_latency_stats(size_stats, nth_pctile,
                                            format_numbers)
            else:
                stats['size_stats'].pop(size_str)
        time_series_data = [req_completion_seconds.get(t, 0)
                            for t in range(completion_time_min,
                                           completion_time_max + 1)]
        stats['time_series'] = dict(start=completion_time_min,
                                    start_time=start_time,
                                    stop=completion_time_max,
                                    data=time_series_data)

        return stats

    def _compute_latency_stats(self, stat_dict, nth_pctile, format_numbers):
        try:
            for latency_type in ('first_byte_latency', 'last_byte_latency'):
                stat_dict[latency_type] = self._series_stats(
                    stat_dict.get(latency_type, []), nth_pctile,
                    format_numbers)
        except KeyError:
            logging.exception('stat_dict: %r', stat_dict)
            raise

    def _compute_req_per_sec(self, stat_dict):
        try:
            sd_start = stat_dict['start']
        except KeyError:
            stat_dict['avg_req_per_sec'] = 0.0
        else:
            delta_t = stat_dict['stop'] - sd_start
            if delta_t != 0:
                stat_dict['avg_req_per_sec'] = round(
                    stat_dict['req_count'] / delta_t,
                    6)
            else:
                stat_dict['avg_req_per_sec'] = float("inf")

    def _compute_retry_rate(self, stat_dict):
        stat_dict['retry_rate'] = round((float(stat_dict['retries']) /
                                         stat_dict['req_count']) * 100, 6)

    def _add_result_to(self, stat_dict, result):
        if 'errors' not in stat_dict:
            stat_dict['errors'] = 0
        try:
            res_start = result['start']
        except KeyError:
            pass
        else:
            try:
                sd_start = stat_dict['start']
            except KeyError:
                stat_dict['start'] = res_start
            else:
                if res_start < sd_start:
                    stat_dict['start'] = res_start
        try:
            sd_stop = stat_dict['stop']
        except KeyError:
            stat_dict['stop'] = result['completed_at']
        else:
            if result['completed_at'] > sd_stop:
                stat_dict['stop'] = result['completed_at']
        stat_dict['retries'] = \
            stat_dict.get('retries', 0) + int(result['retries'])
        if 'exception' not in result:
            stat_dict['req_count'] = stat_dict.get('req_count', 0) + 1
            self._rec_latency(stat_dict, result)
        else:
            stat_dict['errors'] += 1

    def _series_stats(self, sequence, nth_pctile, format_numbers):
        sequence = filter(None, sequence)
        if not sequence:
            # No data available
            return dict(min=' N/A  ', max='  N/A  ', avg='  N/A  ',
                        pctile='  N/A  ', std_dev='  N/A  ', median='  N/A  ')
        sequence.sort()
        try:
            minval = min(sequence)
            maxval = max(sequence)
            mean = ssbench.util.mean(sequence)
        except ZeroDivisionError:
            # Handle the case of a single-element sequence (population standard
            # deviation divides by N-1)
            minval = sequence[0]
            maxval = sequence[0]
            mean = sequence[0]
        if format_numbers:
            return dict(
                min='%6.3f' % minval,
                max='%7.3f' % maxval,
                avg='%7.3f' % mean,
                pctile='%7.3f' % self.pctile(sequence, nth_pctile),
                std_dev='%7.3f' % ssbench.util.uncorrected_stdev(sequence),
                median='%7.3f' % ssbench.util.median(sequence))
        else:
            return dict(
                min=round(minval, 6),
                max=round(maxval, 6),
                avg=round(mean, 6),
                pctile=round(self.pctile(sequence, nth_pctile), 6),
                std_dev=round(ssbench.util.uncorrected_stdev(sequence), 6),
                median=round(ssbench.util.median(sequence), 6))

    def pctile(self, sequence, nth_pctile):
        seq_len = len(sequence)
        rank = seq_len * nth_pctile / 100.0
        if float(int(rank)) == rank:
            # integer rank means we interpolate between two values
            rank = int(rank)
            return (sequence[rank - 1] + sequence[rank]) / 2.0
        else:
            return sequence[int(math.ceil(rank)) - 1]

    def _rec_latency(self, stats_dict, result):
        for latency_type in ('first_byte_latency', 'last_byte_latency'):
            if latency_type in stats_dict:
                stats_dict[latency_type].append(result[latency_type])
            else:
                stats_dict[latency_type] = [result[latency_type]]
            if result[latency_type] is not None:
                worst_key = 'worst_%s' % latency_type
                if worst_key not in stats_dict \
                        or result[latency_type] > stats_dict[worst_key][0]:
                    stats_dict[worst_key] = (round(result[latency_type], 6),
                                             result['trans_id'])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
from matplotlib import pyplot as plt
import mpl_toolkits.mplot3d.axes3d as p3
from matplotlib.gridspec import GridSpec

#%%

class MrDynamics:

    '''
    This object - MrDynamics will generate list of co-ordinate and velocity of a
    charged particle.

    q and m are the charge and mass of the particle.

    Tmax is the simulation time.
    Δt is the time step which act as a accuracy of Borish scheme.

    r and V are the initial position and velocity, respectively.

    E and B are electro-static field and magnetic field, respectively.

    '''

    def __init__(self, q, m, Tmax, Δt, r, V, E, B):

        # list of position and velocity
        self.ListOfx,  self.ListOfy,  self.ListOfz  = [ r[0] ],  [ r[1] ],  [ r[2] ]
        self.ListOfVx, self.ListOfVy, self.ListOfVz = [ V[0] ],  [ V[1] ],  [ V[2] ]

        self.Boris(q, m, Tmax, Δt, r, V, E, B)

    def Boris(self, q, m, Tmax, Δt, r, V, E, B):

        x,   y,  z = r
        Vx, Vy, Vz = V
        Ex, Ey, Ez = E
        Bx, By, Bz = B

        # Step - 1 : Half step velocity
        Vx += ( (q/m)* (Ex + Bz* Vy - By* Vz) )* (Δt/2)
        Vy += ( (q/m)* (Ey + Bx* Vz - Bz* Vx) )* (Δt/2)
        Vz += ( (q/m)* (Ez + By* Vx - Bx* Vy) )* (Δt/2)

        # Step - 2 : Boris Algorithm
        t  = 0
        qd = (q* Δt)/ (2* m)

        while (t <= Tmax):

            hx  = qd* Bx
            hy  = qd* By
            hz  = qd* Bz

            Sx  = 2* hx/ (1 + ( hx**2 + hy**2 + hz**2 ) )
            Sy  = 2* hy/ (1 + ( hx**2 + hy**2 + hz**2 ) )
            Sz  = 2* hz/ (1 + ( hx**2 + hy**2 + hz**2 ) )

            Ux  = Vx + qd* Ex
            Uy  = Vy + qd* Ey
            Uz  = Vz + qd* Ez

            Ux_d = Ux + Sz* ( Uy + Uz*hx - hz*Ux ) - Sy* ( Uz + Ux*hy - hx*Uy )
            Uy_d = Uy + Sx* ( Uz + Ux*hy - hx*Uy ) - Sz* ( Ux + Uy*hz - hy*Uz )
            Uz_d = Uz + Sy* ( Ux + Uy*hz - hy*Uz ) - Sx* ( Uy + Uz*hx - hz*Ux )

            Vx = Ux_d + qd* Ex
            Vy = Uy_d + qd* Ey
            Vz = Uz_d + qd* Ez

            x += Vx* Δt
            y += Vy* Δt
            z += Vz* Δt

            self.ListOfx.append(x)
            self.ListOfy.append(y)
            self.ListOfz.append(z)

            t += Δt

        # Step - 3 : Find velocity

        Vel = lambda Pos, Δt : [(Pos[i+2] - Pos[i])/ (2* Δt) for i in range(0, len(Pos) - 2)]

        self.ListOfVx = self.ListOfVx + Vel( self.ListOfx, Δt )
        self.ListOfVy = self.ListOfVy + Vel( self.ListOfy, Δt )
        self.ListOfVz = self.ListOfVz + Vel( self.ListOfz, Δt )

        self.ListOfx = self.ListOfx[:-1]
        self.ListOfy = self.ListOfy[:-1]
        self.ListOfz = self.ListOfz[:-1]

    def PlotTrajectory(self):

        fig = plt.figure( figsize = (80, 60) )

        gs = GridSpec(2, 2)

        ax1 = fig.add_subplot( gs[0, 0], projection = '3d' )
        ax2 = fig.add_subplot( gs[0, 1] )
        ax3 = fig.add_subplot( gs[1, 0] )
        ax4 = fig.add_subplot( gs[1, 1] )

        ax1.plot(self.ListOfx, self.ListOfy, self.ListOfz, c = 'r', linewidth = 2)
        ax1.scatter(self.ListOfx[0], self.ListOfy[0], self.ListOfz[0])

        ax1.set_xlabel('X', fontsize = 12, fontweight = 'bold')
        ax1.set_ylabel('Y', fontsize = 12, fontweight = 'bold')
        ax1.set_zlabel('Z', fontsize = 12, fontweight = 'bold')

        ax2.plot(self.ListOfx, self.ListOfVx)
        ax2.scatter(self.ListOfx[0], self.ListOfVx[0], color = 'r')
        ax2.set_title('x | Vx')
        ax2.grid()

        ax3.plot(self.ListOfy, self.ListOfVy)
        ax3.scatter(self.ListOfy[0], self.ListOfVy[0], color = 'r')
        ax3.set_title('y | Vy')
        ax3.grid()

        ax4.plot(self.ListOfz, self.ListOfVz)
        ax4.scatter(self.ListOfz[0], self.ListOfVz[0], color = 'r')
        ax4.set_title('z | Vz')
        ax4.grid()

#%%

if __name__ == '__main__':

    q, m = 1, 1
    Δt, Tmax = 1E-2, 5

    r = [0, 0, 0]
    V = [1, 0, 1]

    E = [0, 0, 0]
    B = [0, 0, 10]

    Answer = MrDynamics(q, m, Tmax, Δt, r, V, E, B)

    Answer.PlotTrajectory()


# Training script for all the models

import numpy as np
import torch
import torch.nn.functional as F
import torch.optim as optim
from tqdm import trange
import argparse
import matplotlib.pyplot as plt
from settings_MIMO import *
from networks_generator_MIMO import generate_real_channels, estimate_channels
from neural_net_MIMO import Regular_Net, Robust_Net

N_EPOCHES = 500
N_MINIBATCHES = 50
MINIBATCH_SIZE = 1000
VALIDATION_SIZE = 2000
TRAINING_DEBUG = False

def plot_training_curves(objs, starting_point):
    print("Plotting training curves...")
    fig, axes = plt.subplots(1,2)
    fig.suptitle("MIMO over {}".format(SETTING_STRING))
    # Regular Net
    axes[0].set_xlabel("Training Epoches")
    axes[0].set_ylabel(f"Nominal Min Rate (Unscaled)")
    axes[0].set_title("Regular Neural Network")
    axes[0].plot(objs[starting_point:, 0], 'r', label="Train")
    axes[0].plot(objs[starting_point:, 2], 'b--', linewidth=0.8, label="Valid")
    axes[0].legend()
    # Robust Net
    axes[1].set_xlabel("Training Epoches")
    axes[1].set_ylabel(f"Robust Min Rate (Unscaled)")
    axes[1].set_title("Robust Neural Network")
    axes[1].plot(objs[starting_point:, 1], 'r', label="Train")
    axes[1].plot(objs[starting_point:, 3], 'b--', linewidth=0.8, label="Valid")
    axes[1].legend()
    plt.show()
    print("Finished plotting!")
    return

if(__name__=="__main__"):
    parser = argparse.ArgumentParser(description="main script argument parser")
    parser.add_argument("--plot", default=False, help="Whether plot the training curves (executed after training)")
    parser.add_argument("--start", type=int, default=0, help="The starting point of training updates to plot training curves")
    args = parser.parse_args()
    if args.plot:
        sinrs = np.load(f"Trained_Models_MIMO/training_curves_Min-Rate_{SETTING_STRING}.npy")
        plot_training_curves(sinrs, args.start)
        exit(0)

    regular_net = Regular_Net().to(DEVICE)
    robust_net = Robust_Net().to(DEVICE)
    regular_optimizer, robust_optimizer = optim.Adam(regular_net.parameters(), lr=1e-4), optim.Adam(robust_net.parameters(), lr=1e-4)
    regular_obj_max, robust_obj_max = -np.inf, -np.inf
    obj_eps = []
    for i in trange(1, N_EPOCHES+1):
        # [TRAIN]
        regular_obj_ep, robust_obj_ep = 0, 0
        regular_net.train()
        robust_net.train()
        for j in range(N_MINIBATCHES):
            # Generate minibatch training data online
            H_est = estimate_channels(generate_real_channels(MINIBATCH_SIZE))
            # [Regular net]
            regular_optimizer.zero_grad()
            robust_optimizer.zero_grad()
            regular_obj, _ = regular_net(torch.tensor(H_est, dtype=torch.cfloat).to(DEVICE))
            robust_obj, _ = robust_net(torch.tensor(H_est, dtype=torch.cfloat).to(DEVICE))
            (-regular_obj).backward(); regular_optimizer.step()
            (-robust_obj).backward(); robust_optimizer.step()
            regular_obj_ep += regular_obj.item() / N_MINIBATCHES
            robust_obj_ep += robust_obj.item() / N_MINIBATCHES
        # [VALIDATE]
        H_est = estimate_channels(generate_real_channels(VALIDATION_SIZE))
        with torch.no_grad():
            regular_obj, _ = regular_net(torch.tensor(H_est, dtype=torch.cfloat).to(DEVICE))
            regular_obj = regular_obj.item()
            robust_obj, _ = robust_net(torch.tensor(H_est, dtype=torch.cfloat).to(DEVICE))
            robust_obj = robust_obj.item()
        obj_eps.append([regular_obj_ep, robust_obj_ep, regular_obj, robust_obj])
        print("[Min Rate Unscaled with RCI alpha {}] [Regular] Tr:{:7.4e}; Va:{:7.4e} [Robust] Tr:{:7.4e}; Va:{:7.4e}".format(
            RCI_BF_ALPHA, regular_obj_ep, regular_obj, robust_obj_ep, robust_obj))
        if regular_obj > regular_obj_max:
            regular_net.save_model()
            regular_obj_max = regular_obj
        if robust_obj > robust_obj_max:
            robust_net.save_model()
            robust_obj_max = robust_obj
        np.save(f"Trained_Models_MIMO/training_curves_Min-Rate_{SETTING_STRING}.npy", np.array(obj_eps))

    print("Training Script finished!")


#!/usr/bin/env python
# coding: utf-8

# Hello everyone,
#
# In this module, it is required to implement the preprocessor for Arabic Font Identification System.
#
# At first, what are the main steps that we should go through in this module?
#
#
# # TODOs:
#
# 1. Understand the problem
# 2. Binarization
# 3. Extract Edges
# 4. Extract Skeleton
# 5. Extract Diacritics
# 6. Extract Text only
# 7. Testing

# In[6]:

##################################################### imports #####################################################
from skimage.morphology import skeletonize
import cv2
import matplotlib.pyplot as plt
from random import randrange
import numpy as np
from skimage.segmentation import flood_fill

# # 1. Understand the problem
#
# We are interested only in the morphology of text letters, all images are first converted into binary (i.e., black text on a white background). It should be known that most images contain either meaningful texture, which is a part of the decoration or a meaningless one that resulted from the noise while capturing. In either case, we illuminate the background so it does not affect the results.
#
# - Input: --
# - Output: --

# In[27]:

def analyze_histogram(binary, show_hist=False):

    """Gets a binary image and returns it reverted if text was black (Makes text white for all images)
        Except for Square Kuffi

    Args:
        binary : The binary image you want to transform for consistency (White text on black bg) (Read using cv2)

    Returns:
        binary || reverted_binary : The outcome of analysis (Binary image with white text on black bg)
    """

    height = (binary.shape)[0]
    white_pixels_in_row = []

    # Calculate number of white pixels in a row
    for row in range(height - 1):
        white_pixels_in_row.append((binary[row, :] == 1).sum())

    # Set the threshold of white pixels in a row to be 70% of the row with most white pixels
    # We use this threshold to see how many rows contain a high number of white pixels
    pixels_threshold = int(0.7 * max(white_pixels_in_row))

    # Get the number of rows above the pixels_threshold
    rows_above_thresh = 0
    for row_pixels in white_pixels_in_row:
        if row_pixels > pixels_threshold:
            rows_above_thresh += 1

    # Calculate the ratio between the number of rows dominated by white pixels to the number of rows in the entire image
    ratio = rows_above_thresh / height

    # if show_hist:
    #     x = np.arange(height)
    #     plt.figure(figsize =(10, 7))
    #     plt.plot(x, white_pixels_in_row)
    #     plt.show()

    # If more than 70% of the image is white rows then the background is white and we have to revert the image
    if ratio > 0.7:
        return 1 - binary
    return binary

# # 2. Binarization

# In[8]:

def binarize(image):
    """Gets an image and returns it binarized having values 0 OR 1 using OTSU thresholding technique

    Args:
        image : The image you want to binarize (Read using cv2)

    Returns:
        binarized: The binarized version of the image
    """
    # Use Otsu for normal binarization
    # The parameters are 1-`The input image`, 2-`The binarization threshold (will not be used if OTSU technique is used)`,
    #                        3-`Maximum value`, 4-`Binarization technique`
    th, binary_image = cv2.threshold(image, 128, 1, cv2.THRESH_OTSU)

    # Perform histogram analysis to make text white for all images (Except for square Kuffi)
    corrected_binary_image = analyze_histogram(binary_image)
    return corrected_binary_image

# In[10]:

def crop(img):
    morph_img = img

    rows = morph_img.sum(axis=1)
    cols = morph_img.sum(axis=0)
    rows = rows != 0
    cols = cols != 0

    c = np.where(cols == cols.max())[0]
    r = np.where(rows == rows.max())[0]

    x = c[0], c[-1]
    y = r[0], r[-1]

    try:
        cropped_img = img[y[0]:y[1], x[0]:x[1]]
    except:
        cropped_img = img

    return cropped_img

# In[11]:

def image_resize(image, width = None, height = None, inter = cv2.INTER_CUBIC):
    # initialize the dimensions of the image to be resized and
    # grab the image size
    dim = None
    (h, w) = image.shape[:2]

    # if both the width and height are None, then return the
    # original image
    if width is None and height is None:
        return image

    # check to see if the width is None
    if width is None:
        # calculate the ratio of the height and construct the
        # dimensions
        t = int(height / h)
        x = int(h / height)
        if x == 0:
            height = int(h*t)
        else:
            height = int(h/x)
        if h > height: #shrink
            inter = cv2.INTER_AREA
        r = height / float(h)
        dim = (int(w * r), height)

    # otherwise, the height is None
    else:
        # calculate the ratio of the width and construct the
        # dimensions
        t = int(width / w)
        x = int(w / width)
        if x == 0:
            width = int(w*t)
        else:
            width = int(w/x)

        if w > width: #shrink
            inter = cv2.INTER_AREA
        r = width / float(w)
        dim = (width, int(h * r))

    # resize the image
    resized = cv2.resize(image, dim, interpolation = inter)

    # return the resized image
    return resized

# # 3. Extract Edges

# In[12]:

def extract_edges(image):
    """Gets the original input image and returns an image containing its edges using canny edge detector

    Args:
        image: The image you want to extract its edges (Read using cv2)

    Returns:
        edge_image: The edge extracted image
    """
    # Blur the image for better edge detection
    if image.max() == 1:
        image = image * 255
        image = image.astype(np.uint8)

    img_blur = cv2.GaussianBlur(image,(5,5), sigmaX=0, sigmaY=0)

    # Use Canny edge detection to extract edges
    edge_image = cv2.Canny(image=img_blur, threshold1=100, threshold2=200)

    # Change image max value from 255 to 1
    edge_image[edge_image == 255] = 1

    return edge_image

# # 4. Extract Skeleton

# In[14]:

def extract_skeleton(image):
    """Gets an image and returns an image containing its skeleton

    Args:
        image: The image you want to skeletonize (Read using cv2)

    Returns:
        skeleton: The skeletonized image
    """
    # Use Skimage's skeletonize method with the array representation of the binary image as input
    skeleton = skeletonize(np.asarray(image)).astype(np.uint8)

    return skeleton

# # 5. Separate Diacritics from Text

# In[15]:

def separate_diacritics_and_text_utility(image):
    """Utility function used by separate_diacritics_and_text
    """

    # binary = binarize(image)
    binary = image

    # height = (binary.shape)[0]
    # white_pixels_in_row = np.zeros(height, dtype=np.uint32)
    # Calculate number of white pixels in a row
    # for row in range(height - 1):
        # white_pixels_in_row[row] = (binary[row, :] == 1).sum()

    white_pixels_in_row = binary.sum(axis=1)

    # Get the row with most white pixels (This row would contain only text and commas no diacritics
    #                                        as it is an imaginary line where text is placed on)
    baseline_row = np.argmax(white_pixels_in_row)
    diacritics_image = binary

    # How Flood Fill works for our case:
    #   It starts at a seed point (pixel) and changes all connected pixels to it to have a certain color
    #   So what we want to do is get the first pixel of each word along the line (it would be black) and
    #   use that point as seed for the flood_fill algorithm changing that word to have the same color as
    #   the background. We keep doing that for all words until only diacritics remain.

    # Loop through the base_line's pixels to apply flood fill on all words
    rowPixels = binary[baseline_row, :]
    for i in range(len(rowPixels) - 1):
        # Handle the case where the first pixel is part of a word so we apply the flood fill from the end of the word
        if rowPixels[i] == 1 and rowPixels[i+1] == 0:
            diacritics_image = flood_fill(diacritics_image, (baseline_row, i), 0)
        elif rowPixels[i] == 0 and rowPixels[i+1] == 1:
            diacritics_image = flood_fill(diacritics_image, (baseline_row, i+1), 0)

    # print(baseline_row, binary.shape[0])
    text_image = binary - diacritics_image
    cv2.line(text_image,(0,baseline_row),(image.shape[1],baseline_row),(0,0,255),2)

    return diacritics_image, text_image

# In[16]:

def separate_diacritics_and_text(image, diacritics_ratio=0.2, max_iterations=10, text_convergence_threshold=5):
    """Gets an input image of the arabic text and returns an image containing only the diacritics in it and another
    image with the text as a tuple (diacritics, text)

    Args:
        image: The image you want to separate its diacritics from its text (Read using cv2)
        diacritics_ratio: threshold ratio of diacritics to text (default 0.2)
        max_iterations: maximum number of iterations to run the separtion algorithm (default 1000)
    Returns:
        diacritics_image: A binary image containing only diacritics (white text on black bg)
        text_image: A binary image containing only text (white text on black bg)
    """

    diacritics_image, text_image = separate_diacritics_and_text_utility(image)
    # while diacritics_image.sum() > diacritics_image.size * diacritics_ratio:
    while diacritics_image.sum() > text_image.sum() * diacritics_ratio and max_iterations:
        diacritics_image, text_image_1 = separate_diacritics_and_text_utility(diacritics_image)
        if text_image_1.sum() < text_convergence_threshold:
            break
        text_image = text_image + text_image_1
        max_iterations = max_iterations - 1

    return diacritics_image, text_image

# # 6. Testing

# In[17]:

def testing():

    # importing io module
    import sys
    sys.path.insert(1, "./../io_utils/")
    from io_utils import read_data, read_classes

    # reading data and class names
    classes_names = read_classes('../ACdata_base/names.txt')
    dataset_images, dataset_labels = read_data('../ACdata_base/')

    assert len(dataset_images) == len(dataset_labels)

    # get the range of each class in the dataset
    ranges = [0]
    tmp = dataset_labels[0]
    for idx, num in enumerate(dataset_labels):
        if num != tmp:
            tmp = num
            ranges.append(idx)
    ranges.append(len(dataset_labels))

    # plt.imshow(crop(binarize(cv2.imread("../ACdata_base/9/1624.jpg", 0))), cmap='gray')

    # Choosing a random example from each class and apply the preprocessing Functions on it
    for i, class_name in enumerate(classes_names):
        index = randrange(ranges[i], ranges[i+1])
        test_image = dataset_images[index]

        if(i == 8):
            test_image = cv2.imread("../ACdata_base/9/1624.jpg", 0)

        binary_image = binarize(test_image)
        cropped_image = crop(binary_image)
        assert len(np.unique(np.asarray(binary_image))) == 2

        edge_image = extract_edges(cropped_image)
        skeleton_image = extract_skeleton(cropped_image)
        diacritics_image, text_image = separate_diacritics_and_text(cropped_image)

        f, axarr = plt.subplots(3,2, figsize=(15, 10))
        f.suptitle(class_name + " test")

        axarr[0,0].imshow(test_image, cmap='gray')
        axarr[0,0].set_title("Original")

        # axarr[0,1].imshow(binary_image, cmap='gray')
        # axarr[0,1].set_title("Binary")
        axarr[0,1].imshow(cropped_image, cmap='gray')
        axarr[0,1].set_title("Cropped Binary")

        axarr[1,0].imshow(edge_image, cmap='gray')
        axarr[1,0].set_title("Edge")

        axarr[1,1].imshow(skeleton_image, cmap='gray')
        axarr[1,1].set_title("Skeleton")

        axarr[2,0].imshow(diacritics_image, cmap='gray')
        axarr[2,0].set_title("Diacritics")

        axarr[2,1].imshow(text_image, cmap='gray')
        axarr[2,1].set_title("Text Only")

# In[21]:

# if __name__ == '__main__':
    # testing()

# In[28]:

def create_py():
    get_ipython().system('jupyter nbconvert --to script preprocessing.ipynb')

# In[29]:

if __name__ == '__main__':
    create_py()

# In[ ]:


from django import template
from django.template.defaultfilters import stringfilter

register = template.Library()

@register.filter
@stringfilter
def string_filter(string):
    try:
        return string.split(',')
    except Exception as e:
        error = e
        return string


# from PIL import Image, ImageTk
# from tkinter import Tk, BOTH
# from tkinter.ttk import Frame, Label, Style
import os
import tkinter as tk
from tkinter import *
from PIL import ImageTk,Image
import pyttsx3
def text_to_speech(user_text):
    engine = pyttsx3.init()
    engine.say(user_text)
    engine.runAndWait()

# print(1)
text_to_speech('Authentication Successful!')
text_to_speech('Welcome!')
text_to_speech('Please browse through your options..')
# print(2)
# text_to_speech('Welcome')

def quit(*args):
    root.destroy()

root = Tk()
root.attributes("-fullscreen", True)
root.configure(background='black')
root.bind("<Escape>", quit)
root.bind("x", quit)

titl=Label(root,text="Smart College!!", bd=20, bg="black", fg="green",
	font=('arial', 48),relief=RIDGE).pack(side=TOP, fill=X)
a=Label(root,text="Welcome to the Facial Recognition", bg="black", fg="yellow",
	font=("arial", 56)).pack()
a=Label(root,text="Attendance System", bg="black", fg="yellow",
	font=("arial", 56)).pack()

ri = Image.open("register.png")
r = ImageTk.PhotoImage(ri)
label1 = Label(root, image=r)
label1.image = r
label1.place(x=180, y=300)

ai = Image.open("attendance.png")
a = ImageTk.PhotoImage(ai)
label2 = Label(root, image=a)
label2.image = a
label2.place(x=980, y=300)

vi = Image.open("verifyy.png")
v = ImageTk.PhotoImage(vi)
label3 = Label(root, image=v)
label3.image = v
label3.place(x=600, y=300)

def register():
  os.system('python Add_New.py')

def verify():
	os.system('python Recog.py')

def attendance():
	os.system('python Attendance.py')

r=Button(root, text="Click to register a new student", bd=10,
	font=('times new roman',12),bg="black",fg= "yellow", height=5).place( x=115, y=570)

r=Button(root, text="Give Attendance", bd=10,
	font=('times new roman',12),bg="black",fg= "yellow", height=5).place( x=615, y=570)

r=Button(root, text="View Attendance", bd=10,
	font=('times new roman',12),bg="black",fg= "yellow", height=5).place( x=1020, y=570)
r=Button(root, text="EXIT", bd=10,
	command=quit, font=('times new roman',12),bg="black",fg= "yellow", height=2, width= 10).place( x=637, y=700)

if __name__ == '__main__':
   root.mainloop()


from typing import Any, Tuple

import cupy as cp
import pandas as pd
import pytest

import cudf
from cudf.core._compat import PANDAS_GE_150
from cudf.core.buffer import as_buffer
from cudf.core.column import as_column, build_column
from cudf.core.df_protocol import (
    DataFrameObject,
    _CuDFBuffer,
    _CuDFColumn,
    _DtypeKind,
    _MaskKind,
    _protocol_buffer_to_cudf_buffer,
    from_dataframe,
    protocol_dtype_to_cupy_dtype,
)
from cudf.testing._utils import assert_eq

@pytest.fixture(
    params=[
        {"a": [1, 2, 3], "b": ["x", "y", "z"]},
        {"a": [1, 2, None], "b": ["x", "y", "z"]},
        {"a": [1, 2, 3], "b": pd.Categorical(["x", "y", None])},
    ]
)
def pandas_df(request):
    data = request.param
    return pd.DataFrame(data)

def assert_validity_equal(protocol_buffer, cudf_buffer, size, null, valid):
    if null == _MaskKind.BYTEMASK:
        protocol_mask = _protocol_buffer_to_cudf_buffer(protocol_buffer)
        assert_eq(
            as_column(protocol_mask, dtype="bool"),
            as_column(cudf_buffer, dtype="bool"),
        )
    elif null == _MaskKind.BITMASK:
        protocol_mask = _protocol_buffer_to_cudf_buffer(protocol_buffer)
        cudf_mask = cudf_buffer
        assert_eq(
            build_column(
                as_buffer(cp.zeros(10, dtype="int8")),
                "int8",
                size=size,
                mask=protocol_mask,
                children=(),
            ),
            build_column(
                as_buffer(cp.zeros(10, dtype="int8")),
                "int8",
                size=size,
                mask=cudf_mask,
                children=(),
            ),
        )
    else:
        raise NotImplementedError()

def assert_buffer_equal(buffer_and_dtype: Tuple[_CuDFBuffer, Any], cudfcol):
    buf, dtype = buffer_and_dtype
    device_id = cp.asarray(cudfcol.data).device.id
    assert buf.__dlpack_device__() == (2, device_id)
    col_from_buf = build_column(
        _protocol_buffer_to_cudf_buffer(buf),
        protocol_dtype_to_cupy_dtype(dtype),
    )
    # check that non null values are the equals as nulls are represented
    # by sentinel values in the buffer.
    # FIXME: In gh-10202 some minimal fixes were added to unblock CI. But
    # currently only non-null values are compared, null positions are
    # unchecked.
    non_null_idxs = ~cudf.Series(cudfcol).isna()
    assert_eq(
        col_from_buf.apply_boolean_mask(non_null_idxs),
        cudfcol.apply_boolean_mask(non_null_idxs),
    )
    array_from_dlpack = cp.from_dlpack(buf.__dlpack__()).get()
    col_array = cp.asarray(cudfcol.data_array_view(mode="read")).get()
    assert_eq(
        array_from_dlpack[non_null_idxs.to_numpy()].flatten(),
        col_array[non_null_idxs.to_numpy()].flatten(),
    )

def assert_column_equal(col: _CuDFColumn, cudfcol):
    assert col.size() == cudfcol.size
    assert col.offset == 0
    assert col.null_count == cudfcol.null_count
    assert col.num_chunks() == 1
    if col.null_count == 0:
        pytest.raises(RuntimeError, col._get_validity_buffer)
        assert col.get_buffers()["validity"] is None
    else:
        assert_validity_equal(
            col.get_buffers()["validity"][0],
            cudfcol.mask,
            cudfcol.size,
            *col.describe_null,
        )

    if col.dtype[0] == _DtypeKind.CATEGORICAL:
        assert_buffer_equal(col.get_buffers()["data"], cudfcol.codes)
        assert col.get_buffers()["offsets"] is None

    elif col.dtype[0] == _DtypeKind.STRING:
        assert_buffer_equal(col.get_buffers()["data"], cudfcol.children[1])
        assert_buffer_equal(col.get_buffers()["offsets"], cudfcol.children[0])

    else:
        assert_buffer_equal(col.get_buffers()["data"], cudfcol)
        assert col.get_buffers()["offsets"] is None

    if col.null_count == 0:
        assert col.describe_null == (0, None)
    else:
        assert col.describe_null == (3, 0)

def assert_dataframe_equal(dfo: DataFrameObject, df: cudf.DataFrame):
    assert dfo.num_columns() == len(df.columns)
    assert dfo.num_rows() == len(df)
    assert dfo.num_chunks() == 1
    assert dfo.column_names() == tuple(df.columns)
    for col in df.columns:
        assert_column_equal(dfo.get_column_by_name(col), df[col]._column)

def assert_from_dataframe_equals(dfobj, allow_copy):
    df2 = from_dataframe(dfobj, allow_copy=allow_copy)

    assert_dataframe_equal(dfobj.__dataframe__(allow_copy), df2)
    if isinstance(dfobj, cudf.DataFrame):
        assert_eq(dfobj, df2)

    elif isinstance(dfobj, pd.DataFrame):
        assert_eq(cudf.DataFrame(dfobj), df2)

    else:
        raise TypeError(f"{type(dfobj)} not supported yet.")

def test_from_dataframe_exception(pandas_df):
    exception_msg = "This operation must copy data from CPU to GPU."
    " Set `allow_copy=True` to allow it."
    with pytest.raises(TypeError, match=exception_msg):
        from_dataframe(pandas_df)

def assert_df_unique_dtype_cols(data):
    cdf = cudf.DataFrame(data=data)
    assert_from_dataframe_equals(cdf, allow_copy=False)
    assert_from_dataframe_equals(cdf, allow_copy=True)

def test_from_dataframe():
    data = dict(a=[1, 2, 3], b=[9, 10, 11])
    df1 = cudf.DataFrame(data=data)
    df2 = cudf.from_dataframe(df1)
    assert_eq(df1, df2)

    df3 = cudf.from_dataframe(df2)
    assert_eq(df1, df3)

def test_int_dtype():
    data_int = dict(a=[1, 2, 3], b=[9, 10, 11])
    assert_df_unique_dtype_cols(data_int)

def test_float_dtype():
    data_float = dict(a=[1.5, 2.5, 3.5], b=[9.2, 10.5, 11.8])
    assert_df_unique_dtype_cols(data_float)

def test_categorical_dtype():
    cdf = cudf.DataFrame({"A": [1, 2, 5, 1]})
    cdf["A"] = cdf["A"].astype("category")
    col = cdf.__dataframe__().get_column_by_name("A")
    assert col.dtype[0] == _DtypeKind.CATEGORICAL
    assert col.describe_categorical == (False, True, {0: 1, 1: 2, 2: 5})
    assert_from_dataframe_equals(cdf, allow_copy=False)
    assert_from_dataframe_equals(cdf, allow_copy=True)

def test_bool_dtype():
    data_bool = dict(a=[True, True, False], b=[False, True, False])
    assert_df_unique_dtype_cols(data_bool)

def test_string_dtype():
    data_string = dict(a=["a", "b", "cdef", "", "g"])
    assert_df_unique_dtype_cols(data_string)

def test_mixed_dtype():
    data_mixed = dict(
        int=[1, 2, 3],
        float=[1.5, 2.5, 3.5],
        bool=[True, False, True],
        categorical=[5, 1, 5],
        string=["rapidsai-cudf ", "", "df protocol"],
    )
    assert_df_unique_dtype_cols(data_mixed)

def test_NA_int_dtype():
    data_int = dict(
        a=[1, None, 3, None, 5],
        b=[9, 10, None, 7, 8],
        c=[6, 19, 20, 100, 1000],
    )
    assert_df_unique_dtype_cols(data_int)

def test_NA_float_dtype():
    data_float = dict(
        a=[1.4, None, 3.6, None, 5.2],
        b=[9.7, 10.9, None, 7.8, 8.2],
        c=[6.1, 19.2, 20.3, 100.4, 1000.5],
    )
    assert_df_unique_dtype_cols(data_float)

def test_NA_categorical_dtype():
    df = cudf.DataFrame({"A": [1, 2, 5, 1]})
    df["B"] = df["A"].astype("category")
    df.at[[1, 3], "B"] = None  # Set two items to null

    # Some detailed testing for correctness of dtype and null handling:
    col = df.__dataframe__().get_column_by_name("B")
    assert col.dtype[0] == _DtypeKind.CATEGORICAL
    assert col.null_count == 2
    assert col.describe_null == (3, 0)
    assert col.num_chunks() == 1
    assert col.describe_categorical == (False, True, {0: 1, 1: 2, 2: 5})
    assert_from_dataframe_equals(df, allow_copy=False)
    assert_from_dataframe_equals(df, allow_copy=True)

def test_NA_bool_dtype():
    data_bool = dict(a=[None, True, False], b=[False, None, None])
    assert_df_unique_dtype_cols(data_bool)

def test_NA_string_dtype():
    df = cudf.DataFrame({"A": ["a", "b", "cdef", "", "g"]})
    df["B"] = df["A"].astype("object")
    df.at[1, "B"] = cudf.NA  # Set one item to null

    # Test for correctness and null handling:
    col = df.__dataframe__().get_column_by_name("B")
    assert col.dtype[0] == _DtypeKind.STRING
    assert col.null_count == 1
    assert col.describe_null == (3, 0)
    assert col.num_chunks() == 1
    assert_from_dataframe_equals(df, allow_copy=False)
    assert_from_dataframe_equals(df, allow_copy=True)

def test_NA_mixed_dtype():
    data_mixed = dict(
        int=[1, None, 2, 3, 1000],
        float=[None, 1.5, 2.5, 3.5, None],
        bool=[True, None, False, None, None],
        categorical=[5, 1, 5, 3, None],
        string=[None, None, None, "df protocol", None],
    )
    assert_df_unique_dtype_cols(data_mixed)

@pytest.mark.skipif(
    not PANDAS_GE_150,
    reason="Pandas versions < 1.5.0 do not support interchange protocol",
)
def test_from_cpu_df(pandas_df):
    cudf.from_dataframe(pandas_df, allow_copy=True)