"""Common functionalities for various data models"""
from common.utils.parent_child_nodes_handler import ParentChildNodesHandler

class CommonAPIHandler:
  """Class containing common functions used by various data models"""

  @classmethod
  def update_document(cls, collection, uuid, request_body):
    """Updates the document with given uuid"""
    existing_doc_object = collection.find_by_uuid(uuid)
    existing_doc_object_dict = existing_doc_object.get_fields()
    update_fields = [key for key in existing_doc_object_dict.keys()\
       if isinstance(existing_doc_object_dict[key], dict)]
    for field in update_fields:
      if request_body[field] is None:
        request_body[field] = existing_doc_object_dict[field]
      else:
        for key, val in request_body[field].items():
          if val is None:
            request_body[field][key] = \
              existing_doc_object_dict[field].get(key,[])
    ParentChildNodesHandler.compare_and_update_nodes_references(
        request_body, existing_doc_object_dict, collection)
    for key, value in request_body.items():
      if value is not None:
        setattr(existing_doc_object, key, value)
      existing_doc_object.update()

    updated_doc_fields = \
        existing_doc_object.get_fields(reformat_datetime=True)

    return updated_doc_fields

  @classmethod
  def update_and_create_version(cls, collection, uuid, request_body):
    """
      Creates a new version of the document for the given uuid
      and update references
    """
    existing_doc_object = collection.find_by_uuid(uuid)
    existing_doc_object_dict = existing_doc_object.get_fields()
    update_fields = [key for key in existing_doc_object_dict.keys()\
       if isinstance(existing_doc_object_dict[key], dict)]
    for field in update_fields:
      if request_body[field] is None:
        request_body[field] = existing_doc_object_dict[field]
      else:
        for key, val in request_body[field].items():
          if val is None:
            request_body[field][key] = \
              existing_doc_object_dict[field][key]
    # Creating a versioned doc
    versioned_doc = collection()
    versioned_doc = versioned_doc.from_dict(request_body)

    # Code to find the correct version number
    collection_manager = collection.collection
    collection_manager = collection_manager.filter(
        "root_version_uuid", "==",
        existing_doc_object.root_version_uuid)
    documents = collection_manager.order("-version").fetch(1)
    latest_document = [i.get_fields(reformat_datetime=True) for i in documents
                      ][0]
    versioned_doc.version = latest_document["version"] + 1
    versioned_doc.parent_version_uuid = uuid

    ParentChildNodesHandler.validate_parent_child_nodes_references(request_body)
    for key, value in request_body.items():
      if value is not None:
        setattr(versioned_doc, key, value)
    versioned_doc.uuid = ""
    versioned_doc.save()
    versioned_doc.uuid = versioned_doc.id
    versioned_doc.update()
    updated_doc_fields = versioned_doc.get_fields(reformat_datetime=True)
    ParentChildNodesHandler.update_child_references(
        updated_doc_fields, collection, operation="add")
    ParentChildNodesHandler.update_parent_references(
        updated_doc_fields, collection, operation="add")
    return updated_doc_fields

  @classmethod
  def create_copy(cls, collection, uuid):
    """Function to create COPY of a documnent"""
    doc = collection.find_by_uuid(uuid)
    doc_dict = doc.get_fields\
      (reformat_datetime=True)
    del doc_dict["uuid"]
    del doc_dict["created_time"]
    del doc_dict["last_modified_time"]
    if doc_dict.get("last_published_on") is not None:
      del doc_dict["last_published_on"]
    ParentChildNodesHandler.validate_parent_child_nodes_references(doc_dict)
    doc_dict["version"] = 1

    copy_doc = collection()
    copy_doc = copy_doc.from_dict(doc_dict)
    copy_doc.uuid = ""
    copy_doc = cls.check_item_is_locked(copy_doc)

    copy_doc.save()
    copy_doc.uuid = copy_doc.id
    copy_doc.root_version_uuid = \
      copy_doc.id
    copy_doc.update()
    copy_doc_fields = copy_doc.get_fields(reformat_datetime=True)
    ParentChildNodesHandler.update_child_references(
        copy_doc_fields, collection, operation="add")
    ParentChildNodesHandler.update_parent_references(
        copy_doc_fields, collection, operation="add")
    return copy_doc_fields

  @classmethod
  def check_item_is_locked(cls, item):
    """Function to assign is_locked flag to items
    in Learning Hierarchy upon creation"""
    item.is_locked = False
    if item.prerequisites:
      for prereq_level in item.prerequisites:
        if item.prerequisites[prereq_level]:
          item.is_locked = True
          break
    return item

  @classmethod
  def create_versioned_document(cls, collection, uuid, request_body):
    """
      Creates a new version of the document for the given uuid
      without updating the references
    """
    existing_doc_object = collection.find_by_uuid(uuid)
    existing_doc_object_dict = existing_doc_object.get_fields()

    update_fields = [key for key in existing_doc_object_dict.keys()\
       if isinstance(existing_doc_object_dict[key], dict)]
    for field in update_fields:
      if request_body.get(field) is None:
        request_body[field] = existing_doc_object_dict[field]
      else:
        for key, val in request_body[field].items():
          if val is None:
            request_body[field][key] = \
              existing_doc_object_dict[field][key]

    # Creating a versioned doc
    versioned_doc = collection()
    versioned_doc = versioned_doc.from_dict(existing_doc_object_dict)

    # Code to find the correct version number
    collection_manager = collection.collection
    collection_manager = collection_manager.filter(
        "root_version_uuid", "==",
        existing_doc_object.root_version_uuid)
    documents = collection_manager.order("-version").fetch(1)
    latest_document = [i.get_fields(reformat_datetime=True) for i in documents
                      ][0]
    versioned_doc.version = latest_document["version"] + 1
    versioned_doc.parent_version_uuid = uuid
    for key, value in request_body.items():
      if value is not None:
        setattr(versioned_doc, key, value)
    versioned_doc.save()
    versioned_doc.uuid = versioned_doc.id
    versioned_doc.is_implicit = True
    versioned_doc.update()
    updated_doc_fields = versioned_doc.get_fields(reformat_datetime=True)

    return updated_doc_fields

  @classmethod
  def remove_uuid_from_prerequisites(cls, collection, key, uuid):
    """
    This method removes the given 'uuid' from the given 'key' field in
    prerequisites of all the documents in the given 'collection'.
    """
    docs = collection.collection.fetch()
    for doc in docs:
      doc_dict = doc.to_dict()
      prerequisites = doc_dict.get("prerequisites", {})
      if prerequisites:
        key_field = prerequisites.get(key, [])
        if key_field and uuid in key_field:
          doc.prerequisites[key].remove(uuid)
          doc.update()


import pymysql

def application(environ, start_response):
    conn = pymysql.connect("localhost", "app_sql", "app_sql", "app_sql")
    with conn.cursor() as cursor:
        post_data = ''
        if environ.get("REQUEST_METHOD") == "POST":
            try:
                request_body_size = int(environ.get('CONTENT_LENGTH', 0))
            except (ValueError):
                request_body_size = 0
            post_data = environ['wsgi.input'].read(request_body_size)

        headers = {k[5:]: v for (k, v) in environ.items() if k.startswith("HTTP_")}
        cursor.execute(
            "CREATE TEMPORARY TABLE IF NOT EXISTS `headers` (`name` VARCHAR(255) PRIMARY KEY, `value` VARCHAR(4095))"
        )

        for k, v in headers.items():
            cursor.execute(
                "INSERT INTO `headers` VALUES (%s, %s) ON DUPLICATE KEY UPDATE `value` = %s",
                (k, v, v),
            )

        app_args = [environ["PATH_INFO"], environ["QUERY_STRING"], post_data, None, None]

        try:
            cursor.callproc("app", app_args)
        except pymysql.Error as e:
            print(e)
            start_response("500 Internal Server Error", [("Content-Type", "text/html")])
            return b"Somethin dun broke"

        cursor.execute("SELECT @_app_3, @_app_4")
        status, resp = cursor.fetchone()
        cursor.execute("SELECT `name`, `value` FROM `resp_headers`")
        headers = list(cursor.fetchall())

        conn.commit()

        start_response(status, headers)
        return resp


import mimetypes
from markdown_it import MarkdownIt
from flask import Flask, url_for, abort, render_template, Response
from pathlib import Path
from urllib.parse import urlparse
import html

from pygments import highlight
from pygments.lexers import get_lexer_by_name
from pygments.formatters import HtmlFormatter

app = Flask(__name__)

BASE_PATH = Path(__file__).parent

ARTICLES_PATH = BASE_PATH / 'articles'
IMAGES_PATH = BASE_PATH / 'static/images'

def highlighter(code, lang_name, lang_attrs):
    if not lang_name:
        return html.escape(code)

    lexer = get_lexer_by_name(lang_name, stripall=True)
    formatter = HtmlFormatter(nowrap=True)
    return highlight(code, lexer, formatter)

def render_img(self, tokens, idx, options, env):
    [token] = tokens

    src = token.attrs['src']
    alt_text = token.content
    title = token.attrs.get('title')

    src_parse = urlparse(src)
    alt_text = html.escape(alt_text, quote=True)

    if title:
        title_part = f'title="{html.escape(title, quote=True)}"'
    else:
        title_part = ""

    if not src_parse.netloc:
        filename = Path(src_parse.path).name
        src = url_for('article_image', filename=filename)
        src = html.escape(src, quote=True)
        return f'\n\n'

    src = html.escape(src, quote=True)
    return f'\n\n'

def render_html(path):
    """Render html content from markdown file
    """
    try:
        file = open(path, mode='r', encoding='UTF-8')
    except FileNotFoundError:
        return abort(404)

    with file:
        md_content = file.read()

    renderer = MarkdownIt("commonmark", {"highlight": highlighter})
    renderer.add_render_rule("image", render_img)
    return renderer.render(md_content)

@app.route('/')
def index():
    """Start page with list of articles."""
    concepts_path = ARTICLES_PATH / 'python_concepts.md'
    paths = sorted(ARTICLES_PATH.glob('*.md'))
    post_info = []

    for path in paths:
        with open(path, encoding='utf-8') as file:
            title = file.readline()
            if not title.startswith("#"):
                raise ValueError("Post must begin with a title.")

        slug = path.stem
        title = title.lstrip("# ").strip()
        post_info.append((slug, title))
        concepts = render_html(concepts_path)

    return render_template(
        'index.html', post_info=post_info, concepts=concepts
    )

@app.route('/<slug>/')
def post(slug):
    path = ARTICLES_PATH / f'{slug}.md'
    html_content = render_html(path)

    return render_template('post.html', content=html_content)

@app.route('/article_image/<filename>')
def article_image(filename):
    """Route to returns images saved in static/images"""
    img_path = IMAGES_PATH / filename
    (mimetype, encoding) = mimetypes.guess_type(filename)
    if mimetype:
        img_bytes = img_path.read_bytes()

        return Response(img_bytes, mimetype=mimetype)

    return None


import arcade

import settings

import os

import time

from chapter_1 import Chapter1View

class MenuView(arcade.View):
    def on_show(self):
        arcade.set_background_color(arcade.color.BLACK)
        self.background = settings.BACKGROUND
        self.switch = True
        self.count = 0
        self.background_music = settings.BACKGROUND_MUSIC
        self.ring = settings.RING
        arcade.play_sound(self.background_music)

    def on_draw(self):
        arcade.start_render()

        arcade.draw_texture_rectangle(settings.WIDTH // 2, settings.HEIGHT // 2,
                                      settings.WIDTH, settings.HEIGHT, self.background)

        if self.switch:
            arcade.draw_text("ENTER ", settings.WIDTH // 2, settings.HEIGHT // 4,
                            arcade.color.GOLD, 30, 0,  "center", 'Comic Sans', True, False,
                            "center", "center")
            self.count += 1
            if self.count >= 50:
                self.switch = False
                self.count = 0

        elif not self.switch:
            arcade.draw_text("ENTER ", settings.WIDTH // 2, settings.HEIGHT // 4,
                            arcade.color.GOLDENROD, 30, 0,  "center", 'Comic Sans', True, False,
                            "center", "center")
            self.count += 1
            if self.count >= 50:
                self.switch = True
                self.count = 0

    def on_key_press(self, key, modifiers):
        if key == arcade.key.ENTER:
            arcade.play_sound(self.ring)
            arcade.pause(0.6)
            if len(os.listdir("saves")) == 0:
                self.director.specific_view(Chapter1View)
            else:
                self.director.next_view()

if __name__ == "__main__":
    """This section of code will allow you to run your View
    independently from the main.py file and its Director.

    You can ignore this whole section. Keep it at the bottom
    of your code.

    It is advised you do not modify it unless you really know
    what you are doing.
    """
    from utils import FakeDirector
    window = arcade.Window(settings.WIDTH, settings.HEIGHT)
    my_view = MenuView()
    my_view.director = FakeDirector(close_on_next_view=True)
    window.show_view(my_view)
    arcade.run()


from unittest.mock import Mock, patch

import pytest
from chainsail.common.spec import JobSpec
from chainsail.scheduler.config import GeneralNodeConfig, K8sNodeConfig, SchedulerConfig
from chainsail.scheduler.nodes.base import NodeStatus, NodeType

@pytest.fixture
def mock_scheduler_config():
    config = GeneralNodeConfig(
        image="foo:latest",
        cmd="bash",
        args=["-c", "'echo foo'"],
        ports=[8080],
        user_code_image="bar:earliest",
        httpstan_image="moo:sometag",
    )
    node_config = K8sNodeConfig(
        "fake-configmap-name",
        "xxxxxxxxxxxxx",
        "path/to/key.pem",
        "foo/controller.yaml",
        "foo/storage.yaml",
        "1600m",
        "5000000Ki",
    )
    scheduler_config = SchedulerConfig(
        controller=config,
        worker=config,
        node_type=NodeType.KUBERNETES_POD,
        node_config=node_config,
        results_url_expiry_time=42,
        remote_logging_config_path=None,
        results_endpoint_url="foo",
        results_access_key_id="id",
        results_secret_key="secret",
        results_bucket="results",
        results_dirname="results_dir",
    )
    return scheduler_config

@patch("chainsail.scheduler.config.K8sNodeConfig.create_node_driver")
def test_k8s_node_from_representation(mock_driver, mock_scheduler_config):
    from chainsail.common.spec import JobSpec
    from chainsail.scheduler.db import TblNodes
    from chainsail.scheduler.nodes.base import NodeStatus, NodeType
    from chainsail.scheduler.nodes.k8s_pod import K8sNode

    job_spec = JobSpec("gs://my-bucket/scripts")
    node_rep = TblNodes(
        id=1,
        job_id=1,
        name="dummy-1",
        node_type=NodeType.KUBERNETES_POD,
        entrypoint="echo 'hello world'",
        status=NodeStatus.RUNNING,
        address="127.0.0.1",
        ports="[8080]",
    )

    node = K8sNode.from_representation(job_spec, node_rep, mock_scheduler_config)
    # This method should bind node_rep to the new node
    assert node.representation

@patch("chainsail.scheduler.config.K8sNodeConfig.create_node_driver")
def test_k8s_node_from_config_with_job(mock_driver, mock_scheduler_config):
    from chainsail.common.spec import JobSpec
    from chainsail.scheduler.db import TblJobs
    from chainsail.scheduler.nodes.k8s_pod import K8sNode

    job_spec = JobSpec("gs://my-bucket/scripts")
    node = K8sNode.from_config("dummy-1", mock_scheduler_config, job_spec, job_rep=TblJobs(id=1))
    assert node.representation
    assert node.representation.job.id == 1

@patch("chainsail.scheduler.config.K8sNodeConfig.create_node_driver")
def test_k8s_node_from_representation_then_create(mock_driver, mock_scheduler_config):
    from chainsail.common.spec import JobSpec, PipDependencies
    from chainsail.scheduler.db import TblJobs, TblNodes
    from chainsail.scheduler.nodes.base import NodeStatus, NodeType
    from chainsail.scheduler.nodes.k8s_pod import K8sNode

    job_spec = JobSpec("gs://my-bucket/scripts", dependencies=[PipDependencies(["numpy"])])
    node_rep = TblNodes(
        id=1,
        job_id=1,
        name="new-node",
        node_type=NodeType.KUBERNETES_POD,
        entrypoint="echo 'hello world'",
        # This node has not been actually created yet
        status=NodeStatus.INITIALIZED,
        job=TblJobs(),
    )

    # Create the node object
    with patch("chainsail.scheduler.nodes.k8s_pod.monitor_deployment") as mock_monitor_deployment:
        node = K8sNode.from_representation(
            job_spec,
            node_rep,
            mock_scheduler_config,
        )
        node.refresh_status()
        (is_created, _) = node.create()

    mock_monitor_deployment.assert_called_once()
    assert is_created
    assert node.status == NodeStatus.RUNNING

@patch("chainsail.scheduler.config.K8sNodeConfig.create_node_driver")
def test_k8s_node_lifecycle(mock_driver, mock_scheduler_config):
    from chainsail.common.spec import JobSpec
    from chainsail.scheduler.db import TblJobs, TblNodes
    from chainsail.scheduler.nodes.base import NodeStatus
    from chainsail.scheduler.nodes.k8s_pod import K8sNode

    job_spec = JobSpec("gs://my-bucket/scripts")

    with patch("chainsail.scheduler.nodes.k8s_pod.monitor_deployment"):
        node_rep = TblNodes(
            id=1,
            job_id=1,
            name="new-node",
            node_type=NodeType.KUBERNETES_POD,
            job=TblJobs(id=3),
        )
        node = K8sNode(
            name="test",
            is_controller=True,
            config=mock_scheduler_config.controller,
            node_config=mock_scheduler_config.node_config,
            spec=job_spec,
            representation=node_rep,
        )

        statuses = [node.status]
        is_created, _ = node.create()
        statuses.append(node.status)
        is_restarted = node.restart()
        statuses.append(node.status)
        is_deleted = node.delete()
        statuses.append(node.status)

    # Check that all operations succeeded
    assert all([is_created, is_restarted, is_deleted])
    # If so, the following statuses should have been observed:
    assert statuses == [
        NodeStatus.INITIALIZED,
        NodeStatus.RUNNING,
        NodeStatus.RESTARTING,
        NodeStatus.EXITED,
    ]


from typing import List

class Solution:
    def findRedundantConnection(self, edges: List[List[int]]) -> List[int]:
        # Inicializa o vetor de pais para cada nó
        parents = [i for i in range(len(edges) + 1)]

        # Função para encontrar o pai de um nó usando Union-Find
        def find(x):
            if parents[x] == x:
                return x
            else:
                parents[x] = find(parents[x])
                return parents[x]

        # Função para unir dois conjuntos usando Union-Find
        def union(x, y):
            px, py = find(x), find(y)
            if px != py:
                parents[px] = py

        # Percorre as arestas e verifica se a adição de cada uma delas forma um ciclo
        for edge in edges:
            if find(edge[0]) == find(edge[1]):
                return edge
            else:
                union(edge[0], edge[1])

        # Se não houver ciclo, retorna a última aresta
        return edges[-1]


# -*- coding: utf-8 -*-
# This is a management command, rather than a migration script, for two primary reasons:
#   1. It makes no changes to database structure (e.g. AlterField), only database content.
#   2. It may need to be ran more than once, as it skips failed registrations.

from __future__ import unicode_literals
from datetime import timedelta
import logging

import django
django.setup()

from django.core.management.base import BaseCommand
from django.db import transaction

from osf.models import Registration, Retraction, Sanction
from scripts import utils as script_utils

logger = logging.getLogger(__name__)

def set_date_retracted(*args):
    registrations = (
        Registration.objects.filter(retraction__state=Sanction.APPROVED, retraction__date_retracted=None)
        .select_related('retraction')
        .prefetch_related('registered_from__logs')
        .prefetch_related('registered_from__guids')
    )
    total = registrations.count()
    logger.info('Migrating {} retractions.'.format(total))

    for registration in registrations:
        if not registration.registered_from:
            logger.warn('Skipping failed registration {}'.format(registration._id))
            continue
        retraction_logs = registration.registered_from.logs.filter(action='retraction_approved', params__retraction_id=registration.retraction._id)
        if retraction_logs.count() != 1 and retraction_logs.first().date - retraction_logs.last().date > timedelta(seconds=5):
            msg = (
                'There should be a retraction_approved log for retraction {} on node {}. No retraction_approved log found.'
                if retraction_logs.count() == 0
                else 'There should only be one retraction_approved log for retraction {} on node {}. Multiple logs found.'
            )
            raise Exception(msg.format(registration.retraction._id, registration.registered_from._id))
        date_retracted = retraction_logs[0].date
        logger.info(
            'Setting date_retracted for retraction {} to be {}, from retraction_approved node log {}.'.format(
                registration.retraction._id, date_retracted, retraction_logs[0]._id
            )
        )
        registration.retraction.date_retracted = date_retracted
        registration.retraction.save()

def unset_date_retracted(*args):
    retractions = Retraction.objects.filter(state=Sanction.APPROVED).exclude(date_retracted=None)
    logger.info('Migrating {} retractions.'.format(retractions.count()))

    for retraction in retractions:
        retraction.date_retracted = None
        retraction.save()

class Command(BaseCommand):
    """
    Backfill Retraction.date_retracted with `RETRACTION_APPROVED` log date.
    """
    def add_arguments(self, parser):
        super(Command, self).add_arguments(parser)
        parser.add_argument(
            '--dry',
            action='store_true',
            dest='dry_run',
            help='Run migration and roll back changes to db',
        )
        parser.add_argument(
            '--reverse',
            action='store_true',
            dest='reverse',
            help='Unsets date_retraction'
        )

    def handle(self, *args, **options):
        reverse = options.get('reverse', False)
        dry_run = options.get('dry_run', False)
        if not dry_run:
            script_utils.add_file_logger(logger, __file__)
        with transaction.atomic():
            if reverse:
                unset_date_retracted()
            else:
                set_date_retracted()
            if dry_run:
                raise RuntimeError('Dry run, transaction rolled back.')


# -*- coding: utf-8 -*-
import pickle
import gzip
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

def load_data():
    f = gzip.open('../mnist.pkl.gz', 'rb')
    train_data, val_data, test_data = pickle.load(f, encoding='bytes')
    f.close()
    return train_data, val_data, test_data

def load_data_wrapper():
    train_data, val_data, test_data = load_data()
    train_X = [x.reshape(784, 1) for x in train_data[0]]
    train_y = [vectorize(y) for y in train_data[1]]

    val_X = [x.reshape(784, 1) for x in val_data[0]]
    test_X = [x.reshape(784, 1) for x in test_data[0]]

    return (list(zip(train_X, train_y)),
            list(zip(val_X, val_data[1])),
            list(zip(test_X, test_data[1]))
            )

def vectorize(i):
    y = np.zeros((10, 1))
    y[i] = 1.0
    return y

def plot_iamges6 (data):
    ilist = np.random.permutation(range(len(data)))
    fig = plt.figure()

    for j in range(1, 7):
        image, _ = data[ilist[j]]
        image = image.reshape(28, -1)
        ax = fig.add_subplot(1, 6, j)
        ax.matshow(image, cmap=matplotlib.cm.binary)
        plt.xticks(np.array([]))
        plt.yticks(np.array([]))

    plt.show()


from django.shortcuts import render, redirect
from django.http import HttpRequest, HttpResponse

from games.models import Game
from member.models import Member
from .models import TagItem, Tag

def tag(request: HttpRequest, game_id: int) -> HttpResponse:
    game = Game.objects.get(pk=game_id)
    member = request.user.member
    tag_items = game.tag_items.all()
    tags = Tag.objects.get_tags_of_game(game)
    context = {'member':member, 'game':game, 'tags':tags}
    return render(request, 'tags/tag.html', context)

def add(request: HttpRequest, game_id: int) -> HttpResponse:
    game = Game.objects.get(pk=game_id)
    member = request.user.member
    label = request.GET.get('tag')
    tag = Tag.objects.get_or_create(label=label)[0]
    tag_item = TagItem.create(tag, member, game)
    tag_item.save()
    return redirect('games:game', gameID=game_id)

def view_games_by_tag(request: HttpRequest, label: str) -> HttpResponse:
    tag = Tag.objects.get(label=label)
    games = Tag.objects.get_games_by_tag(tag)
    context = {'games': games}
    return render(request, 'tags/view_games_by_tag.html', context)

def view_all_tags(request: HttpRequest) -> HttpResponse:
    tags = Tag.objects.all()
    context = {'tags':tags}
    return render(request, 'tags/tags.html', context)


import torch
from torch.utils.data.dataset import Dataset as BaseDataset
import numpy as np

"""
Class, representing observed items by user
UserItemDataset takes as argument array-like element,
each element contains tuple (user_id, item_id)
By default, class will reindex user_ids and item_ids to match
GMF, MLP, NeuMF embeddings' vocablulary
Also, class will flood data with negative samples
All parameters can be tuned via config
"""
class UserItemDataset(BaseDataset):
    def __init__(self, data_dict, config):
        self.config = config
        self.data = data_dict['samples']
        if 'user_features' in data_dict:
            self.user_features = data['user_features']
        else:
            self.user_features = None

        if 'item_features' in data_dict:
            self.item_features = data['item_features']
        else:
            self.item_features = None

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        user, item, label = list(map(lambda x: torch.tensor(x), self.data[idx]))
        user, item = user.long(), item.long()
        sample = {}
        sample['user'] = user
        sample['item'] = item
        sample['label'] = label
        if self.user_features is not None:
            sample['user_features'] = self.user_features[user.item()]
        if self.item_features is not None:
            sample['item_features'] = self.item_features[item.item()]
        return sample

    """
    Data reindexing was moved to class, because if you create negative
    samples when init data, you can accidentally miss indicies or samples, which
    contains in test split
    So, you should use #reindex_data and #flood_negative, before train/test split.
    It will ensure, that no data was leaked and all users and items were mapped to
    new indicies
    Also, method returns both data and mappers, so reindexing user and item features
    indicies is user's responsibility, because features are too general for trying
    to do it with no information about its format.
    """
    @staticmethod
    def reindex_data(data_samples):
        user_mapper = {}
        item_mapper = {}
        reindexed_data = []
        for user, item in data_samples:
            uid = insert(user, user_mapper)
            iid = insert(item, item_mapper)
            reindexed_data.append((uid, iid))
        return (reindexed_data, user_mapper, item_mapper)

    @staticmethod
    def flood_negative(positive_data_samples, config):
        per_user = config.negatives_per_user
        #For fast search we need to create set of tuples like (user, item)
        positive_set = set(map(lambda x: (x[0], x[1]), positive_data_samples))
        #To iterate over each user once (and don't search in already created negative samples)
        user_set = set(map(lambda x: x[0], positive_data_samples))

        #Cause items have been already reindexed, MAX_ITEM_ID = len(set(ITEMS)) - 1
        item_next_idx = len(set(map(lambda x: x[1], positive_data_samples)))
        negative_samples = []
        for user in user_set:
            sample = set()
            while len(sample) != per_user:
                item_idx = np.random.randint(0, item_next_idx)
                pair = (user, item_idx)
                if pair not in sample and pair not in positive_set:
                    sample.add(pair)
            #Generate samples with zero labels
            negative_samples.extend(list(sample))
        return negative_samples

    @staticmethod
    def union_samples(positive, negative):
        data = []
        positive = list(map(lambda x: (*x, 1.0), positive))
        negative = list(map(lambda x: (*x, 0.0), negative))
        data.extend(positive)
        data.extend(negative)
        return data

def insert(idx, idx_dict):
    if idx not in idx_dict:
        new_idx = len(idx_dict)
        idx_dict[idx] = new_idx
    else:
        new_idx = idx_dict[idx]
    return new_idx

if __name__ == "__main__":

    from hparams.utils import Hparam
    from torch.utils.data.dataloader import DataLoader
    import sys

    config = Hparam(sys.argv[1]).dataset
    data = [[420, 606], [420, 101], [69, 101], [69, 202]]
    reindexed_data, user_mapper, item_mapper = UserItemDataset.reindex_data(data)
    negative_samples = UserItemDataset.flood_negative(reindexed_data, config)
    data = UserItemDataset.union_samples(reindexed_data, negative_samples)
    data_dict = { "samples" : data }

    dataset = UserItemDataset(data_dict, config)
    dataloader = DataLoader(dataset, batch_size=2, shuffle=True)

    print('converted user ids ', user_mapper)
    print('converted item ids ', item_mapper)
    # Ensure, that dataset supports batches
    for _, data in enumerate(dataloader, 0):
        print(data)


import re
import matplotlib.pyplot as plt
import argparse
import numpy as np
import os

def main_grid(flog):

    assert os.path.isfile(flog), 'The file at the path does not exist.'
    file = open(flog, 'r')
    top1 = []
    top5 = []
    for line in file:
        match = re.search('done, Batch', line)
        if match:
            value = re.findall('[0-9]+\.[0-9]+', line)
            if value is not None:
                top1.append(float(value[5]))
                top5.append(float(value[3]))
    file.close()
    print('#points = ', len(top1))

    x = np.linspace(0, len(top1), len(top1))
    plt.plot(x, top1, linestyle='-', label='top1')
    plt.plot(x, top5, linestyle='--', label='top5')

    plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Log extractor.')
    parser.add_argument('--path', type=str, help='The path of the file which need to be extracted.')

    args = parser.parse_args()

    path = args.path
    main_grid(path)


from django.shortcuts import render, redirect
from django.utils.decorators import method_decorator
from django.contrib.auth.models import User, Group, Permission
from django.views.decorators.csrf import csrf_protect
from django.contrib import messages
from django.db.models import Q
from rest_framework import fields
from rest_framework.serializers import ModelSerializer, PrimaryKeyRelatedField
from rest_framework.viewsets import ModelViewSet

from QuanLyNhaSach.views.base import BaseITSAdminView

class ProfileView(BaseITSAdminView):
    """
    Profile View (Information my account)
    """

    def __init__(self):
        super(ProfileView, self).__init__()
        self.template_name = 'pages/profile.html'

class UserListView(BaseITSAdminView):
    """
    List users view
    """

    def __init__(self):
        super(UserListView, self).__init__()
        self.template_name = 'pages/user_list.html'

class UserEditView(BaseITSAdminView):
    """
    Update users view
    """

    def __init__(self):
        super(UserEditView, self).__init__()
        self.template_name = 'pages/user_edit.html'

    @method_decorator(csrf_protect)
    def get(self, request, user_id, **params):
        user = User.objects.get(pk=user_id)
        selected_permissions = Permission.objects.filter(Q(user=user), Q(content_type__app_label='QuanLyNhaSach') | Q(
            content_type__app_label='auth'))
        unselected_permissions = Permission.objects.exclude(user=user).filter(
            Q(content_type__app_label='QuanLyNhaSach') | Q(
                content_type__app_label='auth'))
        selected_groups = Group.objects.filter(user=user)
        unselected_groups = Group.objects.exclude(user=user)
        self.extra.update({'selected_permissions': selected_permissions,
                           'unselected_permissions': unselected_permissions,
                           'selected_groups': selected_groups,
                           'unselected_groups': unselected_groups,
                           'user_selected': user,
                           }
                          )
        params.update(self.extra)
        return render(request, self.template_name, params)

class UserAddView(BaseITSAdminView):
    """
    Add user view
    """

    def __init__(self):
        super(UserAddView, self).__init__()
        self.template_name = 'pages/user_add.html'

    @method_decorator(csrf_protect)
    def post(self, request, **kwargs):
        email = request.POST['email']
        first_name = request.POST['first_name']
        last_name = request.POST['last_name']
        user_exist = User.objects.filter(email=email, username=email).count()
        if user_exist:
            messages.warning(request, 'Your email was exist please use another')
            return redirect('QuanLyNhaSach:users_add')

        user = User(username=email, email=email, first_name=first_name,
                    last_name=last_name)
        user.save()
        user.username = user.id
        user.save()
        unselected_permissions = Permission.objects.filter(Q(content_type__app_label='QuanLyNhaSach') | Q(
            content_type__app_label='auth'))
        unselected_groups = Group.objects.all()
        self.extra.update({'unselected_permissions': unselected_permissions,
                           'unselected_groups': unselected_groups,
                           'user_selected': user,
                           }
                          )

        return render(request, 'pages/user_edit.html', self.extra)

class ErrorView(BaseITSAdminView):
    def __init__(self):
        super(ErrorView, self).__init__()
        self.template_name = 'pages/error.html'

"""
API users
include: Serializer class, Permission class, ModelViewSet class
"""

class UserSerializer(ModelSerializer):
    id = fields.CharField(required=False)
    email = fields.EmailField(required=False)
    username = fields.CharField(required=False)
    first_name = fields.CharField(required=False, allow_blank=True, allow_null=True)
    last_name = fields.CharField(required=False, allow_blank=True, allow_null=True)
    last_login = fields.DateTimeField(required=False)
    date_joined = fields.DateTimeField(required=False)
    is_active = fields.BooleanField(required=False, default=True)
    is_staff = fields.BooleanField(required=False, default=True)
    is_superuser = fields.BooleanField(required=False, default=True)
    groups_ids = PrimaryKeyRelatedField(required=False, many=True, read_only=False, queryset=Group.objects.all(),
                                        source='groups')
    permissions_ids = PrimaryKeyRelatedField(required=False, many=True, read_only=False,
                                             queryset=Permission.objects.filter(
                                                 Q(content_type__app_label='QuanLyNhaSach') | Q(
                                                     content_type__app_label='auth')),
                                             source='user_permissions')

    class Meta:
        model = User
        fields = (
            'id', 'username', 'email', 'first_name', 'last_name', 'last_login', 'date_joined', 'is_active',
            'groups_ids', 'permissions_ids', 'is_staff', 'is_superuser')
        databases_always_serialize = (
            'id', 'username', 'email', 'first_name', 'last_name', 'last_login', 'date_joined', 'is_active',
            'groups_ids', 'permissions_ids', 'is_staff', 'is_superuser')

class UserListViewSet(ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UserSerializer


from django.shortcuts import render, redirect
from .forms import DjangoForm
from .models import Pustak

# Create your views here.
def addbook(request):
    if request.method == 'POST':
        form_data = DjangoForm(request.POST)
        if form_data.is_valid():
            title = request.POST['title']
            author = request.POST.get('author')
            edition = request.POST['edition']
            price = request.POST['price']
            Pustak.objects.create(title=title, author=author, edition=edition, price=price)
            return redirect('home')
    form = DjangoForm()
    return render(request, 'djangoForm/book.html', context={'forms': form})

def home(request):
    book_data = Pustak.objects.all()
    return render(request, 'djangoForm/base.html', context={'booklist': book_data})


#!/usr/bin/env python

import wx

#from scipy import *

import os, cPickle, pdb, sys

import reusablepanels
from reusablepanels.rwkSingleFileChooser_v2 import rwkSingleFileChooser
from reusablepanels.rwkdirchooserpanel import rwkdirchooserpanel

#import pydocument
#reload(pydocument)

#import pypres
#reload(pypres)

import rwkmisc
#reload(rwkmisc)

import rwkos
#reload(rwkos)

import texpypanel
import latexpanel
import pyppanel
import fancyhdrpanel
import wltpanel

settingsname = 'pytexguisettings.pkl'

class MyFrame(wx.Frame):
    def __init__(self, parent, title):
        wx.Frame.__init__(self, parent, -1, title,
                          pos=(150, 150), size=(650, 725))
        # Create the menubar
        menuBar = wx.MenuBar()

        # and a menu
        menu = wx.Menu()

        # add an item to the menu, using \tKeyName automatically
        # creates an accelerator, the third param is some help text
        # that will show up in the statusbar
        menu.Append(wx.ID_EXIT, "E&xit\tAlt-X", "Exit the application")

        # bind the menu event to an event handler
        self.Bind(wx.EVT_MENU, self.OnTimeToClose, id=wx.ID_EXIT)

        # and put the menu on the menubar
        menuBar.Append(menu, "&File")
        self.SetMenuBar(menuBar)

        #create top level controls (i.e. those not on a notebook page)
        #create the main directory chooser
        self.topdirchooser = rwkdirchooserpanel(self, -1)
        #and the main exit button
        self.exitbutton = wx.Button(self, -1, "Exit")

        #create the notebook
        self.notebook = wx.Notebook(self, -1, style=0)

        #notebook panes
        self.fancyhdrpane = fancyhdrpanel.FancyHdrPanel(self.notebook, dircontrol=self.topdirchooser)
        self.texpypane = texpypanel.TexPyPanel(self.notebook, dircontrol=self.topdirchooser, fancyhdrpanel=self.fancyhdrpane)
        self.purelatexpane = latexpanel.LatexPanel(self.notebook, dircontrol=self.topdirchooser, settingsname='purelatexsettings.pkl')
        self.pyppane = pyppanel.PYPPanel(self.notebook, dircontrol=self.topdirchooser, fancyhdrpanel=self.fancyhdrpane)
        self.wltpane = wltpanel.WLTPanel(self.notebook, dircontrol=self.topdirchooser, fancyhdrpanel=self.fancyhdrpane)

        #create the sizer for the first pane
        grid_sizer = wx.FlexGridSizer(2, 1, 0, 0)

        #create widgets for pane one
        #self.panel = wx.Panel(self)
        #self.label_4 = wx.StaticText(self.notebook_pane_1, -1, "label_4")
        #self.button_1 = wx.Button(self.notebook_pane_1, -1, "button_1")

        #add the widgets to pane one's sizer
        #grid_sizer.Add(self.label_4, 0, 0, 0)
        #grid_sizer.Add(self.button_1, 0, 0, 0)

        #self.notebook_pane_1.SetSizer(grid_sizer)

        grid_sizer.AddGrowableCol(0)
        grid_sizer.AddGrowableRow(0)
        #Add pane one to the notebook
        self.notebook.AddPage(self.texpypane, "TexPy/TexMaxima")
        self.notebook.AddPage(self.pyppane, "PYP")
        self.notebook.AddPage(self.purelatexpane, "Pure LaTeX")
        self.notebook.AddPage(self.fancyhdrpane, "fancyhdr")
        self.notebook.AddPage(self.wltpane, "WLT")
        #create the sizer for the main frame
        mainsizer = wx.FlexGridSizer(3, 1, 0, 0)
        mainsizer.AddGrowableCol(0)
        mainsizer.AddGrowableRow(1)

        #add the controls to the main sizer
        mainsizer.Add(self.topdirchooser, 1, wx.EXPAND|wx.ALL, 5)
        mainsizer.Add(self.notebook, 1, wx.EXPAND, 5)
        mainsizer.Add(self.exitbutton, 1, wx.ALL|wx.ALIGN_RIGHT, 5)

        self.Bind(wx.EVT_BUTTON, self.OnTimeToClose, self.exitbutton)

        self.startdir = os.getcwd()
        self.settingspath = os.path.join(self.startdir, settingsname)

        if self.startdir not in sys.path:
            sys.path.insert(0, self.startdir)

        self.SetSizer(mainsizer)
        self.savepanellist = [self.texpypane, self.purelatexpane, self.pyppane, self.fancyhdrpane, self.wltpane]
        #mainsizer.Fit(self)
        self.Layout()
        self.LoadSettings()

    def SaveSettings(self): # wxGlade: MyFrame.<event_handler>
        mydict = {}
#        for key, chooser in zip(self.pathlist, self.filechoosers):
#            curpath = chooser.GetPath()
#            mydict[key] = curpath
#        wlt = self.wltcheckbox.GetValue()
        topdir = self.topdirchooser.GetPath()
        mydict['topdir'] = topdir
        nbind = self.notebook.GetSelection()
        mydict['nbind'] = nbind
        mypkl = open(self.settingspath,'wb')
        #print('mydict='+str(mydict))
        cPickle.dump(mydict,mypkl)
        mypkl.close()
        #for key, val in mydict.iteritems():
        #    print str(key) + ':' + str(val)
        for panel in self.savepanellist:
            panel.SaveSettings()

    def LoadSettings(self): # wxGlade: MyFrame.<event_handler>
        if os.path.exists(self.settingspath):
            mypkl = open(self.settingspath,'rb')
            mydict = cPickle.load(mypkl)
            mypkl.close()
            #print('mydict='+str(mydict))
            if mydict.has_key('topdir'):
                self.topdirchooser.SetPath(mydict['topdir'])
            if mydict.has_key('nbind'):
                self.notebook.SetSelection(mydict['nbind'])
        else:
            self.topdirchooser.SetPath(os.getcwd())
        for panel in self.savepanellist:
            panel.LoadSettings()

    def OnTimeToClose(self, evt):
        """Event handler for the button click."""
        self.SaveSettings()
        self.Close()

class MyApp(wx.App):
    def OnInit(self):
        frame = MyFrame(None, "PyTex Office Suite")
        self.SetTopWindow(frame)

        #print "Print statements go to this stdout window by default."

        frame.Show(True)
        return True

if __name__ == "__main__":
    app = MyApp(redirect=False)
    app.MainLoop()


import json
import argparse
import sys

def strip_irrelevant_vacancy_info(raw_vacancy):
    payment = raw_vacancy['payment'] or raw_vacancy['payment_from']
    payment = payment or raw_vacancy['payment_to']
    trimmed_vacancy = { 'profession': raw_vacancy['profession'],
                        'candidat': raw_vacancy['candidat'],
                        'payment': payment }
    return trimmed_vacancy

def get_stripped_vacancy_list(raw_vacancies):
    stripped_vacancy_list = []
    for raw_vacancy in raw_vacancies:
        stripped_vacancy_list.append(strip_irrelevant_vacancy_info(raw_vacancy))
    return stripped_vacancy_list

def get_cli_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--infile', type=argparse.FileType('r'),
                        default=sys.stdin,
                        help='JSON file with vacancies from SuperJob, stdin by default')
    parser.add_argument('-o', '--outfile', type=argparse.FileType('w'),
                        default=sys.stdout,
                        help='JSON file for trimmed vacancies, stdout by default')
    return parser.parse_args()

if __name__ == '__main__':
    args = get_cli_arguments()
    raw_vacancies = json.load(args.infile)
    stripped_vanacies = get_stripped_vacancy_list(raw_vacancies)
    json.dump(stripped_vanacies, args.outfile)


import os
import sys
import traceback
import codecs
import logging
import random
import time
import rdflib
import dateutil.parser

from optparse    import OptionParser
from nltools     import misc
from config      import RDF_PREFIXES

DEFAULT_OUTPUT     = 'bar.pl'
DEFAULT_LOGLEVEL   = logging.INFO
LEM_FN             = 'etc/lem.csv'
LPM_FN             = 'etc/lpm.csv'
RDF_SCHEMA_LABEL   = rdflib.URIRef('http://www.w3.org/2000/01/rdf-schema#label')
LANGUAGES          = set([u'en', u'de'])
LEGAL_ENTITY_CHARS = set(u"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖÙÚÛÜÝßàáâãäåæçèéêëìíîïðñòóôõöùúûüý")

def mangle_prolog(n, start_uppercase=False):

    res = u''
    make_upper = start_uppercase # to enforce CamelCase

    for c in n:
        if not c.isalnum():
            make_upper = True
            continue
        if make_upper:
            c = c.upper()
            make_upper=False

        if c in LEGAL_ENTITY_CHARS:
            res += c
        else:
            res += u"U%d" % ord(c)
    return res

def mangle_url(s, do_exc=True):
    prefix = None
    for p, url in RDF_PREFIXES.items():
        if not s.startswith(url):
            continue
        n      = s[len(url):]
        prefix = p
        break

    if not prefix:
        if do_exc:
            raise Exception ('no prefix for %s found' % s)
        return None

    return prefix + mangle_prolog(n, start_uppercase=True)

def entity_label(s, label):

    prefix = None
    for p, url in RDF_PREFIXES.items():
        if not s.startswith(url):
            continue
        prefix = p
        break

    if not prefix:
        raise Exception ('no prefix for %s found' % s)

    l = prefix + mangle_prolog(label, start_uppercase=True)

    return l[:100]

def property2entity_mapper(p):

    up = unicode(p)

    if up.startswith('http://www.wikidata.org/prop/direct/'):
        return 'wdpd', rdflib.URIRef('http://www.wikidata.org/entity/' + p[36:])
    if up.startswith('http://www.wikidata.org/qualifier/'):
        return 'wdpq', rdflib.URIRef('http://www.wikidata.org/entity/' + p[34:])
    if up.startswith('http://www.wikidata.org/prop/statement/'):
        return 'wdps', rdflib.URIRef('http://www.wikidata.org/entity/' + p[39:])
    if up.startswith('http://www.wikidata.org/prop/'):
        return 'wdp', rdflib.URIRef('http://www.wikidata.org/entity/' + p[29:])
    return None, None

def property_label(p):

    global lpm, plm, g

    if p in plm:
        return plm[p]

    prefix, pe = property2entity_mapper(p)
    if not pe:

        l = mangle_url(p)

    else:

        triples = list(g.triples((pe, RDF_SCHEMA_LABEL, None)))

        l      = None
        for t in triples:

            s, p2, o = t

            if o.language != 'en':
                continue

            l = prefix + mangle_prolog(unicode(o), start_uppercase=True)

        if not l:
            return None

    # make unique

    label = l
    cnt = 1
    while label in lpm and lpm[label] != p:
        label = l + unicode(cnt)
        cnt += 1

    lpm[label] = p
    plm[p]     = label

    return label

#
# init, cmdline
#

misc.init_app('rdf2prolog')

parser = OptionParser("usage: %prog [options] foo.n3")

parser.add_option ("-v", "--verbose", action="store_true", dest="verbose",
                   help="verbose output")
parser.add_option ("-o", "--output", dest="outputfn", type = "string", default=DEFAULT_OUTPUT,
                   help="output file, default: %s" % DEFAULT_OUTPUT)

(options, args) = parser.parse_args()

if len(args) != 1:
    parser.print_usage()
    sys.exit(1)

if options.verbose:
    logging.basicConfig(level=logging.DEBUG)
else:
    logging.basicConfig(level=logging.INFO)

inputfn  = args[0]
outputfn = options.outputfn

#
# get everything
#

logging.info ('parsing %s ...' % inputfn)

g = rdflib.Graph()
g.parse(inputfn, format='n3')

logging.info ('parsing %s ... extracting triples ...' % inputfn)
triples = list(g)

logging.info ('parsing %s ... extracting triples ... got %d triples.' % (inputfn, len(triples)))

#
# entity labels
#

lem = {} # label -> entity
elm = {} # entity -> label

#
# read existing entity labels
#

if os.path.isfile(LEM_FN):

    with codecs.open(LEM_FN, 'r', 'utf8') as f:

        while True:

            line = f.readline()
            if not line:
                break

            line = line.strip()
            if len(line)==0:
                continue

            if 'Unlabeled' in line:
                logging.info ('ignoring line: %s' % line)
                continue

            if 'unlabeled' in line:
                logging.info ('ignoring line: %s' % line)
                continue

            parts = line.split(',')
            elu = mangle_prolog(parts[0])
            entity = rdflib.URIRef(u','.join(parts[1:]))

            if elu in lem:
                raise Exception (u'error while parsing %s: entity label %s is not unique!' % (LEM_FN, elu))

            lem[elu]    = entity
            elm[entity] = elu

#
# generate missing entity labels from rdfsLabels
#

for t in triples:

    s, p, o = t

    if s in elm:
        continue

    label_base = None

    # 1: see if we have an rdfsLabel

    for s2, p2, o2 in g.triples((s, RDF_SCHEMA_LABEL, None)):
        if not isinstance(o2, rdflib.Literal) or o2.language != 'en':
            continue
        label_base = unicode(o2)
        break

    if label_base:
        el = entity_label (s, label_base)
    else:
        el = mangle_url(unicode(s))

    # ensure label is unique

    elu = el

    cnt = 1
    while elu in lem and lem[elu] != s:
        elu = el + unicode(cnt)
        cnt += 1

    lem[elu] = s
    elm[s]   = elu

    #print s, elu

logging.debug ('have unique labels for %d entities' % len(lem))

#
# load existing property labels
#

lpm = {} # label    -> property
plm = {} # property -> label

if os.path.isfile(LPM_FN):
    with codecs.open(LPM_FN, 'r', 'utf8') as f:

        while True:

            line = f.readline()
            if not line:
                break

            line = line.strip()
            if len(line)==0:
                continue

            parts = line.split(',')
            label = parts[0]
            prop  = rdflib.URIRef(u','.join(parts[1:]))

            if label in lpm:
                raise Exception (u'error while parsing %s: entity label %s is not unique!' % (LEM_FN, elu))

            lpm[label] = prop
            plm[prop]  = label

def prolog_string_escape (o):

    s = unicode(o)
    # s = s.replace('\\',' ').replace ("'", "\\'")
    s = s.replace('\\',' ').replace ('"', '\\"')

    return s

#
# generate prolog, generate and cache property labels as we encounter them
#

cnt = 0

prolog_code = []
predicate_set = set()
for elu in lem:

    entity = lem[elu]

    cnt += 1

    triples = list(g.triples((entity, None, None)))

    if len(triples)==0:
        continue

    logging.info ('%5d/%5d generating prolog code for %s ...' % (cnt, len(lem), elu))

    # f.write(u'\n%% URI: %s\n\n' % unicode(entity))

    for s, p, o in triples:

        if not s in elm:
            continue
        el = elm[s]

        try:
            el.decode('ascii')
        except UnicodeDecodeError:
            el = u"'" + el + u"'"

        pl = property_label(p)
        if not pl:
            logging.debug (u'Skipping1: %s(%s, %s).\n' % (unicode(p), el, unicode(o)))
            continue

        try:

            if isinstance(o, rdflib.term.Literal):
                if o.datatype:

                    datatype = str(o.datatype)

                    if datatype == 'http://www.w3.org/2001/XMLSchema#decimal':
                        prolog_code.append(u"%s(%s, %s).\n" % (pl, el, unicode(o)))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    elif datatype == 'http://www.w3.org/2001/XMLSchema#float':
                        prolog_code.append(u"%s(%s, %s).\n" % (pl, el, unicode(o)))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    elif datatype == 'http://www.w3.org/2001/XMLSchema#integer':
                        prolog_code.append(u"%s(%s, %s).\n" % (pl, el, unicode(o)))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    elif datatype == 'http://www.w3.org/2001/XMLSchema#dateTime':
                        dt = dateutil.parser.parse(unicode(o))
                        prolog_code.append(u'%s(%s, "%s").\n' % (pl, el, dt.isoformat()))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    elif datatype == 'http://www.w3.org/2001/XMLSchema#date':
                        dt = dateutil.parser.parse(unicode(o))
                        prolog_code.append(u'%s(%s, "%s").\n' % (pl, el, dt.isoformat()))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    elif datatype == 'http://www.opengis.net/ont/geosparql#wktLiteral':
                        # FIXME
                        prolog_code.append(u'%s(%s, "%s").\n' % (pl, el, unicode(o)))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    elif datatype == 'http://www.w3.org/1998/Math/MathML':
                        # FIXME
                        continue

                    else:
                        raise Exception('unknown literal datatype %s (value: %s) .' % (datatype, unicode(o)))
                else:
                    if o.value is None:
                        prolog_code.append(u"%s(%s, []).\n" % (pl, el))
                        predicate_set.add(u'%s/2' % pl)
                        continue
                    if o.language:
                        if o.language in LANGUAGES:
                            prolog_code.append(u'%s(%s, %s, "%s").\n' % (pl, el, o.language, prolog_string_escape(o)))
                            predicate_set.add(u'%s/3' % pl)
                        continue

                    prolog_code.append(u'%s(%s, "%s").\n' % (pl, el, prolog_string_escape(o)))
                    predicate_set.add(u'%s/2' % pl)
                    continue

            elif isinstance (o, rdflib.term.URIRef):
                if o in elm:
                    ol = elm[o]
                else:
                    # ol = u"'" + unicode(o) + "'"
                    ol = mangle_url(str(o), do_exc=False)
                if not ol:
                    logging.debug (u'Skipping2: %s(%s, %s).\n' % (pl, el, unicode(o)))
                    continue

                try:
                    ol.decode('ascii')
                except UnicodeDecodeError:
                    ol = u"'" + ol + u"'"

                prolog_code.append(u"%s(%s, %s).\n" % (pl, el, ol))
                predicate_set.add(u'%s/2' % pl)
            else:
                raise Exception ('unknown term: %s (%s %s)' % (unicode(o), type(o), o.__class__))
        except:
            logging.error(u'Failed to convert %s: %s' % (unicode(o), traceback.format_exc()))

with codecs.open(outputfn, 'w', 'utf8') as f:

    f.write('%prolog\n')

    for p in sorted(predicate_set):
        f.write(u':- multifile %s.\n' % p)

    f.write('\n')

    for pc in sorted(prolog_code):
        f.write(pc)

logging.info (' %s written' % outputfn)

#
# dump out all property and entity labels to keep them the same in future runs
#

with codecs.open(LPM_FN, 'w', 'utf8') as f:
    for plu in sorted(lpm):
        if 'Unlabeled' in plu:
            continue
        if 'unlabeled' in plu:
            continue
        prop = lpm[plu]
        f.write(u'%s,%s\n' % (plu, prop))

logging.info (' %s written' % LPM_FN)

with codecs.open(LEM_FN, 'w', 'utf8') as f:
    for elu in sorted(lem):
        if 'Unlabeled' in elu:
            continue
        if 'unlabeled' in elu:
            continue
        entity = lem[elu]
        f.write(u'%s,%s\n' % (elu, unicode(entity)))

logging.info (' %s written' % LEM_FN)


# The base of this code was copied from the below link:
# https://www.proxiesapi.com/blog/recursively-scraping-webpages-with-scrapy.html.php
# We will still be obeying the ROBOTS.txt file, and we will be taking in 'start_urls'
# from the google_search.py file.

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
import bisect

suggested_URLs = []

def lister(suggested_url):
    global suggested_URLs
    bisect.insort(suggested_URLs, suggested_url)
    # print(suggested_URLs)
    # print("\n\n\n\n")
    # suggested_URLs += [suggested_url]

def getURLs():
    return suggested_URLs

class WebWalkerSpider(CrawlSpider):
    name = 'web_walker'
    # start_urls = ['http://www.geeksforgeeks.org/']
    start_urls = []
    cont = True

    # Want to read in seeds from file walker_seeds.txt
    seeds = open('walker/spiders/walker_seeds.txt', 'r')
    num_url = 0

    while cont:
        num_url += 1
        url = seeds.readline()
        # print("Read URL: " + url)
        start_urls += [url]

        if not url:
            cont = False
            break

    rules = (
        Rule(LinkExtractor(), callback='parse_item', follow=True),
    )

    custom_settings = {
        'CONCURRENT_REQUESTS': 10,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 25,
        'CONCURRENT_ITEMS': 100,
        'REACTOR_THREADPOOL_MAXSIZE': 400,
        # Hides printing item dicts
        'LOG_LEVEL': 'INFO',
        'RETRY_ENABLED': False,
        'REDIRECT_MAX_TIMES': 1,
        # Stops loading page after 5mb
        'DOWNLOAD_MAXSIZE': 5592405,
        # Grabs xpath before site finish loading
        'DOWNLOAD_FAIL_ON_DATALOSS': False,
        'DEPTH_LIMIT' : 5,
        'DEPTH_PRIORITY': 1,
        'SCHEDULER_DISK_QUEUE' : 'scrapy.squeues.PickleFifoDiskQueue',
        'SCHEDULER_MEMORY_QUEUE' :'scrapy.squeues.FifoMemoryQueue'
    }

    def parse_item(self, response):
        # print('Downloaded... ' + response.url)
        # print(response.text)
        # print("ECDSA appears: " + str(response.text.count("ECDSA")))
        suggested = {}

        if (response.text.count(self.query) > 4):
            suggested = tuple((response.text.count(self.query), response.url))
            print(suggested)
            lister(suggested)

        yield

    def close(self, reason):
        print("****************************************SPIDER CLOSING****************************************")
        final_list = getURLs()
        final_list.reverse()
        for url in final_list:
            print(url)

        # In here, we should parse the page, identify the count of words that
        # match the keywords, and weight them according to matches and extensions

        # filename = 'storage/' response.url.split("/")[-1]   '.html'
        # print('Saving as :' filename)
        # with open(filename, 'wb') as f:
        #    f.write(response.body)


from typing import Callable

from kombu import Connection, Message
from kombu.mixins import ConsumerProducerMixin
from . import definitions
from .definitions import main_queue, main_exchange
from .events import BaseEvent, PingPong, EVENT_TYPE_PING, EVENT_TYPE_PONG, EVENT_INTERNAL_READY

HandlersType = list[dict[str, Callable[[BaseEvent], None]]]

class Worker(ConsumerProducerMixin):
    def __init__(self, connection: Connection, handlers: HandlersType):
        self.connection = connection
        self.handlers = handlers

        self.active_clients = set()

    def on_consume_ready(self, connection, channel, consumers, **kwargs):
        self.ping()

        self.process_handlers({
            'event_type': EVENT_INTERNAL_READY
        })

    def ping(self):
        ping: PingPong = {
            'event_type': EVENT_TYPE_PING,
            'pong': False
        }

        self.producer.publish(ping,
                              exchange=main_exchange,
                              routing_key=PingPong.__name__,
                              headers={'client_id': definitions.client_id})

    def get_consumers(self, consumer_class, channel):
        return [consumer_class(main_queue, callbacks=[self.on_message])]

    def on_message(self, body: BaseEvent, message: Message):
        print(body, message)

        if body['event_type'] == EVENT_TYPE_PING:
            self.on_ping()
        elif body['event_type'] == EVENT_TYPE_PONG:
            self.on_pong(message)

        self.process_handlers(body)

        message.ack()

    def process_handlers(self, event: BaseEvent):
        for handler in self.handlers:
            handler_func = handler.get(event['event_type'])
            if handler_func is not None:
                handler_func(event)

    def on_ping(self):
        pong: PingPong = {
            'event_type': EVENT_TYPE_PONG,
            'pong': True
        }
        print('publish')
        self.producer.publish(pong,
                              exchange=main_exchange,
                              routing_key='event',
                              headers={'client_id': definitions.client_id})

    def on_pong(self, message: Message):
        self.active_clients.add(message.headers['client_id'])


#!/usr/bin/env python3
import requests
import sys
from bs4 import BeautifulSoup
from colorama import Fore
import re

def checkLen(url,spcw="11116661111",):
    global result
    url = url.replace(r"[clm]",spcw) if url.find(r"[clm]") != -1 else False
    result = requests.get(url)
    ptn = spcw.replace("(","\(") if spcw.find("(") else spcw
    ptn = ptn.replace(")","\)") if ptn.find(")") else False
    k = len(re.findall(ptn,result.text))
    contentLen = len(result.text) - (len(spcw) * k)
    return contentLen

# def infoCheck(url):
#     try:
#         result = requests.get(url)
#         if result.status_code == 200:
#             return 1, len(result.text)
#         # if(contentL > defS) :
#             # print(f"{furl:50s}   {Fore.GREEN}[Sucess]")
#             # soup = BeautifulSoup(result.text, "html.parser")
#             # myF =  soup.find_all("span", class_="fontc1")[0].getText()
#             # print(myF)
#         # else :
#             # print(f"{furl:50s}   {Fore.RED}[Failed]")
#     except Exception as e:
#         print(e)
#         return 0, 0

def frontend(argv):
    global host, payload, url, furl
    if len(argv) == 1:
        host = input(f"{Fore.LIGHTGREEN_EX}Enter host : {Fore.WHITE}")
        host = "http://" + host if host.startswith("http://") != True else host
        payload = "?" + \
             input(f"{Fore.LIGHTGREEN_EX}Enter payload : {Fore.WHITE}")

    elif len(argv) == 3 and argv[1] == "-u":
        host = sys.argv[2].split("?")[0]
        payload = "?"+sys.argv[2].split("?")[1]

    else:
        print(f"USAGE : python {argv[0]} [-u] Url [file] ")

    url = host + payload

if __name__ == "__main__":
    # x = requests.get(url)
    # defS = len(x.text)
    url = ""
    param = ""
    frontend(sys.argv)

    while(True):

        furl = input(f"{Fore.GREEN}Enter file location : {Fore.WHITE}")
        loadF = "LOAD_FILE"
        clm = loadF + "('" + furl + "')"
        if checkLen(url,clm) > checkLen(url) :
                r = checkLen(url,clm)
                print(f"{furl:50s}   {Fore.GREEN}[Sucess]")
                soup = BeautifulSoup(result.text, "html.parser")
                myF =  soup.find_all("span", class_="fontc1")[0].getText()
                print(f"{Fore.YELLOW}{myF}")
        else :
                print(f"{furl:50s}   {Fore.RED}[Failed]")


from typing import Generator, Any

import pytest
from graphene.test import Client
from graphene_django.utils.testing import graphql_query
from rest_framework.test import APIClient

from app.authentication.models import User
from test_fixtures import fixture_users

@pytest.fixture(scope='session')
def query_client() -> Generator[Client, Any, None]:
    """
    GraphQL Test Client
    :return: Client
    :rtype: Client
    """
    from django_graphql.schema import schema
    yield Client(schema)

@pytest.fixture(scope='session')
def gql_query(client):
    def func(*args, **kwargs):
        return graphql_query(*args, **kwargs, client=client)

    return func

@pytest.fixture(scope='session')
def api_client() -> Generator[APIClient, Any, None]:
    """
    DRF API Client
    :return APIClient
    :rtype APIClient
    """
    yield APIClient()

@pytest.fixture(scope='session')
def django_db_setup():
    """
    Override dependency django_db_setup
    """
    from django.conf import settings
    settings.DATABASE = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'db_test.sqlite3',
            # 'NAME': ':memory',
        }
    }

@pytest.fixture
def create_superuser():
    User.objects.create_superuser(
        "raynor@dev.com",
        "raynor@dev.com",
        "passw0rd"
    )

@pytest.fixture(scope='function')
def create_random_users():
    """
    Create Users with faker
    """
    fixture_users.create_users()

@pytest.fixture(scope='function')
def create_random_articles_with_random_users():
    """
    Create Users with faker
    Create Articles with fake users
    """
    users = fixture_users.create_users()
    fixture_users.create_articles(users)

# def kakao_sender():
#     yield FakeKakaoSender()


import glob, os
import xml.etree.ElementTree as ET
import pickle
import json
import re
from pathlib import Path

from Runner import BaseRunner, create_runner
from SRDataParser import SRDataParser
from utils import execute, show, get_absolute_path

class SwarmRunner:
    """
    The SwarmRunner orchestrates the execution of the optimisation, the benchmark and the loading of the parameters defined in the
    config_file.json file.
    """
    _loop_functions = {}        # description of the loop functions
    _data_logger_exe = ""       # execution path of the data logger

    def __init__(self, argos_file, execution_dir, results_dir = None, robots_num = None, loop_function = None) -> None:
        self.robots_num = robots_num
        self.base_argos_file = get_absolute_path(argos_file)                    # absolute path the argos file used in the experiment
        self.execution_dir = get_absolute_path(execution_dir)
        self.current_argos_file = f"{self.execution_dir}/experiment.argos"      # absolute path the argos file used by the optimisers
        self.benchmar_res_file = f"{self.execution_dir}/data.csv"
        self.results_dir = get_absolute_path(results_dir)
        self.runner_list = []                                                   # list of runners
        self.benchmark_results = {}
        self.id_dict = {}                                                       # dictionary of the runners' ids with the number of apparitions
        self.load_config()
        self.add_loop_function(loop_function)

    def load_config(self):
        """Loads the config_file.json file"""
        if (len(self._loop_functions) == 0):
            p = Path(__file__).with_name("config_file.json")

            with p.open("r") as config:
                data = json.load(config)

            data = get_absolute_paths_relative_to_file(data)

            SwarmRunner._loop_functions = data["loop_functions"]
            SwarmRunner._data_logger_exe = data["exe_paths"]["data_logger_exe"]

        BaseRunner.load_config(data)

    def add_loop_function(self, loop_function):
         if loop_function is not None:
            try:
                self.loop_function = self._loop_functions[loop_function]
            except KeyError:
                print(f"Loop function {loop_function} was not found in the config file")
                raise

    def add_runner(self,description):
        # makes sure that each runner has a unique id
        id = description.setdefault("id", str(description["runner"]))
        self.id_dict.setdefault(id, 0)
        if self.id_dict[id] != 0:
            description["id"] += str(self.id_dict[id])
        self.id_dict[id] += 1

        self.runner_list.append(create_runner(description, self.current_argos_file))

    def run_optimisation(self):
        for runner in self.runner_list:

            self.clear_exe_folder()
            self.build_argos_file(runner, self.robots_num)

            runner.run_optimisation()

            if self.results_dir is not None:
                runner.save_optimisation_results(self.results_dir)

    def clear_exe_folder(self):
        files = glob.glob(f'{self.execution_dir}/*')
        for f in files:
            os.remove(f)

    def benchmark(self, metrics, runs=1):

        run_number_pattern = re.compile(r"(?<=Run : )\d+")      # pattern used to keep track of the run number so that a progress
                                                                # bar can be displayed
        metrics = list(set(metrics))

        print(f'\n\n {"-"*30} Benchmark {"-"*30}\n\n')

        print(f"Metrics being collected : {metrics} \n")

        for runner in self.runner_list:
            self.build_argos_file(runner, self.robots_num, metrics)
            cmd = f"{SwarmRunner._data_logger_exe} -n -c {self.current_argos_file} -r {runs} -f data.csv --metrics"
            for metric in metrics:
                cmd += " " + metric

            for output in execute(cmd, self.execution_dir):
                run_num = run_number_pattern.findall(output)
                if (len(run_num) != 0):
                    show(f'Benchmarking {runner.description["id"]} ', int(run_num[0]) + 1, runs)
            self.pars_results(runner)

        self.save_results()
        return self.benchmark_results

    def save_results(self):
        if self.results_dir is None:
            return
        if not os.path.exists(self.results_dir):
            os.makedirs(self.results_dir)
        with open(f"{self.results_dir}/benchamrk_results.pkl", "wb") as output_file:
            pickle.dump(self.benchmark_results, output_file)

    def pars_results(self, runner):
        SRDataParser.parse_results(self.benchmar_res_file, runner.description["id"], self.benchmark_results)

    def build_argos_file(self, runner, entities_nbr = None, metrics = None):
        base = ET.parse(self.base_argos_file)
        root = base.getroot()

        runner.add_controller(root, metrics is not None)    # if metrics are not none it means the .argos file should be build for a benchmark
        self.set_number_of_entities(root, entities_nbr)
        self.set_loop_function(root)

        with open(self.current_argos_file, 'wb') as f:
            base.write(f, xml_declaration=True)

    def set_number_of_entities(self, experiment, entities):
        if entities is None:
            return
        if isinstance(entities, int):
            entities = [entities]

        variables = experiment.findall("arena/distribute/entity")
        for i, robots in enumerate(variables):
            robots.set("quantity", str(entities[i]))

    def set_loop_function(self, experiment):
        if len(self.loop_function) == 0:
            return
        loop_func = experiment.findall("loop_functions")[0]
        loop_func.set("library", self.loop_function["lib"])
        loop_func.set("label", self.loop_function["label"])

def get_absolute_paths_relative_to_file(files):
    """
    Recursively searches through the varible files until the value is a string. In this case the value
    is assumed to be a path if it contains a "/" character. If so, the path is replace by the absolute path
    relative to the location of this file.
    """
    path = os.path.abspath(os.path.dirname(__file__))
    for keys in files:
        if isinstance(files[keys], dict):
            files[keys] = get_absolute_paths_relative_to_file(files[keys])
            continue
        if "/" not in files[keys]:
            continue
        if os.path.isabs(files[keys]):
            continue
        files[keys] = os.path.join(path, files[keys])
    return files


from __future__ import print_function, with_statement, absolute_import
import shutil
import keras
import logging
import os
import sys

from ..version import __version__, __registry__
from .deployer_utils import save_python_function
from ..exceptions import ClipperException

logger = logging.getLogger(__name__)

def create_endpoint(clipper_conn,
                    name,
                    input_type,
                    func,
                    model_path_or_object,
                    default_output="None",
                    version=1,
                    slo_micros=3000000,
                    labels=None,
                    registry=None,
                    base_image="default",
                    num_replicas=1,
                    batch_size=-1,
                    pkgs_to_install=None):
    """Registers an app and deploys the provided predict function with Keras model as
    a Clipper model.

    Parameters
    ----------
    clipper_conn : :py:meth:`clipper_admin.ClipperConnection`
        A ``ClipperConnection`` object connected to a running Clipper cluster.
    name : str
        The name to be assigned to both the registered application and deployed model.
    input_type : str
        The input_type to be associated with the registered app and deployed model.
        One of "integers", "floats", "doubles", "bytes", or "strings".
    func : function
        The prediction function. Any state associated with the function will be
        captured via closure capture and pickled with Cloudpickle.
    model_path_or_object : keras Model object or a path to a saved Model ('.h5')
    default_output : str, optional
        The default output for the application. The default output will be returned whenever
        an application is unable to receive a response from a model within the specified
        query latency SLO (service level objective). The reason the default output was returned
        is always provided as part of the prediction response object. Defaults to "None".
    version : str, optional
        The version to assign this model. Versions must be unique on a per-model
        basis, but may be re-used across different models.
    slo_micros : int, optional
        The query latency objective for the application in microseconds.
        This is the processing latency between Clipper receiving a request
        and sending a response. It does not account for network latencies
        before a request is received or after a response is sent.
        If Clipper cannot process a query within the latency objective,
        the default output is returned. Therefore, it is recommended that
        the SLO not be set aggressively low unless absolutely necessary.
        100000 (100ms) is a good starting value, but the optimal latency objective
        will vary depending on the application.
    labels : list(str), optional
        A list of strings annotating the model. These are ignored by Clipper
        and used purely for user annotations.
    registry : str, optional
        The Docker container registry to push the freshly built model to. Note
        that if you are running Clipper on Kubernetes, this registry must be accesible
        to the Kubernetes cluster in order to fetch the container from the registry.
    base_image : str, optional
        The base Docker image to build the new model image from. This
        image should contain all code necessary to run a Clipper model
        container RPC client.
    num_replicas : int, optional
        The number of replicas of the model to create. The number of replicas
        for a model can be changed at any time with
        :py:meth:`clipper.ClipperConnection.set_num_replicas`.
    batch_size : int, optional
        The user-defined query batch size for the model. Replicas of the model will attempt
        to process at most `batch_size` queries simultaneously. They may process smaller
        batches if `batch_size` queries are not immediately available.
        If the default value of -1 is used, Clipper will adaptively calculate the batch size for
        individual replicas of this model.
    pkgs_to_install : list (of strings), optional
        A list of the names of packages to install, using pip, in the container.
        The names must be strings.
    """

    clipper_conn.register_application(name, input_type, default_output,
                                      slo_micros)
    deploy_keras_model(clipper_conn, name, version, input_type, func,
                       model_path_or_object, base_image, labels,
                       registry, num_replicas, batch_size,
                       pkgs_to_install)

    clipper_conn.link_model_to_app(name, name)

def deploy_keras_model(clipper_conn,
                       name,
                       version,
                       input_type,
                       func,
                       model_path_or_object,
                       base_image="default",
                       labels=None,
                       registry=None,
                       num_replicas=1,
                       batch_size=-1,
                       pkgs_to_install=None):
    """Deploy a Python prediction function with a Keras Model object or model file ('.h5').

    Parameters
    ----------
    clipper_conn : :py:meth:`clipper_admin.ClipperConnection`
        A ``ClipperConnection`` object connected to a running Clipper cluster.
    name : str
        The name to be assigned to both the registered application and deployed model.
    version : str
        The version to assign this model. Versions must be unique on a per-model
        basis, but may be re-used across different models.
    input_type : str
        The input_type to be associated with the registered app and deployed model.
        One of "integers", "floats", "doubles", "bytes", or "strings".
    func : function
        The prediction function. Any state associated with the function will be
        captured via closure capture and pickled with Cloudpickle.
    model_path_or_object : keras Model object or a path to a saved Model ('.h5')
    base_image : str, optional
        The base Docker image to build the new model image from. This
        image should contain all code necessary to run a Clipper model
        container RPC client.
    labels : list(str), optional
        A list of strings annotating the model. These are ignored by Clipper
        and used purely for user annotations.
    registry : str, optional
        The Docker container registry to push the freshly built model to. Note
        that if you are running Clipper on Kubernetes, this registry must be accesible
        to the Kubernetes cluster in order to fetch the container from the registry.
    num_replicas : int, optional
        The number of replicas of the model to create. The number of replicas
        for a model can be changed at any time with
        :py:meth:`clipper.ClipperConnection.set_num_replicas`.
    batch_size : int, optional
        The user-defined query batch size for the model. Replicas of the model will attempt
        to process at most `batch_size` queries simultaneously. They may process smaller
        batches if `batch_size` queries are not immediately available.
        If the default value of -1 is used, Clipper will adaptively calculate the batch size for
        individual replicas of this model.
    pkgs_to_install : list (of strings), optional
        A list of the names of packages to install, using pip, in the container.
        The names must be strings.

    Example
    -------
    Deploy a Keras Model::

        from clipper_admin import ClipperConnection, DockerContainerManager
        from clipper_admin.deployers import keras as keras_deployer
        import keras

        # creating a simple Keras model
        inpt = keras.layers.Input(shape=(1,))
        out = keras.layers.multiply([inpt, inpt])
        model = keras.models.Model(inputs=inpt, outputs=out)

        clipper_conn = ClipperConnection(DockerContainerManager())

        # Connect to an already-running Clipper cluster
        clipper_conn.connect()

        def predict(model, inputs):
            return [model.predict(x) for x in inputs]

        keras_deployer.deploy_keras_model(clipper_conn=clipper_conn, name="pow", version="1", input_type="ints",
                                  func=predict,
                                  model_path_or_object=model,
                                  base_image='keras-container')

        # sending an inference request
        import requests
        import json

        req_json = json.dumps({
                "input": [1, 2, 4, 6]
            })

        headers = {"Content-type": "application/json"}
        response = requests.post("http://localhost:1337/keras-pow/predict", headers=headers,
                      data=req_json)

    """
    # save predict function
    serialization_dir = save_python_function(name, func)
    # save Keras model or copy the saved model into the image
    if isinstance(model_path_or_object, keras.Model):
        model_path_or_object.save(os.path.join(serialization_dir, "keras_model.h5"))
    elif os.path.isfile(model_path_or_object):
        try:
            shutil.copy(model_path_or_object,
                        os.path.join(serialization_dir, "keras_model.h5"))
        except Exception as e:
            logger.error("Error copying keras model: %s" % e)
            raise e
    else:
        raise ClipperException(
            "%s should be wither a Keras Model object or a saved Model ('.h5')" % model_path_or_object)

    py_minor_version = (sys.version_info.major, sys.version_info.minor)
    # Check if Python 2 or Python 3 image
    if base_image == "default":
        if py_minor_version < (3, 0):
            logger.info("Using Python 2 base image")
            base_image = "{}/keras-container:{}".format(
                __registry__, __version__)
        elif py_minor_version == (3, 5):
            logger.info("Using Python 3.5 base image")
            base_image = "{}/keras35-container:{}".format(
                __registry__, __version__)
        elif py_minor_version == (3, 6):
            logger.info("Using Python 3.6 base image")
            base_image = "{}/keras36-container:{}".format(
                __registry__, __version__)
        elif py_minor_version == (3, 7):
            logger.info("Using Python 3.7 base image")
            base_image = "{}/keras37-container:{}".format(
                __registry__, __version__)
        else:
            msg = (
                "Keras deployer only supports Python 2.7, 3.5, 3.6, and 3.7. "
                "Detected {major}.{minor}").format(
                major=sys.version_info.major, minor=sys.version_info.minor)
            logger.error(msg)
            # Remove temp files
            shutil.rmtree(serialization_dir)
            raise ClipperException(msg)

    # Deploy model
    clipper_conn.build_and_deploy_model(
        name, version, input_type, serialization_dir, base_image, labels,
        registry, num_replicas, batch_size, pkgs_to_install)

    # Remove temp files
    shutil.rmtree(serialization_dir)


from math import ceil, sqrt
from random import randint
import matplotlib.pyplot as plt
from timeit import timeit
import time

def power(n, p, mod):
    if p == 1:
        return n
    n %= mod
    t = 1

    while p > 0:
        if p & 1:
            t = (t * n) % mod
        n = (n * n) % mod
        p = p >> 1

    return t

def trial_division(n):
    if n < 2:
        raise Exception('Value error: number must be greater than 1')
    if n % 2 == 0:
        return False

    for divisor in range(1, ceil(sqrt(n) + 1), 2):
        if n % divisor == 0:
            return False
    return True

def Miller_iteration(d, n):
    a = randint(3, n - 2)
    a_exp_d = power(a, d, n)

    if a_exp_d == 1 or a_exp_d == n - 1:
        return True

    while d != n - 1:
        a_exp_d = (a_exp_d ** 2) % n
        d *= 2
        if a_exp_d == 1:
            return False

        if a_exp_d == n - 1:
            return True
    return False

def Miller_Rabin(n, checks = 30):
    if n <= 1:
        return False

    if n <= 3:
        return True

    if n % 2 == 0:
        return False

    d = n - 1
    while d % 2 == 0:
        d //= 2

    for check in range(checks):
        if not Miller_iteration(d, n):
            return False

    return True

def gcd(a, b):
    while b:
        a %= b
        a, b = b, a
    return a

def extended_gcd(a, b):
    if a == 0:
        return b, 0, 1
    res, x1, y1 = extended_gcd(b % a, a)
    x = y1 - (b // a) * x1
    y = x1
    return res, x, y

def Jacobi(n, m):
    s = 1
    if gcd(n, m) != 1:
        return 0

    while True:
        n = n % m
        k = 0
        while n % 2 == 0:
            n = n >> 1
            k += 1
        k %= 2

        if k == 1 and (m % 8 == 3 or m % 8 == 5):
            s *= -1

        if n == 1 or (n == m - 1 and m % 4 == 1):
            return s

        if n == m - 1 and m % 4 == 3:
            return -s

        n, m = m, n
        if (n - 1) % 4 and (m - 1) % 4:
            s *= -1

def Solovay_Strassen(n, checks = 30):
    if n % 2 == 0:
        return False

    if n % 3 == 0:
        return False

    for _ in range(checks):
        a = randint(3, n - 1)
        if power(a, (n - 1) // 2, n) != (Jacobi(a, n) % n):
            return False
    return True

def rsa(bit_length, is_prime = Miller_Rabin):
    p, q = generate_prime(bit_length, is_prime), generate_prime(bit_length, is_prime)
    fi = (p - 1) * (q - 1)
    e = generate_public(fi)
    d = generate_secret(e, fi)
    n = p * q
    return (e, n), (d, n)

def generate_prime(bit_length, is_prime):
    num = randint(2 ** (bit_length - 1) + 1, 2 ** bit_length)
    while not is_prime(num):
        num = randint(2 ** (bit_length - 1), 2 ** bit_length)
    return num

def generate_public(fi):
    max_length = len(str(fi)) - 2
    while True:
        length = randint(max_length // 2, max_length)
        if length < 3:
            length = max_length
        one_bit_amount = length // 10
        if one_bit_amount < 3:
            one_bit_amount = 2

        num = (1 << length) + 1
        for i in range(one_bit_amount):
            num |= (1 << randint(1, length - 1))
            if gcd(num, fi) == 1:
                return num

def generate_secret(e, fi):
    d = extended_gcd(e, fi)[1]
    if d < 0:
        d += fi
    return d

def encode(message, public_key):
    result = ''
    for c in message:
        result += str(power(ord(c), public_key[0], public_key[1])) + ' '

    return result

def decode(message, private_key):
    result = ''
    for c in message.split():
        result += chr(power(int(c), private_key[0], private_key[1]))

    return result

def lineplot(ax, x_data, y_data, x_label="", y_label="", color="#539caf"):
    ax.plot(x_data, y_data, lw = 2, color=color, alpha = 1)
    ax.set_xlabel(x_label)
    ax.set_ylabel(y_label)

_range = range(5, 11)
_, ax = plt.subplots()
ax.set_title('RSA: comparasion of prime test algorithms')

def test(_range, ax, algorithm, color='blue'):
    times = []

    for i in _range:
        t = time.time()
        rsa(2 ** i, algorithm)
        times.append(time.time() - t)
    lineplot(ax, _range, times, y_label = 'seconds', x_label = 'length: 2^', color=color)

test(_range, ax, Miller_Rabin, 'red')
test(_range, ax, Solovay_Strassen, 'blue')
plt.show()


import re

from datetime import date
from dateutil.relativedelta import relativedelta

from trytond.model import ModelView, ModelSQL, fields
from trytond.pool import Pool
from trytond.pyson import Bool, Eval, Not

__all__ = ['GGDNotifyRecipient', 'GGD']

DEFAULT_SUBJECT = "Garantie de Gestion Durable expiration for %(company)s"
DEFAULT_CONTENT = """
This is an automated mail to remind you that the GGD
of %(company)s will expire in %(months)i months.
Please connect to the Tryton server to update their certification
informations.

Thanks.
"""

_GGD = [
    ('RTG', u'Réglement Type de Gestion'),
    ('PSG', u'Plan Simple de Gestion'),
    ('CBPS', u'Code de Bonnes Partiques Sylvicoles')
]

class GGDNotifyRecipient(ModelView, ModelSQL):
    "GGD notification recipient"
    __name__ = 'ggd.ggd_notify_recipient'
    sender = fields.Many2One('party.party', 'Sender',
                             required=True,
                             help='User used for the From address in notifications about '
                                  'GGD expiration')
    recipient = fields.Many2One('party.party', 'Recipient',
                                required=True,
                                help='User that will receive mail notifications about '
                                     'GGD expiration')
    msg_subject = fields.Char('Message subject',
                              translate=True,
                              required=True)
    msg_content = fields.Text('Message content',
                              translate=True,
                              required=True)

    @classmethod
    def __setup__(cls):
        super(GGDNotifyRecipient, cls).__setup__()
        cls._error_messages = {'invalid_party': 'Recipients of GGD notifications must have a valid mail address!'}

    @staticmethod
    def default_msg_subject():
        """Default mail subject"""
        return DEFAULT_SUBJECT

    @staticmethod
    def default_msg_content():
        """Default mail content"""
        return DEFAULT_CONTENT

    @classmethod
    def check_party(cls, records):
        """Check the recipient user has a mail address"""
        for record in records:
            if record.sender.email == '' or record.recipient.email == '':
                cls.raise_user_error('invalid_party')

class GGD(ModelView, ModelSQL):
    "GGD"
    __name__ = 'ggd.ggd'
    _rec_name = 'label'

    enabled = fields.Boolean(
            string=u'Active',
            help=u'Active'
        )

    STATES = {
        'readonly': Not(Bool(Eval('enabled'))),
        'required': Bool(Eval('enabled')),
    }

    typo = fields.Selection(
            _GGD,
            string=u'Type',
            help=u'Type de garanties de gestion durable',
            states=STATES,
            required=True
        )

    ggd_no = fields.Char(
            string=u'GGD ID',
            help='GGD ID number in the form "XX-XXXX-XXXXX"',
            states=STATES,
            required=True
        )
    date = fields.Date(
            string=u'Agreement date',
            help=u'GGD agreement date',
            states=STATES
        )
    duration = fields.Integer(
            string=u'Duration (m)',
            help=u'Garanty duration in months',
            states=STATES
        )
    date_fin = fields.Date(
            string=u'End of Agreement date',
            help=u'En of GGD agreement date',
            states=STATES,
            on_change_with=['date', 'duration'],
        )

    def on_change_with_date_fin(self):
        if (self.date or self.duration) is not None:
            until = self.date + relativedelta(months=self.duration)
            return until

    label = fields.Function(fields.Char(
                        string=u'Label',
                        depends=['date', 'duration', 'ggd_no', 'departement'],
                        readonly=True,
                        on_change_with=['date', 'duration', 'ggd_no', 'departement']
            ), 'get_label'
        )
    surface  = fields.Float(
            string=u'Surface (ha)',
            help=u'Surface garanty in hectare',
            digits=(16, 4),
            states=STATES,
        )
    departement = fields.Many2One(
            'country.subdivision',
            'Department',
            domain=[('type', '=', 'metropolitan department')],
            required=True,
            states=STATES
        )
    validity = fields.Function(fields.Boolean('Validity',
                               depends=['date', 'duration'], readonly=True,
                               on_change_with=['date', 'duration']),
                               'is_valid',
                               searcher='search_is_valid'
       )
    party = fields.Many2One(
            'party.party',
            string=u'Party',
            ondelete='RESTRICT',
            domain=[('categories', 'child_of', [1], 'parent')]
        )

    @classmethod
    def __setup__(cls):
        super(GGD, cls).__setup__()
        cls._sql_constraints += [('ggd_uniq', 'UNIQUE(ggd_no, date)', 'The garanty date must be unique!')]
        cls._error_messages = {'invalid_no': 'You must provide a GGD agreement ID in the form "XX-XXXX-XXXXX"!'}

    @classmethod
    def validate(cls, records):
        """Check GGD ID format"""
        for record in records:
            if not record.enabled:
                continue

            if not re.match(u"\d{2}-\d{4}-\d{1,}$", record.ggd_no):
                cls.raise_user_error('invalid_no')

    def is_valid(self, ids):
        if self.date is None or self.duration is None:
            return False
        return self.date + relativedelta(months=self.duration) > date.today()

    @classmethod
    def search_is_valid(cls, name, clause):
        GGD = Pool().get('ggd.ggd')
        ggds = GGD.search([('enabled', '=', True)])

        ids = []
        if ggds is not None:
            for ggd in ggds:
                if ggd.is_valid(None):
                    ids.append(ggd.id)
        return [('id', 'in', ids)]

    def on_change_with_validity(self):
        return self.is_valid(None)

    def on_change_with_label(self):
        return self.get_label(None)

    def get_label(self, ids):
        """User friendly garanty name"""
        until = self.date + relativedelta(months=self.duration)
        return '%s %s - %s' % (self.departement.name, self.ggd_no, until.isoformat())


from django.core.management.base import BaseCommand, CommandError
from puppetmanager.models import Node

class Command(BaseCommand):
    help = "Returns a puppet compadible external node classifier YAML output."

    def add_arguments(self, parser):
        parser.add_argument("node_name", nargs="+", type=str)

    def handle(self, *args, **options):
        try:
            if len(options["node_name"]) != 1:
                raise CommandError("only one node name is accepted")
        except KeyError:
            raise CommandError("node_name required")

        node_name = options["node_name"][0]
        try:
            node = Node.objects.get(name=node_name)
        except Node.DoesNotExist:
            node = Node(name=node_name)
            node.save()

        node.update()
        print(node.external_classify())


import logging
from PyQt5.QtWidgets import QPushButton, QHBoxLayout, QVBoxLayout, QDialog, QFileDialog, QLineEdit, QGridLayout
from PyQt5.QtGui import QPixmap
from subprocess import PIPE, Popen

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s ::  %(name)s :: %(funcName)s :: %(levelname)s :: %(message)s')
file_handler = logging.FileHandler("run.log")
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)

class ArtChanger(QDialog):
    def __init__(self, album, art):
        QDialog.__init__(self)
        self.album = album
        self.art = art
        logger.debug("Inside ArtChanger __init__")
        self.setWindowTitle("Change Album Art")
        self.btnWeb = QPushButton("From Web")
        self.btnLocal = QPushButton("From Local")
        self.btnCancel = QPushButton("Cancel")
        self.leImageBox = QLineEdit()
        self.leImageBox.setPlaceholderText("Enter image url")
        self.buttonLayout = QGridLayout()
        self.buttonLayout.addWidget(self.btnWeb, 0, 0)
        self.buttonLayout.addWidget(self.btnLocal, 0, 1)
        self.buttonLayout.addWidget(self.btnCancel, 0, 2)
        logger.debug("Layout created and elements added")
        self.btnWeb.clicked.connect(self.from_web)
        self.btnLocal.clicked.connect(self.from_local)
        self.btnCancel.clicked.connect(self.cancel_and_close)
        self.setLayout(self.buttonLayout)
        logger.debug("Buttons connected")

    def from_web(self):
        logger.debug("Inside ArtChanger from_web")
        self.buttonLayout.addWidget(self.leImageBox, 1, 0, 2, 0)

    def from_local(self):
        logger.debug("Inside ArtChanger from_local")
        filename, filter= QFileDialog.getOpenFileName(parent=self, caption='Open file', filter='*.jpg, *.jpeg, *.png')
        pixmap = QPixmap(filename)
        copy = Popen(['cp', filename, "/tmp/sample.jpg"], stdout=PIPE)
        copy.communicate()
        pixmap = pixmap.scaledToHeight(240)
        pixmap = pixmap.scaledToWidth(240)
        self.art.setPixmap(pixmap)

    def cancel_and_close(self):
        logger.debug(("Inside ArtChanger cancel_and_close"))
        self.close()


import numpy as np
import matplotlib.pyplot as plt

# load iris from numpy
iris = np.genfromtxt('iris.data', delimiter=',', dtype=None, encoding=None)
iris = np.array([list(x) for x in iris])

flower_features = ["sepalLength", "sepalWidth", "petalLength", "petalWidth"]
features = iris[:,0:4]
features = features.astype(np.float64)

# box chart for each feature for all the flower
data = []
for i in range(4):
    data.append(features[:,i])

# make the boxplot become colorful and pretty
plt.figure(figsize=(15,8))
plt.boxplot(data, labels=flower_features, patch_artist=True, medianprops={'linewidth':2})
plt.xlabel("Features")
plt.ylabel("cm")

plt.title("Boxplot of Iris Flower Features")
plt.show()