from django.shortcuts import render, redirect
from django.views.generic.base import View
from django.views.generic import ListView, DetailView, CreateView, UpdateView, DeleteView
from django.db.models import Q, Avg
from django.http import JsonResponse

from .models import Game, Genre, Category, Developer
from .forms import ReviewForm, RatingForm, Rating
# Create your views here.

class GenreYear:
    """Жанры и года выхода фильмов"""
    def get_genres(self):
        return Genre.objects.all()

    def get_years(self):
        return Game.objects.filter(draft=False).order_by('year').values("year").distinct()

class GamesView(GenreYear, ListView):
	"""Список игр"""
	model = Game
	queryset = Game.objects.filter(draft=False)
	template_name = "game/games.html"

	paginate_by = 10

class GameDetailView(GenreYear, DetailView):
    """Полное описание игры"""
    model = Game
    slug_field = "url"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context["star_form"] = RatingForm()
        return context

class GenreView(GenreYear, DetailView):
    model = Genre
    template_name = "games/genre.html"
    slug_field = "url"

class AddReview(View):
	"""Отзывы"""
	def post(self, request, pk):
		form = ReviewForm(request.POST)
		game = Game.objects.get(id=pk)
		if form.is_valid():
			form = form.save(commit=False)
			if request.POST.get("parent", None):
				form.parent_id = int(request.POST.get("parent"))
			form.game_id = pk
			form.save()

		return redirect(game.get_absolute_url())

class DeveloperView(GenreYear, DetailView):
	"""Полное описание разработчика или публ"""
	model = Developer
	template_name = 'games/developer.html'
	slug_field = "name"

class FilterGamesView(GenreYear, ListView):
    """Фильтр игр"""
    paginate_by = 6

    def get_queryset(self):
        queryset = Game.objects.filter(
            Q(year__in=self.request.GET.getlist("year")) |
            Q(genre__in=self.request.GET.getlist("genre"))
        ).distinct()
        return queryset

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        context["year"] = ''.join([f"year={x}&" for x in self.request.GET.getlist("year")])
        context["genre"] = ''.join([f"genre={x}&" for x in self.request.GET.getlist("genre")])
        return context

class AddStarRating(View):
    """Добавление рейтинга фильму"""
    def get_client_ip(self, request):
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        if x_forwarded_for:
            ip = x_forwarded_for.split(',')[0]
        else:
            ip = request.META.get('REMOTE_ADDR')
        return ip

    def post(self, request):
        form = RatingForm(request.POST)
        if form.is_valid():
            Rating.objects.update_or_create(
                ip=self.get_client_ip(request),
                game_id=int(request.POST.get("game")),
                user=request.POST.get("username"),
                defaults={'star_id': int(request.POST.get("star"))},

            )
            return HttpResponse(status=201)
        else:
            return HttpResponse(status=400)

class Search(ListView):
    """Поиск фильмов"""
    paginate_by = 4

    def get_queryset(self):
        return Game.objects.filter(title__icontains=self.request.GET.get("q"))

    def get_context_data(self, *args, **kwargs):
        context = super().get_context_data(*args, **kwargs)
        context["q"] = f'q={self.request.GET.get("q")}&'
        return context


# Solution to part 2 of day 12 of AOC 2015, JSAbacusFramework.io
# https://adventofcode.com/2015/day/12

import json

def sum_nums(d) -> int:
    """Return the sum of all integers in the parm dictionary."""
    if type(d) == int:
        return d

    elif type(d) == list:
        total = 0
        for i in d:
            total += sum_nums(i)
        return total

    elif type(d) == dict:
        total = 0
        for k in d:
            if type(d[k]) == str and d[k] == 'red':     # Special rule for Part 2.
                return 0
            total += sum_nums(d[k])
        return total

    else:
        return 0

assert sum_nums([1, 2, 3]) == 6
assert sum_nums([1,{"c":"red","b":2},3]) == 4
assert sum_nums({"d":"red","e":[1,2,3,4],"f":5}) == 0
assert sum_nums([1,"red",5]) == 6

f = open('input.txt')
whole_text = f.read()
f.close()

j = json.loads(whole_text)

print(sum_nums(j))


import uuid
from json import dumps, loads

from boto3 import client
from syndicate.commons.log_helper import get_logger
from syndicate.connection.helper import apply_methods_decorator, retry

_LOG = get_logger('syndicate.connection.sns_connection')

@apply_methods_decorator(retry)
class SNSConnection(object):
    def __init__(self, region=None, aws_access_key_id=None,
                 aws_secret_access_key=None, aws_session_token=None):
        self.client = client('sns', region,
                             aws_access_key_id=aws_access_key_id,
                             aws_secret_access_key=aws_secret_access_key,
                             aws_session_token=aws_session_token)
        _LOG.debug('Opened new SNS connection.')

    def create_topic(self, name):
        """ Crete SNS topic and return topic arn.

        :type name: str
        """
        return self.client.create_topic(Name=name)['TopicArn']

    def subscribe(self, endpoint, topic_name, protocol):
        """
        :param protocol:
            http -- delivery of JSON-encoded message via HTTP POST
            https -- delivery of JSON-encoded message via HTTPS POST
            email -- delivery of message via SMTP
            email-json -- delivery of JSON-encoded message via SMTP
            sms -- delivery of message via SMS
            sqs -- delivery of JSON-encoded message to an Amazon SQS queue
            application -- delivery of JSON-encoded message to an EndpointArn
                           for a mobile app and device.
            lambda -- delivery of JSON message to an AWS Lambda function
        :type protocol: str
        :type topic_name: str
        :param endpoint:
            http protocol, the endpoint is an URL beginning with "http://"
            https protocol, the endpoint is a URL beginning with "https://"
            email protocol, the endpoint is an email address
            email-json protocol, the endpoint is an email address
            sms protocol, the endpoint is a phone number of an SMS-enabled
                device
            sqs protocol, the endpoint is the ARN of an Amazon SQS queue
            application protocol, the endpoint is the EndpointArn of a mobile
                app and device.
            lambda protocol, the endpoint is the ARN of an AWS Lambda function
        :type endpoint: str
        """
        topic_arn = self.get_topic_arn(topic_name)
        if topic_arn is None:
            raise AssertionError(
                'Topic does not exist: {0}.'.format(topic_name))
        self.client.subscribe(TopicArn=topic_arn,
                              Protocol=protocol,
                              Endpoint=endpoint)
        return topic_arn

    def get_topic_arn(self, name):
        """ Get topic arn by name.

        :type name: str
        """
        topics = self.get_topics()
        for each in topics:
            if name == each['TopicArn'].split(':')[-1]:
                return each['TopicArn']

    def get_platform_application(self, name):
        """ Get application arn by name.

        :type name: str
        """
        applications = self.get_platform_applications()
        for each in applications:
            resolved_item = each['PlatformApplicationArn'].split(':')[-1]
            if name == resolved_item.split('/')[-1]:
                return each['PlatformApplicationArn']

    def is_user_subscribed(self, endpoint, topic_name):
        topic_arn = self.get_topic_arn(topic_name)
        subscriptions = self.client.list_subscriptions_by_topic(
            TopicArn=topic_arn)['Subscriptions']
        for each in subscriptions:
            if endpoint == each['Endpoint']:
                return True

    def publish_message(self, topic_name, message):
        topic_arn = self.get_topic_arn(topic_name)
        return self.client.publish(
            TargetArn=topic_arn,
            Message=message,
            MessageAttributes={
                'string': {
                    'DataType': 'String',
                    'StringValue': ' '
                }
            }
        )

    def get_topics(self):
        """ Get all topics."""
        topics = []
        response = self.client.list_topics()
        topics.extend(response.get('Topics'))
        token = response.get('NextToken')
        while token:
            response = self.client.list_topics(NextToken=token)
            topics.extend(response.get('Topics'))
            token = response.get('NextToken')
        return topics

    def get_platform_applications(self):
        """ Get all platform applications."""
        applications = []
        response = self.client.list_platform_applications()
        applications.extend(response.get('PlatformApplications'))
        token = response.get('NextToken')
        while token:
            response = self.client.list_platform_applications(NextToken=token)
            applications.extend(response.get('PlatformApplications'))
            token = response.get('NextToken')
        return applications

    def remove_topic_by_arn(self, topic_arn):
        """ Remove topic by arn.

        :type topic_arn: str
        """
        self.client.delete_topic(TopicArn=topic_arn)

    def remove_topic_by_name(self, topic_name):
        """ Remove topic by arn.

        :type topic_name: str
        """
        arn = self.get_topic_arn(topic_name)
        if arn:
            self.client.delete_topic(TopicArn=arn)

    def set_topic_attribute(self, topic_arn, attr_name, attr_value):
        self.client.set_topic_attributes(
            TopicArn=topic_arn,
            AttributeName=attr_name,
            AttributeValue=attr_value
        )

    def allow_service_invoke(self, topic_arn, service):
        existing_attr = self.get_topic_attributes(topic_arn)
        existing_policy = existing_attr['Attributes']['Policy']
        existing_policy_dict = loads(existing_policy)
        policy = {
            "Sid": str(uuid.uuid1()),
            "Effect": "Allow",
            "Principal":
                {
                    "Service": "{0}".format(service)
                },
            "Action": "sns:Publish",
            "Resource": "{0}".format(topic_arn)
        }
        existing_policy_dict['Statement'].append(policy)
        self.set_topic_attribute(topic_arn, 'Policy',
                                 dumps(existing_policy_dict))

    def get_topic_attributes(self, topic_arn):
        return self.client.get_topic_attributes(
            TopicArn=topic_arn
        )

    def get_platform_application_attributes(self, application_arn):
        return self.client.get_platform_application_attributes(
            PlatformApplicationArn=application_arn
        )

    def add_account_permission(self, topic_arn, account_id, action, label):
        if isinstance(account_id, str):
            account_id = [account_id]
        if not isinstance(account_id, list):
            raise AssertionError('Incorrect account id {0}'.format(account_id))

        if isinstance(action, str):
            action = [action]
        if not isinstance(action, list):
            raise AssertionError('Incorrect action {0}'.format(action))

        self.client.add_permission(TopicArn=topic_arn, Label=label,
                                   AWSAccountId=account_id,
                                   ActionName=action)

    def revoke_account_permission(self, topic_arn, label):
        self.client.remove_permission(TopicArn=topic_arn, Label=label)

    def list_subscriptions_by_topic(self, topic_arn):
        subscriptions = []
        response = self.client.list_subscriptions_by_topic(TopicArn=topic_arn)
        subscriptions.extend(response.get('Subscriptions'))
        token = response.get('NextToken')
        while token:
            response = self.client.list_subscriptions_by_topic(
                TopicArn=topic_arn, NextToken=token)
            subscriptions.extend(response.get('Subscriptions'))
            token = response.get('NextToken')

        return subscriptions

    def unsubscribe(self, subscription_arn):
        self.client.unsubscribe(SubscriptionArn=subscription_arn)

    def create_platform_endpoint(self, platform_application_arn, token):
        response = self.client.create_platform_endpoint(
            PlatformApplicationArn=platform_application_arn,
            Token=token
        )
        return response.get('EndpointArn')

    def create_platform_application(self, name, platform, attributes):
        response = self.client.create_platform_application(
            Name=name, Platform=platform, Attributes=attributes)
        return response.get('PlatformApplicationArn')

    def remove_application_by_arn(self, application_arn):
        """ Remove application by arn.

        :type application_arn: str
        """
        self.client.delete_platform_application(
            PlatformApplicationArn=application_arn)


import json
from typing import Any, Dict, Iterable, List, Optional

from pyspark.sql.types import Row

from spark_pipeline_framework.utilities.api_helper.http_request import (
    HelixHttpRequest,
    RequestType,
)

class HttpDataSenderProcessor:
    @staticmethod
    # function that is called for each partition
    # noinspection PyUnusedLocal
    def send_partition_to_server(
        *,
        partition_index: int,
        rows: Iterable[Row],
        url: Optional[str],
        content_type: str,
        headers: Dict[str, Any],
        post_as_json_formatted_string: Optional[bool],
        parse_response_as_json: Optional[bool],
    ) -> Iterable[Row]:
        """
        This function processes a partition

        This has to be a static function to avoid creating a closure around a class
        https://spark.apache.org/docs/latest/rdd-programming-guide.html#passing-functions-to-spark

        """
        json_data_list: List[Dict[str, Any]] = [r.asDict(recursive=True) for r in rows]
        print(
            f"----- Sending batch {partition_index} containing {len(json_data_list)} rows -----"
        )

        # logger = get_logger(__name__)
        if len(json_data_list) == 0:
            yield Row(
                url=None,
                status=0,
                result=None,
                headers=None,
                payload=None,
                request_type=None,
            )

        assert url
        json_data: Dict[str, Any]
        for json_data in json_data_list:
            headers["Content-Type"] = content_type
            request: HelixHttpRequest = HelixHttpRequest(
                request_type=RequestType.POST,
                url=url,
                headers=headers,
                payload=json_data,
                post_as_json_formatted_string=post_as_json_formatted_string,
            )
            if parse_response_as_json:
                response_json = request.get_result()
                yield Row(
                    url=url,
                    status=response_json.status,
                    result=response_json.result,
                    headers=json.dumps(headers, default=str),
                    payload=json.dumps(json_data, default=str),
                    request_type=str(RequestType.POST),
                )
            else:
                response_text = request.get_text()
                yield Row(
                    url=url,
                    status=response_text.status,
                    result=response_text.result,
                    headers=json.dumps(headers, default=str),
                    payload=json.dumps(json_data, default=str),
                    request_type=str(RequestType.POST),
                )


import numpy as np

import bpy
from bpy.props import FloatProperty, EnumProperty, BoolProperty, IntProperty
from mathutils import Matrix

import sverchok
from sverchok.node_tree import SverchCustomTreeNode
from sverchok.data_structure import updateNode, has_element
from sverchok.utils.logging import info, exception
from sverchok.utils.nodes_mixins.recursive_nodes import SvRecursiveNode

import numpy as np
from sverchok_open3d.dependencies import open3d as o3d
from sverchok.utils.dummy_nodes import add_dummy
from sverchok_open3d.utils.point_cloud import calc_point_cloud_normals
if o3d is None:
    add_dummy('SvO3PointCloudInNode', 'Point Cloud In', 'open3d')
else:
    class SvO3PointCloudInNode(bpy.types.Node, SverchCustomTreeNode, SvRecursiveNode):
        """
        Triggers: Point Cloud In
        Tooltip: Point Cloud In
        """
        bl_idname = 'SvO3PointCloudInNode'
        bl_label = 'Point Cloud In'
        bl_icon = 'OUTLINER_OB_POINTCLOUD'
        sv_icon = 'SV_O3_POINT_CLOUD_IN'
        methods = [
            ('STANDARD', "Standard", "Standard", 0),
            ('TANGENT', "Tangent Plane", "Better results with more processing time", 1),
            ('NONE', "None", "Do not calculate them", 2),
        ]
        normal_method: EnumProperty(
            name="Normal Method",
            description='Method used to calculate normals if not supplied',
            items=methods,
            default='STANDARD',
            update=updateNode)
        normal_quality: IntProperty(
            name="Normal Quality",
            default=30,
            min=1,
            update=updateNode)
        def sv_init(self, context):
            self.width = 200
            self.inputs.new('SvVerticesSocket', "Vertices").is_mandatory = True

            self.inputs.new('SvVerticesSocket', "Normals").nesting_level = 3
            self.inputs.new('SvColorSocket', "Colors").nesting_level = 3

            self.outputs.new('SvO3PointCloudSocket', 'O3D Point Cloud')

        def draw_buttons(self, context, layout):
            if not self.inputs['Normals'].is_linked:
                layout.prop(self, 'normal_method')
                layout.prop(self, 'normal_quality')

        def process_data(self, params):

            point_clouds_out = []
            for vertices, normals, colors in zip(*params):
                pcd = o3d.geometry.PointCloud()
                pcd.points = o3d.utility.Vector3dVector(np.array(vertices))
                if has_element(normals):
                    pcd.normals = o3d.utility.Vector3dVector(np.array(normals))
                elif self.normal_method != 'NONE':
                    calc_point_cloud_normals(pcd, self.normal_quality, self.normal_method)

                if has_element(colors):
                    pcd.colors = o3d.utility.Vector3dVector(np.array(colors)[:,:3])
                point_clouds_out.append(pcd)

            return point_clouds_out

def register():
    if o3d is not None:
        bpy.utils.register_class(SvO3PointCloudInNode)

def unregister():
    if o3d is not None:
        bpy.utils.unregister_class(SvO3PointCloudInNode)


import numpy as np
import torch

def cost_matrix_batch_torch(x, y):
	"Returns the cosine distance batchwise"
	# x is the image feature: bs * d * m * m
	# y is the audio feature: bs * d * nF
	# return: bs * n * m
	# print(x.size())
	bs = list(x.size())[0]
	D = x.size(1)
	assert(x.size(1)==y.size(1))
	# x = x.contiguous().view(bs, D, -1) # bs * d * m^2
	x = x.div(torch.norm(x, p=2, dim=1, keepdim=True) + 1e-12)
	y = y.div(torch.norm(y, p=2, dim=1, keepdim=True) + 1e-12)
	cos_dis = torch.bmm(torch.transpose(x, 1, 2), y)#.transpose(1,2)
	cos_dis = 1 - cos_dis # to minimize this value
	# cos_dis = - cos_dis
	return cos_dis.transpose(2,1)

def cos_batch_torch(x, y):
	"Returns the cosine distance batchwise"
	# x is the image feature: bs * d * m * m
	# y is the audio feature: bs * d * nF
	# return: bs * n * m
	# print(x.size())
	bs = x.size(0)
	D = x.size(1)
	assert(x.size(1)==y.size(1))
	x = x.contiguous().view(bs, D, -1) # bs * d * m^2
	x = x.div(torch.norm(x, p=2, dim=1, keepdim=True) + 1e-12)
	y = y.div(torch.norm(y, p=2, dim=1, keepdim=True) + 1e-12)
	cos_dis = torch.bmm(torch.transpose(x,1,2), y)#.transpose(1,2)
	cos_dis = 1 - cos_dis # to minimize this value
	# return cos_dis.transpose(2,1)
	# TODO:
	beta = 0.1
	min_score = cos_dis.min()
	max_score = cos_dis.max()
	threshold = min_score + beta * (max_score - min_score)
	res = cos_dis - threshold
	# res = torch.nn.ReLU()

	return torch.nn.functional.relu(res.transpose(2,1))

def pairwise_distances(x, y=None):
	'''
	Input: x is a Nxd matrix
		   y is an optional Mxd matirx
	Output: dist is a NxM matrix where dist[i,j] is the square norm between x[i,:] and y[j,:]
			if y is not given then use 'y=x'.
	i.e. dist[i,j] = ||x[i,:]-y[j,:]||^2
	'''
	x_norm = (x ** 2).sum(1).view(-1, 1)
	if y is not None:
		y_t = torch.transpose(y, 0, 1)
		y_norm = (y ** 2).sum(1).view(1, -1)
	else:
		y_t = torch.transpose(x, 0, 1)
		y_norm = x_norm.view(1, -1)

	dist = x_norm + y_norm - 2.0 * torch.mm(x, y_t)
	# Ensure diagonal is zero if x=y
	# if y is None:
	#     dist = dist - torch.diag(dist.diag)
	return torch.clamp(dist, 0.0, np.inf)

def IPOT_distance_torch_batch_uniform(C, bs, n, m, iteration=50):
	C = C.float().cuda()
	T = IPOT_torch_batch_uniform(C  , bs, n, m, iteration=iteration)
	temp = torch.bmm(torch.transpose(C,1,2), T)
	distance = batch_trace(temp, m, bs)
	return -distance

def IPOT_torch_batch_uniform(C, bs, n, m, beta=0.5, iteration=50):
	# C is the distance matrix
	# c: bs by n by m
	sigma = torch.ones(bs, int(m), 1).cuda()/float(m)
	T = torch.ones(bs, n, m).cuda()
	A = torch.exp(-C/beta).float().cuda()
	for t in range(1):
		Q = A * T # bs * n * m
		del T
		for k in range(iteration):
			delta = 1 / (n * torch.bmm(Q, sigma))
			a = torch.bmm(torch.transpose(Q,1,2), delta)
			sigma = 1 / (float(m) * a)
		T = delta * Q * sigma.transpose(2,1)
		del Q

	return T#.detach()

def GW_distance(X, Y, p, q, lamda=0.5, iteration=5, OT_iteration=20, **kwargs):
	'''
	:param X, Y: Source and target embeddings , batchsize by embed_dim by n
	:param p, q: probability vectors
	:param lamda: regularization
	:return: GW distance
	'''
	Cs = cos_batch_torch(X, X).float().cuda()
	Ct = cos_batch_torch(Y, Y).float().cuda()

	# pdb.set_trace()
	bs = Cs.size(0)
	m = Ct.size(2)
	n = Cs.size(2)
	T, Cst = GW_torch_batch(Cs, Ct, bs, n, m, p, q, beta=lamda, iteration=iteration, OT_iteration=OT_iteration)
	temp = torch.bmm(torch.transpose(Cst,1,2), T)
	distance = batch_trace(temp, m, bs)
	return distance

def GW_torch_batch(Cs, Ct, bs, n, m, p, q, beta=0.5, iteration=5, OT_iteration=20):
	one_m = torch.ones(bs, m, 1).float().cuda()
	one_n = torch.ones(bs, n, 1).float().cuda()

	Cst = torch.bmm(torch.bmm(Cs**2, p), torch.transpose(one_m, 1, 2)) + \
	      torch.bmm(one_n, torch.bmm(torch.transpose(q,1,2), torch.transpose(Ct**2, 1, 2))) # bs by n by m
	gamma = torch.bmm(p, q.transpose(2,1)) # outer product, init
	# gamma = torch.einsum('bi,bj->bij', (torch.squeeze(p), torch.squeeze(q))) # outer product, initialization
	for i in range(iteration):
		C_gamma = Cst - 2 * torch.bmm(torch.bmm(Cs, gamma), torch.transpose(Ct, 1, 2))
		# # Sinkhorn iteration
		# b = torch.ones(bs, m, 1).cuda()
		# K = torch.exp(-C_gamma/beta)
		# for i in range(50):cd
		# 	a = p/(torch.bmm(K, b))
		# 	b = q/torch.bmm(K.transpose(1,2), a)
		# gamma = a * K * b
		gamma = IPOT_torch_batch_uniform(C_gamma, bs, n, m, beta=beta, iteration=OT_iteration)
	Cgamma = Cst - 2 * torch.bmm(torch.bmm(Cs, gamma), torch.transpose(Ct, 1, 2))
	return gamma.detach(), Cgamma

def GW_distance_uniform(X, Y, lamda=1e-1, iteration=5, OT_iteration=20, **kwargs):
	m = X.size(2)
	n = Y.size(2)
	bs = X.size(0)
	p = (torch.ones(bs, m, 1)/m).cuda()
	q = (torch.ones(bs, n, 1)/n).cuda()
	return GW_distance(X, Y, p, q, lamda=lamda, iteration=iteration, OT_iteration=OT_iteration, **kwargs)

def batch_trace(input_matrix, n, bs):
	a = torch.eye(n).cuda().unsqueeze(0).repeat(bs, 1, 1)
	b = a * input_matrix
	return torch.sum(torch.sum(b,-1),-1).unsqueeze(1)


import asyncio
import logging
from decimal import Decimal

from tinkoff.invest import OrderState

from configuration.settings import BlogSettings, StrategySettings
from invest_api.utils import moneyvalue_to_decimal
from trade_system.signal import SignalType
from trade_system.strategies.base_strategy import IStrategy
from trading.trade_results import TradeOrder

__all__ = ("Blogger")

logger = logging.getLogger(__name__)

class Blogger:
    """
    Class formats and sends messages to telegram chat.
    """
    def __init__(
            self,
            blog_settings: BlogSettings,
            trade_strategies: list[StrategySettings],
            messages_queue: asyncio.Queue
    ) -> None:
        self.__blog_status = blog_settings.blog_status
        self.__trade_strategies: dict[str, StrategySettings] = {x.figi: x for x in trade_strategies}
        self.__messages_queue = messages_queue

    def __send_text_message(self, text: str) -> None:
        try:
            logger.debug(f"Put message to telegram messages queue: {text}")

            self.__messages_queue.put_nowait(text)
        except Exception as ex:
            logger.error(f"Error put message to telegram messages queue: {repr(ex)}")

    def start_trading_message(
            self,
            today_trade_strategy: dict[str, IStrategy],
            rub_before_trade_day: Decimal
    ) -> None:
        """
        The method sends message about depo size, list of stocks for today trading and greetings also.
        """
        if self.__blog_status:
            self.__send_text_message("Greetings! We are starting.")
            self.__send_text_message(f"Depo size: {rub_before_trade_day:.2f} rub")
            self.__send_text_message("Stocks list:")
            for figi_key, strategy_value in today_trade_strategy.items():
                self.__send_text_message(
                    f"Ticker: {strategy_value.settings.ticker}. "
                    f"Short trade status: {strategy_value.settings.short_enabled_flag}"
                )

    def finish_trading_message(self) -> None:
        """
        The method sends information that trading is stopping.
        """
        if self.__blog_status:
            self.__send_text_message("We are closing trading day.")

    def close_position_message(self, trade_order: TradeOrder) -> None:
        """
        The method sends information about closed position.
        """
        if self.__blog_status and trade_order:
            signal_type = Blogger.__signal_type_to_message_test(trade_order.signal.signal_type)
            self.__send_text_message(
                f"{self.__trade_strategies[trade_order.signal.figi].ticker} position {signal_type} has been closed."
            )

    def open_position_message(self, trade_order: TradeOrder) -> None:
        """
        The method sends information about opened position.
        """
        if self.__blog_status and trade_order:
            signal_type = Blogger.__signal_type_to_message_test(trade_order.signal.signal_type)
            self.__send_text_message(
                f"{self.__trade_strategies[trade_order.signal.figi].ticker} position {signal_type} has been opened. "
                f"Take profit level: {trade_order.signal.take_profit_level:.2f}. "
                f"Stop loss level: {trade_order.signal.stop_loss_level:.2f}."
            )

    def trading_depo_summary_message(
            self,
            rub_before_trade_day: Decimal,
            current_rub_on_depo: Decimal
    ) -> None:
        """
        The method sends information about trading day summary.
        """
        if self.__blog_status:
            self.__send_text_message(
                f"Start depo: {rub_before_trade_day:.2f} close depo:{current_rub_on_depo:.2f}."
            )

            today_profit = current_rub_on_depo - rub_before_trade_day
            today_percent_profit = (today_profit / rub_before_trade_day) * 100
            self.__send_text_message(f"Today leverage: {today_profit:.2f} rub ({today_percent_profit:.2f} %)")

    def fail_message(self):
        """
        The method sends information about emergency situation in bot.
        """
        if self.__blog_status:
            self.__send_text_message(
                f"Something went wrong. We are trying to close all positions. "
                f"If we fail, please try to do it himself."
            )

    def summary_message(self):
        """
        The method sends just summary title.
        """
        if self.__blog_status:
            self.__send_text_message(f"Trading day summary:")

    def final_message(self):
        """
        The method sends just goodbye title.
        """
        if self.__blog_status:
            self.__send_text_message(f"Trading has been completed. See you on next trade day!")

    def summary_open_signal_message(self, trade_order: TradeOrder, open_order_state: OrderState):
        """
        The method sends summary information about only open positions (not closed)
        """
        if self.__blog_status:
            signal_type = Blogger.__signal_type_to_message_test(trade_order.signal.signal_type)
            summary_commission = moneyvalue_to_decimal(open_order_state.executed_commission) + \
                                 moneyvalue_to_decimal(open_order_state.service_commission)
            self.__send_text_message(
                f"Open {signal_type} position for {self.__trade_strategies[trade_order.signal.figi].ticker}. "
                f"Lots executed: {open_order_state.lots_executed}. "
                f"Average price: "
                f"{moneyvalue_to_decimal(open_order_state.average_position_price):.2f}. "
                f"Total order price: "
                f"{moneyvalue_to_decimal(open_order_state.total_order_amount):.2f}. "
                f"Total commissions: "
                f"{summary_commission:.2f}. "
                f"You have to close position manually."
            )

    def summary_closed_signal_message(self,
                                      trade_order: TradeOrder,
                                      open_order_state: OrderState,
                                      close_order_state: OrderState
                                      ) -> None:
        """
        The method sends summary information about closed positions
        """
        if self.__blog_status:
            signal_type = Blogger.__signal_type_to_message_test(trade_order.signal.signal_type)
            summary_commission = moneyvalue_to_decimal(open_order_state.executed_commission) + \
                                 moneyvalue_to_decimal(open_order_state.service_commission) + \
                                 moneyvalue_to_decimal(close_order_state.executed_commission) + \
                                 moneyvalue_to_decimal(close_order_state.service_commission)
            self.__send_text_message(
                f"Close {signal_type} position for {self.__trade_strategies[trade_order.signal.figi].ticker}. "
                f"Lots executed: {close_order_state.lots_executed}. "
                f"Average open price: "
                f"{moneyvalue_to_decimal(open_order_state.average_position_price):.2f}. "
                f"Average close price: "
                f"{moneyvalue_to_decimal(close_order_state.average_position_price):.2f}. "
                f"Summary: "
                f"{moneyvalue_to_decimal(close_order_state.total_order_amount) - moneyvalue_to_decimal(open_order_state.total_order_amount):.2f}. "
                f"Total commissions: "
                f"{summary_commission:.2f}."
            )

    @staticmethod
    def __signal_type_to_message_test(signal_type: SignalType) -> str:
        return "long" if signal_type == SignalType.LONG else "short"


import time
import logging
import re

from anime_downloader.sites.anime import Anime, AnimeEpisode, SearchResult
from anime_downloader.sites import helpers

logger = logging.getLogger(__name__)

class NineAnime(Anime, sitename='nineanime'):
        sitename = '9anime'
        url = f'https://{sitename}.to/search'
        @classmethod
        def search(cls, query):
            # Only uses the first page of search results, but it's sufficent.
            search_results = helpers.soupify(helpers.get(cls.url, params={'keyword': query})).select('a.name')
            return [
                SearchResult(
                    title = i.text,
                    url = i.get('href'),
                    meta_info = {
                        'version_key_dubbed':'(Dub)',
                        'version_key_subbed':''
                        }
                    )
                for i in search_results
            ]

        def _scrape_episodes(self):
            soup = helpers.soupify(helpers.get(self.url))
            # Assumptions can cause errors, but if this fails it's better to get issues on github.
            title_id = soup.select("div#player")[0]
            title_id = title_id.get('data-id')
            episode_html = helpers.get(f"https://9anime.to/ajax/film/servers?id={title_id}").text
            # Only using streamtape, MyCloud can get added, but it uses m3u8.
            streamtape_regex = r'data-id=\\"40\\".*?(data-name|$)'
            streamtape_episodes = re.search(streamtape_regex, episode_html)
            if not streamtape_episodes:
                logger.error('Unable to find streamtape server')
                return ['']

            streamtape_episodes = streamtape_episodes.group()
            # You can use helpers.soupify on all this, but it's resource intensive, unreliable
            # and can give recursion errors on large series like naruto.

            # Group 3 is the actual id.
            # Matches stuff like: id=\"a9cfdbd2029a467d2b2c9f156cbedc02b25a23199812ad4dbe5a25afa9edb140\"
            episode_regex = r'id=(\\|)(\'|")(.{64}?)(\\|)(\'|")'
            episodes = re.findall(episode_regex, streamtape_episodes)
            if not episodes:
                logger.error('Unable to find any episodes')
                return ['']

            # Returns an ID instead of actual URL
            return [i[2] for i in episodes]

        def _scrape_metadata(self):
            self.title = helpers.soupify(helpers.get(self.url)).select('h1.title')[0].text

class NineAnimeEpisode(AnimeEpisode, sitename='9anime'):
        def _get_sources(self):
            if not self.url:
                return ''

            # Arbitrary timeout to prevent spamming the server which will result in an error.
            time.sleep(0.3)
            # Server 40 is streamtape, change this if you want to add other servers
            episode_ajax = f"https://9anime.to/ajax/episode/info?id={self.url}&server=40"
            target = helpers.get(episode_ajax).json().get('target','')
            logger.debug('Videolink: {}'.format(target))
            if not target:
                return ''

            videolink = helpers.soupify(helpers.get(target)).select('div#videolink')
            logger.debug('Videolink: {}'.format(videolink))
            if not videolink:
                return ''

            videolink = videolink[0].text

            # Appends https
            videolink = 'https:' + videolink if videolink.startswith('//') else videolink
            return [('no_extractor', videolink,)]


# ###
# Mathematical expression recognition tool.
# Written as a part of masters thesis at VUT FIT Brno, 2022

# Author: Vladislav Halva
# Login: xhalva04
# ###

from torch import nn

from src.model.Decoder import Decoder
from src.model.Encoder import Encoder

class Model(nn.Module):
    """
    Encoder-decoder model for mathematical expressions recognition wrapper.
    """
    def __init__(
            self, device, edge_features, edge_h_size,
            enc_in_size, enc_h_size, enc_out_size, dec_in_size, dec_h_size, emb_size, dec_att_size,
            vocab_size, end_node_token_id, tokenizer,
            enc_vgg_dropout_p, enc_gat_dropout_p, dec_emb_dropout_p, dec_att_dropout_p):
        """
        :param device: device
        :param edge_features: in edge features size
        :param edge_h_size: hidden and out edge features size
        :param enc_in_size: encoder GATs in nodes features size
        :param enc_h_size: encoder GATs hidden nodes features size
        :param enc_out_size: encoder GATs out nodes features size
        :param dec_in_size: decoder in node features size
        :param dec_h_size: decoder hidden node features size
        :param emb_size: decoder out node features=embeddings size
        :param dec_att_size: decoder source graph attention vector size
        :param vocab_size: vocabulary size
        :param end_node_token_id: id of end-leaf-node token [EOS]
        :param tokenizer: tokenizer
        :param enc_vgg_dropout_p: dropout probability for encoder VGG linear layers
        :param enc_gat_dropout_p: dropout probability for encoder GAT layers
        :param dec_emb_dropout_p: dropout probability for decoder initial embedding layer
        :param dec_att_dropout_p: dropout probability for decoder attention to source graph
        """
        super(Model, self).__init__()

        self.encoder = Encoder(
            edge_features, edge_h_size,
            enc_in_size, enc_h_size, enc_out_size, vocab_size,
            enc_vgg_dropout_p, enc_gat_dropout_p)
        self.decoder = Decoder(
            device, enc_out_size, dec_in_size, dec_h_size, emb_size,
            dec_att_size, vocab_size, end_node_token_id,
            tokenizer, dec_emb_dropout_p, dec_att_dropout_p)

        self.lin_x_out = nn.Linear(enc_out_size, vocab_size, bias=False)

    def forward(self, data, beam_search=True, beam_width=3):
        """
        :param data: databatch object
        :param beam_search: True if beam search, False otherwise
        :param beam_width: if beam search sets the beam width - number of simultaneously generated output graphs
        :return: output data
        """
        # encode source graph
        x, edge_index, edge_attr, x_conv_score = self.encoder(data.x, data.edge_index, data.edge_attr)
        # decode output graph
        y, y_batch, y_edge_index, y_edge_type, y_score, y_edge_rel_score, gcn1_alpha, gcn2_alpha, gcn3_alpha, = self.decoder(
            x, data.x_batch, data.tgt_y, data.tgt_edge_index, data.tgt_edge_type, data.tgt_y_batch, beam_search, beam_width)
        # get source graph node predictions and update values in databatch object for source graph
        data.x = x
        data.x_score = self.lin_x_out(x)
        data.edge_index = edge_index
        data.edge_attr = edge_attr
        data.x_conv_score = x_conv_score
        # include output graph in output databatch object
        data.y = y
        data.y_batch = y_batch
        data.y_edge_index = y_edge_index
        data.y_edge_type = y_edge_type
        data.y_score = y_score
        data.y_edge_rel_score = y_edge_rel_score
        data.gcn1_alpha = gcn1_alpha
        data.gcn2_alpha = gcn2_alpha
        data.gcn3_alpha = gcn3_alpha
        return data


#!/usr/bin/env python3
"""Simple pagination"""
import csv
import math
from typing import List, Tuple

class Server:
    """Server class to paginate a database of popular baby names.
    """
    DATA_FILE = "Popular_Baby_Names.csv"

    def __init__(self):
        self.__dataset = None

    def dataset(self) -> List[List]:
        """Cached dataset
        """
        if self.__dataset is None:
            with open(self.DATA_FILE) as f:
                reader = csv.reader(f)
                dataset = [row for row in reader]
            self.__dataset = dataset[1:]

        return self.__dataset

    def get_page(self, page: int = 1, page_size: int = 10) -> List[List]:
        """return list"""
        assert type(page) is int
        assert type(page_size) is int
        assert page > 0
        assert page > page_size
        self.dataset()
        a = index_range(page, page_size)
        if a[0] >= len(self.__dataset):
            return []
        else:
            return self.__dataset[a[0]:a[1]]

def index_range(page: int, page_size: int) -> Tuple[int, int]:
    """return a tuple of size two containing a start index and an end index"""
    total_size = page * page_size
    page_n = total_size - page_size
    return (page_n, page)


import json
from tabulate import tabulate
from functools import reduce
# load full phonem table from json file
with open('vowelDict.json') as f:
    vowelDict = json.load(f)

def unpack(nestedDict: dict):
    # base case: dict does not contain dicts
    if not isinstance(nestedDict[next(iter(nestedDict))], dict):
        return [ nestedDict[key] for key in nestedDict ]
    # recursive case: nestedDict is in fact a nested dict
    else:
        items = []
        for key in nestedDict:
            items += unpack(nestedDict[key])
        return items

def getKeys(nestedDict: dict):
    if isinstance(nestedDict, dict):
        keys = [ key for key in nestedDict ]
        nestedKeys = getKeys(nestedDict[keys[0]])
        if nestedKeys:
            return [ keys ] + nestedKeys
        else:
            return [ keys ]

def getHeader(headerKeys: list):
    # get the total number of columns/titles in the header
    titleCount = reduce((lambda x, y: x * y), [len(keyList) for keyList in headerKeys] )
    header = []
    for i in range(titleCount):
        title = []
        for j in range(len(headerKeys)):
            # add the titles in the right order
            titleSubsetProduct = reduce((lambda x, y: x * y), [len(keyList) for keyList in headerKeys[:j+1]] )
            title.append(headerKeys[j][ int(i*titleSubsetProduct/titleCount) % len(headerKeys[j])])
        header.append('\n'.join(title))
    return header

vowelList = unpack(vowelDict)
keyList = getKeys(vowelDict)
header = getHeader(keyList[1:])
# print(vowelList)
# print()
# print(keyList)
# print()
# print(header)
firstColumn = keyList[0]
vowelMatrix = [ [firstColumn[int(i/len(header))]] + vowelList[i:i+len(header)] for i in range(0, len(vowelList), len(header))]
# print(vowelMatrix)

# for key in vowelDict:
#     row = vowelDict[key]
#     rowList = []
#     for key in row:
#         col = row[key]
#         for key in col:
#             type = col[key]
#             for key in type:
#                 symbol = type[key]
#                 rowList.append(symbol)
#     vowelList.append(rowList)

print(tabulate(vowelMatrix, headers=header, stralign='center'))


import sys
import typing
from PyQt5.QtWidgets import QWidget, QApplication, QLineEdit, QPushButton, QLabel, QPlainTextEdit

class Lz78Code:
    def __init__(self, i, c):
        self.i = i
        self.c = c

    def __str__(self) -> str:
        return f"<{self.i}, '{self.c}'>"

def encode(input_str: str) -> typing.List[Lz78Code]:
    ret = []
    prev_sub = ""
    sub_dict = {"": 0}
    for c in input_str:
        next_sub = prev_sub + c
        if next_sub not in sub_dict:
            i = sub_dict[prev_sub]
            output = Lz78Code(i, c)
            sub_dict[next_sub] = len(sub_dict)
            print(f"'{next_sub}'\t{str(output)}\t{len(sub_dict) - 1}")
            ret.append(f"'{next_sub}'\t{str(output)}\t{len(sub_dict) - 1}")
            prev_sub = ""
        else:
            prev_sub = next_sub
    return ret

def decode(input_ar: typing.List[Lz78Code]) -> str:
    ret = ""
    sub_dict = [""]
    for output in input_ar:
        entry = sub_dict[output.i] + output.c
        sub_dict.append(entry)
        ret += entry
    return ret

class Windows(QWidget):
    def __init__(self):
        super().__init__()
        self.addProperties()
        self.addProperties1()
        self.buton.clicked.connect(self.click)

    def addProperties(self):
        self.resize(400, 400)
        self.move(700, 100)
        self.setWindowTitle('LZ78 Algorithm')

    def addProperties1(self):
        self.label = QLabel("Enter Characters", self)
        self.label.setGeometry(100, 5, 200, 75)

        self.input = QLineEdit(self)
        self.input.setGeometry(100, 60, 200, 25)

        self.buton = QPushButton("Run", self)
        self.buton.setGeometry(100, 100, 200, 25)

        self.label = QLabel("Result:", self)
        self.label.setGeometry(100, 120, 200, 75)

        self.label = QPlainTextEdit(self)
        self.label.setGeometry(75, 170, 250, 190)

    def click(self):
        result = ""
        for i in encode(self.input.text()):
            result += f" {i} \n"
        self.label.clear()
        self.label.insertPlainText(result)

app = QApplication(sys.argv)
form = Windows()
form.show()
sys.exit(app.exec_())


import cv2
import numpy as np
from tensorflow.keras.models import load_model
cap = cv2.VideoCapture(0)
from training_data.model import anti_spoof_face
from processing_data.cropping_face import extract_face

# model_path = 'models/anti_spoof_128_1677228839.5275097.h5'

model_path = 'checkpoint/spoof_face_classification.hdf5'

model = anti_spoof_face()
model.load_weights(model_path)
# model = load_model(model_path)
def pre_processing(image):
    image = cv2.imread(image)
    face = []
    x_total = []
    y_total = []
    w_total = []
    h_total = []
    bbox, confi = extract_face(image)
    for i in bbox:
        x = i[0]
        y = i[1]
        w = i[2]
        h = i[3]
        face_image = image[i[1]: i[3], i[0]: i[2]]
        # face_image = face_image[:,:,::-1]
        face_image = cv2.resize(image, (224,224))
        result_img = np.expand_dims(face_image, axis=0)
        face.append(result_img)
        x_total.append(x)
        y_total.append(y)
        w_total.append(w)
        h_total.append(h)
    return face, x_total[0], y_total[0],w_total[0],h_total[0]

while True:
    ret, frame = cap.read()
    bbox, confi = extract_face(frame)
    for i in bbox:
        facial_img = frame[i[1]: i[3], i[0]: i[2]]
        try:
            face_image = cv2.resize(facial_img, (224,224))
        except:
            continue
        face_image = np.expand_dims(face_image, axis=0)
        result = model.predict(face_image)[0]
        label = np.argmax(result)
        confidence = np.max(result)
        if label == 0:
            labels = 'real'
            color = (0,0,255)
        else:
            labels = 'spoof'
            color = (0,255,0)
        cv2.putText(frame, labels, (i[0], i[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.8, color, 2)
        cv2.rectangle(frame, (i[0],i[1]), (i[2], i[3]), (0,255,255), 1)
        cv2.imshow('a', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
cap.release()
cv2.destroyAllWindows()


import readline
import time
import shlex
import queue
import threading
import sys
import socket
from utils.output import Output
from server.payload_manager import PayloadManager

class Prompt:

    def __init__(self, listener):
        self.listener = listener
        self.state = ('base',)

        self.recv_t = threading.Thread(target=self.receiver_thread)
        self.recv_t.daemon = True
        self.recv_t.start()

    def start(self):
        while True:
            command = input(self.prompt())
            cmd = shlex.split(command)

            if self.state[0] == 'base':
                if len(cmd) < 1:
                    continue

                try:

                    if cmd[0] == 'list':
                        for conn_id, conn in self.listener.connection_dict.items():
                            print(" - %d: %s" % (conn_id, conn['client']))

                    elif cmd[0] == 'use':
                        conn_id = int(cmd[1])

                        if not conn_id in self.listener.connection_dict:
                            print("\033[91mUnable to find connection with ID %d\033[0m" % conn_id)
                            continue

                        self.state = ('interact', conn_id)

                    elif cmd[0] == 'kill':
                        conn_id = int(cmd[1])

                        if not conn_id in self.listener.connection_dict:
                            print("\033[91mUnable to find connection with ID %d\033[0m" % conn_id)
                            continue

                        self.listener.connection_dict[conn_id]['conn'].shutdown(socket.SHUT_RDWR)
                        self.listener.connection_dict[conn_id]['conn'].close()

                    elif cmd[0] == 'payloads':
                        print("\033[92mWindows payloads: \033[0m")
                        modules = PayloadManager.list_payloads('cmd')
                        for name, module in modules.items():
                            print(" - %s %s" % (module.name, ' '.join(['<%s>' % s for s in module.args])))
                        print("\033[92mLinux payloads: \033[0m")
                        modules = PayloadManager.list_payloads('sh')
                        for name, module in modules.items():
                            print(" - %s %s" % (module.name, ' '.join(['<%s>' % s for s in module.args])))

                    elif cmd[0] == 'exec':
                        conn_id = int(cmd[1])

                        if not conn_id in self.listener.connection_dict:
                            print("\033[91mUnable to find connection with ID %d\033[0m" % conn_id)
                            continue

                        payload = PayloadManager.generate_payload(cmd[2], cmd[3:])
                        print("\033[92mExecuting payload \"%s\" on reverse shell %d: \033[0m" % (payload, conn_id))

                        payload = '%s\n' % payload
                        self.listener.connection_dict[conn_id]['conn'].send(payload.encode())

                    elif cmd[0] == 'quit':
                        Output.stop()
                        sys.exit()

                    elif cmd[0] == 'help':
                        print("\033[92mNormal mode: \033[0m")
                        print("  - list:                        List all available reverse_shell")
                        print("  - use <id>:                    Interact with a specific reverse shell")
                        print("  - kill <id>:                   Kill the reverse shell")
                        print("  - exec <id> <payload> <args>*: Execute a specific payload")
                        print("  - payloads:                    List available payloads")
                        print("  - quit:                        Stop the reverse listener")
                        print("\033[92mInteract mode: \033[0m")
                        print("  - bg:                          Stop interacting, go back to normal mode")

                except IndexError:
                    print("\033[91mError while executing command \033[0m")
                except ValueError:
                    print("\033[91mError while executing command \033[0m")

            elif self.state[0] == 'interact':
                if command == 'bg':
                    print("")
                    self.state = ('base',)
                else:
                    command = '%s\n' % command
                    self.listener.connection_dict[conn_id]['conn'].send(command.encode())

    def prompt(self):
        if self.state[0] == 'base':
            return "\033[94mrshl> \033[0m"
        else:
            return ""

    def receiver_thread(self):

        while True:
            time.sleep(0.01)

            if self.state[0] == 'interact':
                conn_id = self.state[1]

                try:
                    data = self.listener.connection_dict[conn_id]['received_queue'].get(block=False)
                    print(data.decode(errors='replace'), end='', flush=True)
                except queue.Empty:
                    pass
                except KeyError:
                    self.state = ('base',)
                    # The connection closed
                    pass


### Python script to use SNAP as InSAR processor for TSX/TDX processing compatible with StaMPS PSI processing
# Author: Jonas Ziemer, adapted from Jose Manuel Delgado Blasco (snap2stamps package)
# Date: 01/01/2023
# Version: 2.1, Python 3.9

# Step 1: unzip downloaded TSX scenes (.tar; .tar.gz)
# Step 2: preparing secondaries in folder structure
# Step 3: subset TSX scenes
# Step 4: Coregistration
# Step 5: Interferogram generation
# Step 6: StaMPS export

import os
import shutil
import sys
import glob
import subprocess
import shlex
import time
import tarfile

import configparser
import argparse
import warnings

warnings.filterwarnings('ignore')

parser = argparse.ArgumentParser(description='This script unpacks the original TSX/TDX data')
parser.add_argument("--ProjFile","-F", type=str)

args = parser.parse_args()
if not args.ProjFile:
    print("No Input File detected")
    sys.exit(-1)
else:
    configfile = args.ProjFile

print(configfile)

config = configparser.ConfigParser()
config.read(configfile)
PROJECTFOLDER=config['PROJECT_DEFINITION'].get('PROJECTFOLDER')
print(PROJECTFOLDER)

bar_message='\n#####################################################################\n'

##############################################################################
################### Unpack zipped files in the zipfolder #####################
##############################################################################
# Folders involved in this processing step
logfolder=os.path.join(PROJECTFOLDER,'logs')
zipfolder=os.path.join(PROJECTFOLDER,'zipfiles')
secondariesfolder=os.path.join(PROJECTFOLDER,'secondaries')

if not os.path.exists(logfolder):
                os.makedirs(logfolder)
if not os.path.exists(zipfolder):
                os.makedirs(zipfolder)
if not os.path.exists(secondariesfolder):
                os.makedirs(secondariesfolder)

errorlog=os.path.join(logfolder,'unpack_proc_stderr.log')
outlog=os.path.join(logfolder,'unpack_proc_stdout.log')
out_file = open(outlog, 'a')
err_file = open(errorlog, 'a')

timeStarted_global = time.time()
# Unpack .tar(.gz) files
try:
    for filename in os.listdir(zipfolder):
        os.chdir(zipfolder)
        filename = zipfolder + '/' + filename
        print(filename)
        if filename.endswith("tar.gz"):
            tar = tarfile.open(filename, "r:gz")
            if filename.endswith("tar"):
                tar = tarfile.open(filename, "r:")
        elif filename.endswith("tar"):
            tar = tarfile.open(filename, "r:")
        tar.extractall()
        tar.close()
except:
    raise Exception("The files have already been extracted!")

# Cut out the data folder only, to get the right naming of the files for later processing steps
for file in os.listdir(zipfolder):
    folder = os.path.join(zipfolder, file)
    if os.path.isdir(folder):
        for tsxmainfolder in glob.iglob(folder + '/*'):
            if tsxmainfolder.endswith('.L1B'):
                for tsxsubfolder in glob.iglob(tsxmainfolder + '/*'):
                    print(tsxsubfolder)
                    shutil.move(tsxsubfolder, secondariesfolder)
timeDelta_global = time.time() - timeStarted_global
print('Finished unzipping in ' + str(timeDelta_global) + ' seconds.\n')


from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
from sqlalchemy import String
from sqlalchemy.orm import sessionmaker, DeclarativeBase, mapped_column
from sqlalchemy import DateTime
from sqlalchemy.sql import func
from src.utils import generate_uuid

SQLALCHEMY_DATABASE_URL = "sqlite+aiosqlite:///./example.db"

# initial db engine
async_engine = create_async_engine(
    SQLALCHEMY_DATABASE_URL,
    connect_args={"check_same_thread": False},
)

# initial session maker
async_session = sessionmaker(
    autocommit=False,
    autoflush=False,
    bind=async_engine,
    class_=AsyncSession,
    expire_on_commit=False,
)

class Base(DeclarativeBase):
    id = mapped_column(String, primary_key=True, default=generate_uuid)
    created_at = mapped_column(DateTime(timezone=True), server_default=func.now())
    updated_at = mapped_column(
        DateTime(timezone=True), onupdate=func.now(), server_default=func.now()
    )

# Dependency
async def get_db() -> AsyncSession:
    async with async_session() as session:
        yield session


from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from beerspider.spiders.beers import (
    beyondbeer,
    bierdeluxe,
    bierkaufen,
    bierlinie,
    biermarket,
    bierothek,
    bierpost,
    bierselect,
    craftbeershop,
    hiergibtsbier,
    holycraft,
    meibier,
    ratsherrn,
)
from beerspider.spiders.beers.beertasting_manual import BeertastingManualSpider
from beerspider.spiders.beers.bierlinie_manual import BierlinieManualSpider
from beerspider.spiders.beers.meibier_manual import MeibierManuelSpider

# from beerspider.spiders.beers.flaschenpost_manual import FlaschenpostManualSpider

if __name__ == "__main__":
    beertasting_spider = BeertastingManualSpider(
        scrape_headless=True, scrape_from_germany=True
    )
    beertasting_spider.parse_urls()
    beertasting_spider.export_results()

    bierlinie_spider = BierlinieManualSpider(scrape_headless=True)
    bierlinie_spider.parse_urls()
    bierlinie_spider.export_results()

    meibier_spider = MeibierManuelSpider(scrape_headless=True)
    meibier_spider.parse_urls()
    meibier_spider.export_results()

    # flaschenpost_spider = FlaschenpostManualSpider(scrape_headless=True)
    # flaschenpost_spider.parse_urls()
    # flaschenpost_spider.export_results()

    settings = get_project_settings()
    process = CrawlerProcess(settings)
    process.crawl(beyondbeer.BeyondBeerSpider)
    # process.crawl(bierdeluxe.BierDeluxeSpider)
    process.crawl(bierkaufen.BierKaufenSpider)
    # process.crawl(bierlinie.BierlineSpider)
    process.crawl(biermarket.BiermarketSpider)
    process.crawl(bierothek.BierothekSpider)
    process.crawl(bierpost.BierPostSpider)
    process.crawl(bierselect.BierSelectSpider)
    process.crawl(craftbeershop.CraftbeerShopSpider)
    process.crawl(hiergibtsbier.HiergibtsbierSpider)
    process.crawl(holycraft.HolyCraftSpider)
    # process.crawl(meibier.MeibierSpider)
    process.crawl(ratsherrn.RatsherrnSpider)
    process.start()


"""Crowdstrike CRE plugin."""
import datetime
import requests
from netskope.integrations.cre.plugin_base import PluginBase, ValidationResult
from netskope.integrations.cre.models import (
    Record,
    RecordType,
    ActionWithoutParams,
    Action,
)
from netskope.common.utils import add_user_agent

PAGE_SIZE = 1000

class CrowdstrikeException(Exception):
    """Crowdstrike exception class."""

    pass

class CrowdstrikePlugin(PluginBase):
    """Crowdstrike plugin implementation."""

    def handle_error(self, resp):
        """Handle the different HTTP response code.

        Args:
            resp (requests.models.Response): Response object returned from API call.
        Returns:
            dict: Returns the dictionary of response JSON when the response code is 200.
        Raises:
            HTTPError: When the response code is not 200.
        """
        if resp.status_code == 200 or resp.status_code == 201:
            try:
                return resp.json()
            except ValueError:
                raise CrowdstrikeException(
                    "Plugin: CrowdStrike,"
                    "Exception occurred while parsing JSON response."
                )

        elif resp.status_code == 401:
            raise CrowdstrikeException(
                "Plugin: CrowdStrike, "
                "Received exit code 401, Authentication Error"
            )

        elif resp.status_code == 403:
            raise CrowdstrikeException(
                "Plugin: CrowdStrike, "
                "Received exit code 403, Forbidden User"
            )

        elif resp.status_code >= 400 and resp.status_code < 500:
            raise CrowdstrikeException(
                f"Plugin: CrowdStrike, "
                f"Received exit code {resp.status_code}, HTTP client Error"
            )

        elif resp.status_code >= 500 and resp.status_code < 600:
            raise CrowdstrikeException(
                f"Plugin: CrowdStrike, "
                f"Received exit code {resp.status_code}, HTTP server Error"
            )

        else:
            raise CrowdstrikeException(
                f"Plugin: CrowdStrike, "
                f"Received exit code {resp.status_code}, HTTP Error"
            )

        resp.raise_for_status()

    def get_agent_ids(self, headers):
        """Get the all the Agent ID list from the Query Endpoint.

        Args:
            headers (dict): Header dict object having OAUTH2 access token.
        Returns:
            dict: JSON response dict received from query endpoint.
        """
        query_endpoint = f"{self.configuration['base_url'].strip()}/devices/queries/devices/v1"
        agent_ids = []
        offset = 0
        while True:
            headers = self.reload_auth_token(headers)
            all_agent_resp = requests.get(
                query_endpoint,
                headers=add_user_agent(headers),
                params={"limit": PAGE_SIZE, "offset": offset},
                proxies=self.proxy,
            )
            agent_resp_json = self.handle_error(all_agent_resp)
            errors = agent_resp_json.get("errors")
            if errors:
                err_msg = errors[0].get("message", "")
                self.notifier.error(
                    "Plugin: CrowdStrike Unable to Fetch agents, "
                    f"Error: {err_msg}"
                )
                self.logger.error(
                    "Plugin: CrowdStrike Unable to Fetch Agents, "
                    f"Error: {err_msg}"
                )
                raise requests.HTTPError(
                    f"Plugin: CrowdStrike Unable to Fetch Agents, "
                    f"Error: {err_msg}"
                )
            resources = agent_resp_json.get("resources", [])
            offset += PAGE_SIZE
            agent_ids.extend(resources)
            if len(resources) < PAGE_SIZE:
                break
        return agent_ids

    def fetch_records(self):
        """Get the all the Agent ID list from the Query Endpoint.

        Args:
            headers (dict): Header dict object having OAUTH2 access token.
        Returns:
            dict: JSON response dict received from query endpoint.
        """
        self.configuration["client_id"] = self.configuration[
            "client_id"
        ].strip()
        self.configuration["client_secret"] = self.configuration[
            "client_secret"
        ].strip()
        try:
            auth_json = self.get_auth_json(
                self.configuration.get("client_id"),
                self.configuration.get("client_secret"),
                self.configuration.get("base_url"),
            )
            auth_token = auth_json.get("access_token")
            headers = {"Authorization": f"Bearer {auth_token}"}
            agent_ids = self.get_agent_ids(headers)
            uids_names = []
            for ids in agent_ids:
                uids_names.append(Record(uid=ids, type=RecordType.HOST))
            return uids_names

        except requests.exceptions.ProxyError:
            self.notifier.error(
                "Plugin: CrowdStrike Invalid proxy configuration."
            )
            self.logger.error(
                "Plugin: CrowdStrike Invalid proxy configuration."
            )
            raise requests.HTTPError(
                "Plugin: CrowdStrike Invalid proxy configuration."
            )
        except requests.exceptions.ConnectionError:
            self.notifier.error(
                "Plugin: CrowdStrike Unable to establish connection with CrowdStrike platform. "
                "Proxy server or CrowdStrike API is not reachable."
            )
            self.logger.error(
                "Plugin: CrowdStrike Unable to establish connection with CrowdStrike platform. "
                "Proxy server or CrowdStrike API is not reachable."
            )
            raise requests.HTTPError(
                "Plugin: CrowdStrike Unable to establish connection with CrowdStrike platform. "
                "Proxy server or CrowdStrike API is not reachable."
            )
        except requests.exceptions.RequestException as e:
            self.logger.error(
                "Plugin: CrowdStrike "
                "Exception occurred while making an API call to CrowdStrike platform"
            )
            raise e
        return []

    def fetch_scores(self, agent_ids):
        """Fetch user scores."""
        self.configuration["client_id"] = self.configuration[
            "client_id"
        ].strip()
        self.configuration["client_secret"] = self.configuration[
            "client_secret"
        ].strip()
        try:
            auth_json = self.get_auth_json(
                self.configuration.get("client_id"),
                self.configuration.get("client_secret"),
                self.configuration.get("base_url"),
            )
            aids = []
            for ids in agent_ids:
                if ids.type == RecordType.HOST:
                    aids.append(ids.uid)

            auth_token = auth_json.get("access_token")
            headers = {"Authorization": f"Bearer {auth_token}"}
            query_endpoint = f"{self.configuration['base_url']}/zero-trust-assessment/entities/assessments/v1"
            payload = {"ids": aids}
            headers = self.reload_auth_token(headers)
            resp = requests.get(
                query_endpoint,
                headers=add_user_agent(headers),
                params=payload,
                proxies=self.proxy,
            )
            all_scores_json = resp.json()
            scores = all_scores_json.get("resources", [])
            dict1 = {}
            for sub in scores:
                dict1[sub["aid"]] = sub["assessment"]["overall"] * 10
            scored_uids = []
            count_host = 0
            for ids, score in dict1.items():
                if score <= int(self.configuration["minimum_score"]):
                    count_host = count_host + 1
                    continue
                else:
                    scored_uids.append(
                        Record(uid=ids, type=RecordType.HOST, score=score)
                    )
            self.logger.info(
                "Plugin: CrowdStrike "
                f"Unable to store scores of {count_host} Hosts to CrowdStrike."
            )
            return scored_uids
        except requests.exceptions.ProxyError:
            self.notifier.error(
                "Plugin: CrowdStrike Invalid proxy configuration."
            )
            self.logger.error(
                "Plugin: CrowdStrike Invalid proxy configuration."
            )
            raise requests.HTTPError(
                "Plugin: CrowdStrike Invalid proxy configuration."
            )
        except requests.exceptions.ConnectionError:
            self.notifier.error(
                "Plugin: CrowdStrike Unable to establish connection with CrowdStrike platform. "
                "Proxy server or CrowdStrike API is not reachable."
            )
            self.logger.error(
                "Plugin: CrowdStrike Unable to establish connection with CrowdStrike platform. "
                "Proxy server or CrowdStrike API is not reachable."
            )
            raise requests.HTTPError(
                "Plugin: CrowdStrike Unable to establish connection with CrowdStrike platform. "
                "Proxy server or CrowdStrike API is not reachable."
            )
        except requests.exceptions.RequestException as e:
            self.logger.error(
                "Plugin: CrowdStrike "
                "Exception occurred while making an API call to CrowdStrike platform"
            )
            raise e
        return []

    def get_actions(self):
        """Get available actions."""
        return [
            ActionWithoutParams(label="No action", value="generate"),
        ]

    def validate_action(self, action: Action):
        """Validate Netskope configuration."""
        if action.value not in ["generate"]:
            return ValidationResult(
                success=False, message="Unsupported action provided."
            )

        return ValidationResult(success=True, message="Validation successful.")

    def get_action_fields(self, action: Action):
        """Get fields required for an action."""
        return []

    def execute_action(self, record: str, action: Action):
        """Execute action on the record."""
        pass

    def reload_auth_token(self, headers):
        """Reload the OAUTH2 token after Expiry."""
        if self.storage["token_expiry"] < datetime.datetime.now():
            self.logger.info(
                "Plugin: Crowdstrike OAUTH2 token expired generating new token"
            )
            auth_json = self.get_auth_json(
                self.configuration.get("client_id"),
                self.configuration.get("client_secret"),
                self.configuration.get("base_url"),
            )
            auth_token = auth_json.get("access_token")
            headers = {"Authorization": f"Bearer {auth_token}"}
            return headers
        else:
            return headers

    def validate(self, data):
        """Validate the Plugin configuration parameters.

        Args:
            data (dict): Dict object having all the Plugin configuration parameters.
        Returns:
            cte.plugin_base.ValidateResult: ValidateResult object with success flag and message.
        """
        self.logger.info(
            "Plugin: CrowdStrike Executing validate method for CrowdStrike plugn"
        )

        if "base_url" not in data or data["base_url"].strip() not in [
            "https://api.crowdstrike.com",
            "https://api.us-2.crowdstrike.com",
            "https://api.laggar.gcw.crowdstrike.com",
            "https://api.eu-1.crowdstrike.com",
        ]:
            self.logger.error(
                "Plugin: CrowdStrike Validation error occurred "
                "Error: Type of Pulling configured should be non-empty string."
            )
            return ValidationResult(
                success=False,
                message="Invalid value base_url.",
            )

        if (
            "client_id" not in data
            or not data["client_id"].strip()
            or type(data["client_id"]) != str
        ):
            self.logger.error(
                "Plugin: CrowdStrike Validation error occurred"
                "Error: Type of Client ID should be non-empty string."
            )
            return ValidationResult(
                success=False,
                message="Invalid Client ID provided.",
            )

        if (
            "minimum_score" not in data
            or not data["minimum_score"]
            or 1 > int(data["minimum_score"])
            or 1000 < int(data["minimum_score"])
        ):
            self.logger.error(
                "Plugin: CrowdStrike Validation error occurred"
                "Error: Type of minimum_score should be non-empty integer."
            )
            return ValidationResult(
                success=False,
                message="Invalid minimum_score provided.Range is from 1 to 1000.",
            )

        if (
            "client_secret" not in data
            or not data["client_secret"]
            or type(data["client_secret"]) != str
        ):
            self.logger.error(
                "Plugin: CrowdStrike Validation error occurred"
                "Error: Type of Client Secret should be non-empty string."
            )
            return ValidationResult(
                success=False,
                message="Invalid Client Secret provided.",
            )

        return self.validate_auth_params(
            data["client_id"], data["client_secret"], data["base_url"]
        )

    def validate_auth_params(self, client_id, client_secret, base_url):
        """Validate the authentication params with CrowdStrike platform.

        Args:
            client_id (str): Client ID required to generate OAUTH2 token.
            client_secret (str): Client Secret required to generate OAUTH2 token.
            base_url (str): Base url of crowd strike
        Returns:
            ValidationResult: ValidationResult object having validation results after making
            an API call.
        """
        try:
            self.check_url_valid(client_id, client_secret, base_url)
            return ValidationResult(
                success=True,
                message="Validation successfull for CrowdStrike Plugin",
            )
        except requests.exceptions.ProxyError:
            self.logger.error(
                "Plugin: CrowdStrike Validation Error, "
                "Invalid proxy configuration."
            )
            return ValidationResult(
                success=False,
                message="Validation Error, Invalid proxy configuration.",
            )
        except requests.exceptions.ConnectionError:
            self.logger.error(
                "Plugin: CrowdStrike Validation Error, "
                "Unable to establish connection with CrowdStrike Platform API"
            )
            return ValidationResult(
                success=False,
                message="Validation Error, Unable to establish connection with CrowdStrike Platform API",
            )
        except requests.HTTPError as err:
            self.logger.error(
                f"Plugin: CrowdStrike Validation Error, "
                f"Error in validating Credentials {repr(err)}"
            )
            return ValidationResult(
                success=False,
                message=f"Validation Error, Error in validating Credentials {repr(err)}",
            )

    def check_url_valid(self, client_id, client_secret, base_url):
        """
        Validate the authentication params with CrowdStrike platform.

        Args:
            client_id (str): Client ID required to generate OAUTH2 token.
            client_secret (str): Client Secret required to generate OAUTH2 token.
            base_url (str): Base url of crowd strike
        Returns:
            Raise error if valid base url is not selected.
        """
        auth_json = self.get_auth_json(client_id, client_secret, base_url)
        auth_token = auth_json.get("access_token")
        headers = {"Authorization": f"Bearer {auth_token}"}
        query_endpoint = f"{base_url}/devices/queries/devices/v1?limit=1"
        all_agent_resp = requests.get(
            query_endpoint, headers=add_user_agent(headers), proxies=self.proxy
        )
        if all_agent_resp.status_code == 401:
            raise requests.HTTPError("Invalid base url.")

        agent_resp_json = self.handle_error(all_agent_resp)
        errors = agent_resp_json.get("errors")
        if errors:
            err_msg = errors[0].get("message", "")
            self.notifier.error(
                "Plugin: CrowdStrike Unable to Fetch agents, "
                f"Error: {err_msg}"
            )
            self.logger.error(
                "Plugin: CrowdStrike Unable to Fetch Agents, "
                f"Error: {err_msg}"
            )
            raise requests.HTTPError(
                f"Plugin: CrowdStrike Unable to Fetch Agents, "
                f"Error: {err_msg}"
            )
        return agent_resp_json

    def get_auth_json(self, client_id, client_secret, base_url):
        """Get the OAUTH2 Json object with access token from CrowdStrike platform.

        Args:
            client_id (str): Client ID required to generate OAUTH2 token.
            client_secret (str): Client Secret required to generate OAUTH2 token.
            base_url (str): Base URL of crowdstrike.
        Returns:
            json: JSON response data in case of Success.
        """
        client_id = client_id.strip()
        client_secret = client_secret.strip()
        auth_endpoint = f"{base_url}/oauth2/token"

        auth_params = {
            "grant_type": "client_credentials",
            "client_id": client_id,
            "client_secret": client_secret,
        }
        resp = requests.post(
            auth_endpoint,
            data=auth_params,
            verify=self.ssl_validation,
            proxies=self.proxy,
            headers=add_user_agent()
        )
        auth_json = self.handle_error(resp)
        auth_errors = auth_json.get("errors")
        if auth_errors:
            err_msg = auth_errors[0].get("message", "")
            self.notifier.error(
                "Plugin: CrowdStrike Unable to generate Auth token. "
                f"Error: {err_msg}"
            )
            self.logger.error(
                "Plugin: CrowdStrike Unable to generate Auth token. "
                f"Error: {err_msg}"
            )
            raise requests.HTTPError(
                f"Plugin: CrowdStrike Unable to generate Auth token. "
                f"Error: {err_msg}"
            )
        if self.storage is not None:
            self.storage[
                "token_expiry"
            ] = datetime.datetime.now() + datetime.timedelta(
                seconds=int(auth_json.get("expires_in", 1799))
            )
        return auth_json


import io
import sys

_INPUT = """\
5 2
1 2
2 3
3 4
4 5
1 3
1 5
"""
sys.stdin = io.StringIO(_INPUT)

n, q = map(int, input().split())
connect = [[] for i in range(n+1)]
visited = [False for _ in range(n+1)]
# 0が赤、1が青
color = [-1] * (n+1)

for i in range(n - 1):
    a, b = map(int, input().split())
    connect[a].append(b)
    connect[b].append(a)

from collections import deque
que = deque()

color[1] = 0
visited[1] = True

que.append(1)

while 0 < len(que):
    now = que.popleft()
    now_color = color[now]

    for to in connect[now]:
        if visited[to] == False:
            visited[to] = True

            if now_color == 0:
                color[to] = 1

            if now_color == 1:
                color[to] = 0

            que.append(to)

for i in range(q):
    c, d = map(int, input().split())
    if color[c] == color[d]:
        print("Town")
    else:
        print("Road")


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.7.1
#   kernelspec:
#     display_name: Manifold 2
#     language: python
#     name: manifold2
# ---

# # Plot Embeddings
#
# Only plots one embedding at a time.

import pandas as pd
import numpy as np
import os.path as osp
import umap
from sklearn.metrics import silhouette_score
from utils.data_info import PRJDIR, task_labels, SBJ_list, UMAP_k_list, wl_sec, tr
import panel as pn
import plotly.express as px
pn.extension('plotly')

SBJ = 'SBJ06' # Subject number
n = 3 # number of dimensions
drop = 'DropData' # Type of data (FullData: all windows, DropData: only pure windows, DropX: keep every X window)
ndrop = 15 # If using DropX embedding, X value (5, 10, or 15) MUST MATCH X
wl_trs = int(wl_sec/1.5) # Window length in TRs (TR = 1.5 sec)

if drop == 'FullData':
    task_df = task_labels(wl_trs, PURE=False)
elif drop == 'Drop5' or drop == 'Drop10' or drop == 'Drop15':
    print('in')
    task_df = task_labels(wl_trs, PURE=False)
    task_df = task_df.loc[range(0, task_df.shape[0], ndrop)].copy()
elif drop == 'DropData':
    task_df = task_labels(wl_trs, PURE=True)
print(task_df.shape)

# +
# Load LE Embedding
# -----------------
k = 60 # k-NN value
metric = 'correlation' # distance metric

file_name = SBJ+'_LE_embedding_wl'+str(wl_sec).zfill(3)+'_k'+str(k).zfill(3)+'_n'+str(n).zfill(2)+'_'+metric+'_'+drop+'.csv'
file_path = osp.join(PRJDIR,'derivatives','LE',file_name)
embed_df  = pd.read_csv(file_path)
print('++ INFO: LE embedding loaded')
print('         Data shape:',embed_df.shape)

# + jupyter={"outputs_hidden": true, "source_hidden": true}
# Load TSNE Embedding
# -------------------
p = 13
metric = 'correlation'

file_name = SBJ+'_TSNE_embedding_wl'+str(wl_sec).zfill(3)+'_p'+str(p).zfill(3)+'_n'+str(n).zfill(2)+'_'+metric+'_'+drop+'.csv'
file_path = osp.join(PRJDIR,'derivatives','TSNE',file_name)
embed_df  = pd.read_csv(file_path)
print('++ INFO: TSNE embedding loaded')
print('         Data shape:',embed_df.shape)

# + jupyter={"outputs_hidden": true, "source_hidden": true}
# Load UMAP Embedding
# -------------------
k = 11
metric = 'correlation'

file_name = SBJ+'_UMAP_embedding_wl'+str(wl_sec).zfill(3)+'_k'+str(k).zfill(3)+'_n'+str(n).zfill(2)+'_'+metric+'_'+drop+'.csv'
file_path = osp.join(PRJDIR,'derivatives','UMAP',file_name)
embed_df  = pd.read_csv(file_path)
print('++ INFO: UMAP embedding loaded')
print('         Data shape:',embed_df.shape)
# -

embed_df.index = task_df.index # Make embeding data frame index and task label data frame index the same

# +
# Plot Embedding
task_cmap = {'Rest': 'gray', 'Memory': 'blue', 'Video': '#F4D03F', 'Math': 'green', 'Inbetween': 'black'}

plot_input = pd.DataFrame(columns=['x','y','z'])
plot_input[['x','y','z']] = embed_df[['1_norm','2_norm','3_norm']].copy()
plot_input['Task'] = task_df['Task']

plot = px.scatter_3d(plot_input, x='x', y='y', z='z', color='Task', color_discrete_map=task_cmap, width=700, height=600, opacity=0.7)
plot = plot.update_traces(marker=dict(size=3,line=dict(width=0)))

plot
# -

# Compute SI Value
if drop == 'FullData' or drop == 'Drop5' or drop == 'Drop10' or drop == 'Drop15':
    drop_index    = task_df.index[task_df['Task'] == 'Inbetween']
    drop_embed_df = embed_df.drop(drop_index).reset_index(drop=True)
    drop_task_df  = task_df.drop(drop_index).reset_index(drop=True)
    SI = silhouette_score(drop_embed_df[['1_norm', '2_norm', '3_norm']], drop_task_df['Task'].values)
elif drop == 'DropData':
    SI = silhouette_score(embed_df[['1_norm', '2_norm', '3_norm']], task_df['Task'].values)
print(SI)


#################################
# Your name: Nathan Bloch
#################################

# Please import and use stuff only from the packages numpy, sklearn, matplotlib
import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm
from sklearn.datasets import make_blobs

"""
Q4.1 skeleton.

Please use the provided functions signature for the SVM implementation.
Feel free to add functions and other code, and submit this file with the name svm.py
"""

# generate points in 2D
# return training_data, training_labels, validation_data, validation_labels
def get_points():
    X, y = make_blobs(n_samples=120, centers=2, random_state=0, cluster_std=0.88)
    return X[:80], y[:80], X[80:], y[80:]

def create_plot(X, y, clf):
    plt.clf()

    # plot the data points
    plt.scatter(X[:, 0], X[:, 1], c=y, s=30, cmap=plt.cm.PiYG)

    # plot the decision function
    ax = plt.gca()
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()

    # create grid to evaluate model
    xx = np.linspace(xlim[0] - 2, xlim[1] + 2, 30)
    yy = np.linspace(ylim[0] - 2, ylim[1] + 2, 30)
    YY, XX = np.meshgrid(yy, xx)
    xy = np.vstack([XX.ravel(), YY.ravel()]).T
    Z = clf.decision_function(xy).reshape(XX.shape)

    # plot decision boundary and margins
    ax.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5,
               linestyles=['--', '-', '--'])

def train_three_kernels(X_train, y_train, X_val, y_val):
    """
    Returns: np.ndarray of shape (3,2):
        A two dimensional array of size 3 that contains the number of support vectors for each class(2) in the three kernels.
    """
    linear_clf = svm.SVC(kernel='linear', C=1000)
    linear_clf.fit(X_train, y_train)

    quadratic_clf = svm.SVC(kernel='poly', C=1000, degree=2)
    quadratic_clf.fit(X_train, y_train)

    rbf_clf = svm.SVC(C=1000)
    rbf_clf.fit(X_train, y_train)

    create_plot(X_train, y_train, rbf_clf)
    # plt.show()

    create_plot(X_train, y_train, quadratic_clf)
    # plt.show()

    create_plot(X_train, y_train, linear_clf)
    # plt.show()

    return np.array([linear_clf.n_support_, quadratic_clf.n_support_, rbf_clf.n_support_])

def linear_accuracy_per_C(X_train, y_train, X_val, y_val):
    """
        Returns: np.ndarray of shape (11,) :
                    An array that contains the accuracy of the resulting model on the VALIDATION set.
    """
    valid_accuracy = np.zeros(11)
    train_accuracy = np.zeros(11)
    C = [pow(10, i) for i in range(-5, 6)]
    for i in range(11):
        clf = svm.SVC(kernel='linear', C = C[i])
        clf.fit(X_train, y_train)
        valid_accuracy[i] = calc_accuracy(X_val, y_val, clf)
        train_accuracy[i] = calc_accuracy(X_train, y_train, clf)

    valid_line, = plt.plot(C, valid_accuracy)
    train_line, = plt.plot(C, train_accuracy)
    plt.xscale('log')
    plt.legend((valid_line, train_line), ('Validation Set', 'Train Set'))
    # plt.show()

    return valid_accuracy

def rbf_accuracy_per_gamma(X_train, y_train, X_val, y_val):
    """
        Returns: np.ndarray of shape (11,) :
                    An array that contains the accuracy of the resulting model on the VALIDATION set.
    """
    valid_accuracy = np.zeros(11)
    train_accuracy = np.zeros(11)
    gammas = [pow(10, i) for i in range(-5, 6)]
    for i in range(11):
        clf = svm.SVC(C = 10, gamma=gammas[i])
        clf.fit(X_train, y_train)
        valid_accuracy[i] = calc_accuracy(X_val, y_val, clf)
        train_accuracy[i] = calc_accuracy(X_train, y_train, clf)

    valid_line, = plt.plot(gammas, valid_accuracy)
    train_line, = plt.plot(gammas, train_accuracy)
    plt.xscale('log')
    plt.legend((valid_line, train_line), ('Validation Set', 'Train Set'))
    # plt.show()

    return valid_accuracy

#====================================  HELPER FUNCTION  =========================================

def calc_accuracy(X_data, Y_data, clf):
    true = 0
    predictions = clf.predict(X_data)
    for i in range(X_data.shape[0]):
        if(predictions[i] == Y_data[i]):
            true += 1

    return true / X_data.shape[0]

# train_data, train_labels, validation_data, validation_labels = get_points();

# 1(a)
# sv = train_three_kernels(train_data, train_labels, validation_data, validation_labels)
# print("# of SVs in linear model:", sum(sv[0]));
# print("# of SVs in quaratic model:", sum(sv[1]));
# print("# of SVs in RBF model:", sum(sv[2]));

# 1(b)
# linear_accuracy_per_C(train_data, train_labels, validation_data, validation_labels)

# 1(c)
# rbf_accuracy_per_gamma(train_data, train_labels, validation_data, validation_labels)


import sqlite3
from static import database
def print_all(database):
    connect = sqlite3.connect('qaoncloud.db')
    crsr = connect.cursor()
    crsr.execute('''SELECT * FROM tag_info''')
    ans = crsr.fetchall()
    for i in ans:
        print(i)

    print('done')

if __name__=='__main__':
    print_all(database)


import os
import random
import re
import xlwt

dataDir = '8.20'
NUMBER_OF_CLASS = 5

def numberSplit(txt):
    res = re.findall(r"X:(.*)mY:(.*)mP:(.*)dB", txt)
    X = res[0][0]
    Y = res[0][1]
    Z = res[0][2]
    if "0.00" in X or "0.00" in Y or "0.00" in Z:
        return 0
    result = X + " " + Y + " " + Z + '\n'
    return result

def makeExl(filename):
    fopen = open(dataDir+"/"+filename)
    try:
        lines = fopen.readlines()
    except:
        print(filename+"有问题")
    else:
        random.shuffle(lines)

        a = 0
        b = 0
        c = 0
        A = ""
        B = ""
        C = ""
        for line in lines:
            line = line.replace(" ", "")
            if line[4] == '0' and a < NUMBER_OF_CLASS and numberSplit(line) != 0:
                A += numberSplit(line)
                a += 1

            elif line[4] == '1' and b < NUMBER_OF_CLASS and numberSplit(line) != 0:
                B += numberSplit(line)
                b += 1

            elif line[4] == '2' and c < NUMBER_OF_CLASS and numberSplit(line) != 0:
                C += numberSplit(line)
                c += 1

        Date = "0: \n"+A+'1: \n'+B+"2: \n"+C

        temp = open("temp", 'w')
        temp.write(Date)
        temp.close()

        tempOpen = open("temp")
        tempLines = tempOpen.readlines()

        sheet = exlFile.add_sheet(filename)

        exlX = 0
        exlY = 0
        for tLine in tempLines:
            if ":" in tLine:
                continue
            lineDate = tLine.split(" ")
            sheet.write(exlX, exlY, lineDate[0])
            exlY += 1
            sheet.write(exlX, exlY, lineDate[1])
            exlY += 1
            sheet.write(exlX, exlY, lineDate[2])
            exlX += 1
            exlY = 0

        exlFile.save(dataDir+".xls")
        # print(filename + '完成')

print(dataDir)
exlFile = xlwt.Workbook(encoding='utf-8', style_compression=0)
for files in os.listdir(dataDir):
    makeExl(files)


import os
import zipfile

def backupToZip(folder):
    # A function to backup the folders and files to a zip file

    folder = os.path.abspath(folder)  # make sure the folder path is absolute

    # Figure out the filename this code should use based on what files already exist.
    number = 1
    while True:

        zipFileName = "{}_{}.zip".format(os.path.basename(folder), number)
        if not os.path.exists(zipFileName):
            break
        number += 1

    # Create the ZIP file
    print("Creating the {}".format(zipFileName))
    backupZip = zipfile.ZipFile(zipFileName, "w")

    # Walk the entire folder tree and compress the files in each folder
    for foldernames, subfolders, filenames in os.walk(folder):

        print("Adding files in {}".format(foldernames))

        # ALl the files in this folder to the zip file
        for filename in filenames:
            newBase = os.path.basename(folder) + "_"
            if filename.startswith(newBase) and filename.endswith(".zip"):
                continue
            backupZip.write(os.path.join(foldernames, filename))
    backupZip.close()

backupToZip(".")