from django.contrib.auth.tokens import default_token_generator
from django.core.mail import send_mail
from django.db.models import Avg
from django.shortcuts import get_object_or_404
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import filters, permissions, status, viewsets
from rest_framework.decorators import action, api_view, permission_classes
from rest_framework.response import Response
from rest_framework_simplejwt.tokens import AccessToken
from reviews.models import Category, Genre, Review, Title
from users.models import User

from .filters import TitleFilter
from .mixins import MasterViewSet
from .permissions import (IsAdminModerAuthor, IsAdminOrSuper,
                          IsAdminUserOrReadOnly)
from .serializers import (CategorySerializer, CommentSerializer,
                          ConfirmationCodeSerializer, GenreSerializer,
                          ReviewSerializer, SignUpSerializer,
                          TitleReadSerializer, TitleSerializer,
                          UserMeSerializer, UsersSerializer)

@api_view(['POST'])
@permission_classes([permissions.AllowAny])
def sign_up(request):
    serializer = SignUpSerializer(data=request.data)
    if serializer.is_valid(raise_exception=True):
        email = serializer.validated_data['email']
        username = serializer.validated_data['username']
        user, create = User.objects.get_or_create(
            email=email,
            username=username
        )
        confirmation_code = default_token_generator.make_token(user)
        user.confirmation_code = confirmation_code
        user.save()
        send_mail(
            'Введите код для продолжения регистрации',
            confirmation_code,
            'admin@gmail.com',
            [email],
            fail_silently=False,
        )
        return Response(serializer.data, status=status.HTTP_200_OK)
    return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

@api_view(['POST'])
@permission_classes([permissions.AllowAny])
def getting_token(request):
    serializer = ConfirmationCodeSerializer(data=request.data)
    if not serializer.is_valid(raise_exception=True):
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
    username = serializer.validated_data['username']
    confirmation_code = serializer.validated_data['confirmation_code']
    user = get_object_or_404(User, username=username)
    if not default_token_generator.check_token(user, confirmation_code):
        return Response(serializer.errors,
                        status=status.HTTP_400_BAD_REQUEST)
    token = str(AccessToken.for_user(user))
    content = {'token': token}
    return Response(content, status=status.HTTP_201_CREATED)

class UsersViewSet(viewsets.ModelViewSet):
    queryset = User.objects.all()
    serializer_class = UsersSerializer
    permission_classes = (permissions.IsAuthenticated, IsAdminOrSuper,)
    filter_backends = (filters.SearchFilter,)
    filterset_fields = ('username',)
    search_fields = ('username',)
    lookup_field = 'username'

    @action(
        methods=['GET', 'PATCH'],
        detail=False,
        url_path='me',
        permission_classes=(permissions.IsAuthenticated,)
    )
    def me(self, request):
        user = get_object_or_404(User, username=self.request.user)
        if request.method == 'GET':
            serializer = UserMeSerializer(user)
            return Response(serializer.data, status=status.HTTP_200_OK)
        if request.method == 'PATCH':
            serializer = UserMeSerializer(user,
                                          data=request.data,
                                          partial=True)
            serializer.is_valid(raise_exception=True)
            serializer.save()
            return Response(serializer.data, status=status.HTTP_200_OK)

class CategoryViewSet(MasterViewSet):
    """Viewset для объектов модели Category."""
    queryset = Category.objects.all()
    serializer_class = CategorySerializer

    @action(
        methods=['DELETE'],
        detail=False,
        url_path=r'(?P<slug>[-a-zA-Z0-9_]+)',
        url_name='delete_category'
    )
    def delete_category(self, request, slug):
        category = get_object_or_404(Category, slug=slug)
        category.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

class GenreViewSet(MasterViewSet):
    """Viewset для объектов модели Genre."""
    queryset = Genre.objects.all()
    serializer_class = GenreSerializer

    @action(
        methods=['DELETE'],
        detail=False,
        url_path=r'(?P<slug>[-a-zA-Z0-9_]+)',
        url_name='delete_genre'
    )
    def delete_genre(self, request, slug):
        genre = get_object_or_404(Genre, slug=slug)
        genre.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)

class TitleViewSet(viewsets.ModelViewSet):
    """Viewset для объектов модели Title."""
    queryset = Title.objects.annotate(rating=Avg('reviews__score'))
    serializer_class = TitleSerializer
    filter_backends = (DjangoFilterBackend,)
    filterset_class = TitleFilter
    permission_classes = (IsAdminUserOrReadOnly,)

    def get_serializer_method(self):
        if self.request.method == 'GET':
            return TitleReadSerializer
        return TitleSerializer

class CommentViewSet(viewsets.ModelViewSet):
    """ViewSet для объектов модели Comment."""
    serializer_class = CommentSerializer
    permission_classes = (IsAdminModerAuthor,)

    def get_review(self):
        review_id = self.kwargs.get('review_id')
        return get_object_or_404(Review, pk=review_id)

    def get_queryset(self):
        return self.get_review().comments.all()

    def perform_create(self, serializer):
        serializer.save(author=self.request.user, review=self.get_review())

class ReviewViewSet(viewsets.ModelViewSet):
    """ViewSet для объектов модели Review."""
    serializer_class = ReviewSerializer
    permission_classes = (IsAdminModerAuthor,)

    def get_title(self):
        title_id = self.kwargs.get('title_id')
        return get_object_or_404(Title, pk=title_id)

    def get_queryset(self):
        return self.get_title().reviews.all()

    def perform_create(self, serializer):
        user = self.request.user
        serializer.save(author=user, title=self.get_title())


# -*- coding: utf-8 -*-
"""
Created on Sun Dec  4 22:34:32 2022

@author: Shahir, Faraz, Pratyush
Modified from: https://github.com/aanna0701/SPT_LSA_ViT
"""

import torch
from torch import nn
import torch.nn.functional as F

from einops import rearrange, repeat
from einops.layers.torch import Rearrange

from cs330_project.utils import make_pair_shape

def max_neg_value(tensor):
    return -torch.finfo(tensor.dtype).max

class DropPath(nn.Module):
    """
    Obtained from: github.com:rwightman/pytorch-image-models
    Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).
    """

    def __init__(
            self,
            drop_prob=None):

        super(DropPath, self).__init__()
        self.drop_prob = drop_prob

    @staticmethod
    def drop_path(
            x,
            drop_prob=0.0,
            training=False):
        """
        Obtained from: github.com:rwightman/pytorch-image-models
        Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).
        This is the same as the DropConnect impl I created for EfficientNet, etc networks, however,
        the original name is misleading as 'Drop Connect' is a different form of dropout in a separate paper...
        # issuecomment-532968956 ... I've opted for
        See discussion: https://github.com/tensorflow/tpu/issues/494
        changing the layer and argument names to 'drop path' rather than mix DropConnect as a layer name and use
        'survival rate' as the argument.
        """
        if drop_prob == 0. or not training:
            return x
        keep_prob = 1 - drop_prob
        # work with diff dim tensors, not just 2D ConvNets
        shape = (x.shape[0],) + (1,) * (x.ndim - 1)
        random_tensor = keep_prob + \
            torch.rand(shape, dtype=x.dtype, device=x.device)
        random_tensor.floor_()  # binarize
        output = x.div(keep_prob) * random_tensor
        return output

    def forward(
            self,
            x):

        return self.drop_path(x, self.drop_prob, self.training)

class PatchShifting2d(nn.Module):
    CARDINAL_4_DIRECTIONS_MODE = 0
    CARDINAL_8_DIRECTIONS_MODE = 1
    DIAGONAL_4_DIRECTIONS_MODE = 2

    def __init__(
            self,
            shift_size,
            mode=0):

        super().__init__()
        self.mode = mode
        self.expansion = 9 if self.mode == self.CARDINAL_8_DIRECTIONS_MODE else 5
        self.half_shift = shift_size // 2

    def forward(
            self,
            x):

        # x is shape [b, c, h, w]
        x_pad = F.pad(
            x,
            (self.half_shift, self.half_shift,
             self.half_shift, self.half_shift))

        # 4 cardinal directions
        if self.mode == self.CARDINAL_4_DIRECTIONS_MODE:
            x_l2 = x_pad[:, :, self.half_shift:-
                         self.half_shift, : -self.half_shift * 2]
            x_r2 = x_pad[:, :, self.half_shift: -
                         self.half_shift, self.half_shift * 2:]
            x_t2 = x_pad[:, :, : -self.half_shift *
                         2, self.half_shift: -self.half_shift]
            x_b2 = x_pad[:, :, self.half_shift * 2:,
                         self.half_shift: -self.half_shift]
            x_cat = torch.cat([x, x_l2, x_r2, x_t2, x_b2], dim=1)

        # 4 diagonal directions
        if self.mode == self.DIAGONAL_4_DIRECTIONS_MODE:
            x_lu = x_pad[:, :, : -self.half_shift * 2, : -self.half_shift * 2]
            x_ru = x_pad[:, :, : -self.half_shift * 2, self.half_shift * 2:]
            x_lb = x_pad[:, :, self.half_shift * 2:, : -self.half_shift * 2]
            x_rb = x_pad[:, :, self.half_shift * 2:, self.half_shift * 2:]
            x_cat = torch.cat([x, x_lu, x_ru, x_lb, x_rb], dim=1)

        # 8 cardinal directions
        if self.mode == self.CARDINAL_8_DIRECTIONS_MODE:
            x_l2 = x_pad[:, :, self.half_shift: -
                         self.half_shift, : -self.half_shift * 2]
            x_r2 = x_pad[:, :, self.half_shift: -
                         self.half_shift, self.half_shift * 2:]
            x_t2 = x_pad[:, :, : -self.half_shift *
                         2, self.half_shift: -self.half_shift]
            x_b2 = x_pad[:, :, self.half_shift * 2:,
                         self.half_shift: -self.half_shift]
            x_lu = x_pad[:, :, : -self.half_shift * 2, : -self.half_shift * 2]
            x_ru = x_pad[:, :, : -self.half_shift * 2, self.half_shift * 2:]
            x_lb = x_pad[:, :, self.half_shift * 2:, : -self.half_shift * 2]
            x_rb = x_pad[:, :, self.half_shift * 2:, self.half_shift * 2:]
            x_cat = torch.cat([
                x, x_l2, x_r2, x_t2, x_b2,
                x_lu, x_ru, x_lb, x_rb], dim=1)

        out = x_cat  # [b, c * self.expansion, h, w]

        return out

class PatchShifting3d(PatchShifting2d):
    def forward(
            self,
            x):

        b, c, t, h, w = x.shape
        x = x.view(b, c * t, h, w)
        x = super().forward(x)
        x = x.view(b, c * self.expansion, t, h, w)

        return x

class ShiftedPatchEmbed2d(nn.Module):
    def __init__(
            self,
            in_img_size,
            in_channels,
            embed_dim,
            patch_size):

        super().__init__()

        self.in_img_size = in_img_size
        self.in_channels = in_channels
        self.embed_dim = embed_dim
        self.patch_size = patch_size

        self.patch_shifting = PatchShifting2d(patch_size)
        self.patch_dim = (
            self.in_channels *
            self.patch_shifting.expansion *
            (self.patch_size ** 2))

        self.num_patches = (
            (self.in_img_size[0] // self.patch_size) * (self.in_img_size[1] // self.patch_size))

        self.patchify = nn.Sequential(
            self.patch_shifting,
            Rearrange(
                'b c (h p1) (w p2) -> b (h w) (p1 p2 c)',
                p1=self.patch_size,
                p2=self.patch_size)
        )
        self.proj = nn.Linear(self.patch_dim, embed_dim)

    def forward(
            self,
            x):

        # x is shape [b, c, h, w]
        x = self.patchify(x)
        x = self.proj(x)

        return x

class ShiftedPatchEmbed3d(nn.Module):
    def __init__(
            self,
            in_img_size,
            in_channels,
            in_num_frames,
            embed_dim,
            patch_size,
            tubelet_size):

        super().__init__()

        self.in_img_size = in_img_size
        self.in_channels = in_channels
        self.in_num_frames = in_num_frames
        self.embed_dim = embed_dim
        self.patch_size = patch_size
        self.tubelet_size = tubelet_size

        self.patch_shifting = PatchShifting3d(patch_size)
        self.patch_dim = (
            self.in_channels *
            self.patch_shifting.expansion *
            (self.patch_size ** 2) *
            self.tubelet_size)

        self.num_patches = ((self.in_num_frames // self.tubelet_size) * (
            self.in_img_size[0] // self.patch_size) * (self.in_img_size[1] // self.patch_size))

        self.patchify = nn.Sequential(
            self.patch_shifting,
            Rearrange(
                'b c (t p0) (h p1) (w p2) -> b (t h w) (p0 p1 p2 c)',
                p0=self.tubelet_size,
                p1=self.patch_size,
                p2=self.patch_size)
        )
        self.proj = nn.Linear(self.patch_dim, self.embed_dim)

    def forward(
            self,
            x):

        # x is shape [b, c, t, h, w]
        x = self.patchify(x)
        x = self.proj(x)

        return x

class PatchEmbed2d(nn.Module):
    def __init__(
            self,
            in_img_size,
            in_channels,
            patch_height,
            patch_width,
            embed_dim):

        super().__init__()

        self.in_img_size = in_img_size
        self.in_channels = in_channels
        self.patch_height = patch_height
        self.patch_width = patch_width
        self.patch_dim = in_channels * patch_height * patch_width
        self.embed_dim = embed_dim

        self.num_patches = (
            (self.in_img_size[0] // patch_height) * (self.in_img_size[1] // patch_width))

        self.patchify = Rearrange(
            'b c (h p1) (w p2) -> b (h w) (p1 p2 c)',
            p1=self.patch_height,
            p2=self.patch_width)
        self.proj = nn.Linear(self.patch_dim, self.embed_dim)

    def forward(
            self,
            x):

        # x is shape [b, c, h, w]
        x = self.patchify(x)
        x = self.proj(x)

        return x

class PatchEmbed3d(nn.Module):
    def __init__(
            self,
            in_img_size,
            in_channels,
            in_num_frames,
            patch_height,
            patch_width,
            tubelet_size,
            embed_dim):

        super().__init__()

        self.in_img_size = in_img_size
        self.in_channels = in_channels
        self.in_num_frames = in_num_frames
        self.patch_height = patch_height
        self.patch_width = patch_width
        self.tubelet_size = tubelet_size
        self.patch_dim = in_channels * patch_height * patch_width * tubelet_size
        self.embed_dim = embed_dim

        self.num_patches = ((in_num_frames // tubelet_size) * (
            self.in_img_size[0] // patch_height) * (self.in_img_size[1] // patch_width))

        self.patchify = Rearrange(
            'b c (t p0) (h p1) (w p2) -> b (t h w) (p0 p1 p2 c)',
            p0=self.tubelet_size,
            p1=self.patch_height,
            p2=self.patch_width)
        self.proj = nn.Linear(self.patch_dim, self.embed_dim)

    def forward(
            self,
            x):

        # x is shape [b, c, t, h, w]
        x = self.patchify(x)
        x = self.proj(x)

        return x

class PreNorm(nn.Module):
    def __init__(
            self,
            num_tokens,
            dim,
            fn):

        super().__init__()

        self.dim = dim
        self.num_tokens = num_tokens
        self.norm = nn.LayerNorm(dim)
        self.fn = fn

    def forward(
            self,
            x,
            **kwargs):

        return self.fn(self.norm(x), **kwargs)

class TransformerInnerMlp(nn.Module):
    def __init__(
            self,
            dim,
            num_patches,
            hidden_dim,
            dropout=0.0):

        super().__init__()

        self.dim = dim
        self.hidden_dim = hidden_dim
        self.num_patches = num_patches

        self.net = nn.Sequential(
            nn.Linear(dim, hidden_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, dim),
            nn.Dropout(dropout)
        )

    def forward(
            self,
            x):

        return self.net(x)

class Attention(nn.Module):
    def __init__(
            self,
            dim,
            num_patches,
            num_heads=8,
            head_dim=64,
            dropout=0.0,
            is_lsa=False):

        super().__init__()
        inner_dim = head_dim * num_heads
        project_out = not (num_heads == 1 and head_dim == dim)
        self.num_patches = num_patches
        self.heads = num_heads
        self.scale = head_dim ** -0.5
        self.dim = dim
        self.inner_dim = inner_dim
        self.softmax = nn.Softmax(dim=-1)
        self.to_qkv = nn.Linear(self.dim, self.inner_dim * 3, bias=False)

        self.projection = nn.Sequential(
            nn.Linear(self.inner_dim, self.dim),
            nn.Dropout(dropout)
        ) if project_out else nn.Identity()

        if is_lsa:
            self.scale = nn.Parameter(self.scale * torch.ones(num_heads))
            self.mask = torch.eye(self.num_patches + 1, self.num_patches + 1)
            self.mask = torch.nonzero((self.mask == 1), as_tuple=False)
        else:
            self.mask = None

    def forward(
            self,
            x):

        b, n, _, h = *x.shape, self.heads
        qkv = self.to_qkv(x).chunk(3, dim=-1)
        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h), qkv)

        if self.mask is None:
            dots = torch.einsum(
                'b h i d, b h j d -> b h i j', q, k) * self.scale
        else:
            scale = self.scale
            dots = torch.mul(
                torch.einsum('b h i d, b h j d -> b h i j', q, k),
                scale.unsqueeze(0).unsqueeze(-1).unsqueeze(-1).expand((b, h, 1, 1)))
            mask_value = max_neg_value(dots)
            dots[:, :, self.mask[:, 0], self.mask[:, 1]] = mask_value

        attn = self.softmax(dots)
        out = torch.einsum('b h i j, b h j d -> b h i d', attn, v)

        out = rearrange(out, 'b h n d -> b n (h d)')

        return self.projection(out)

    def get_num_flops(
            self):
        flops = 0
        if not self.is_coord:
            flops += self.dim * self.inner_dim * 3 * (self.num_patches + 1)
        else:
            flops += (self.dim + 2) * self.inner_dim * 3 * self.num_patches
            flops += self.dim * self.inner_dim * 3

        return flops

class Transformer(nn.Module):
    def __init__(
            self,
            dim,
            num_patches,
            depth,
            num_heads,
            head_dim,
            mlp_dim_ratio,
            dropout=0.0,
            stochastic_depth=0.0,
            is_lsa=False):

        super().__init__()
        self.layers = nn.ModuleList([])
        self.scale = {}

        for _ in range(depth):
            self.layers.append(nn.ModuleList([
                PreNorm(
                    num_tokens=num_patches,
                    dim=dim,
                    fn=Attention(
                        dim=dim,
                        num_patches=num_patches,
                        num_heads=num_heads,
                        head_dim=head_dim,
                        dropout=dropout,
                        is_lsa=is_lsa)),
                PreNorm(
                    num_tokens=num_patches,
                    dim=dim,
                    fn=TransformerInnerMlp(
                        dim=dim, num_patches=num_patches,
                        hidden_dim=dim * mlp_dim_ratio,
                        dropout=dropout))
            ]))

        self.drop_path = DropPath(
            stochastic_depth) if stochastic_depth > 0 else nn.Identity()

    def forward(
            self,
            x):

        for i, (attn, ff) in enumerate(self.layers):
            x = self.drop_path(attn(x)) + x
            x = self.drop_path(ff(x)) + x
            self.scale[str(i)] = attn.fn.scale

        return x

class ViTEncoder(nn.Module):
    def __init__(
            self,
            *,
            in_img_size,
            in_channels=3,
            patch_size,
            spatio_temporal=False,
            in_num_frames=None,
            tubelet_size=None,
            embed_dim,
            depth,
            num_heads,
            mlp_dim_ratio,
            head_dim=16,
            dropout=0.0,
            pos_embed_dropout=0.0,
            stochastic_depth=0.0,
            class_embed=False,
            is_lsa=False,
            is_spt=False):

        super().__init__()

        in_img_size = make_pair_shape(in_img_size)
        patch_height, patch_width = make_pair_shape(patch_size)
        self.embed_dim = embed_dim
        self.class_embed = bool(class_embed)
        self.spatio_temporal = spatio_temporal
        self.tublet_size = tubelet_size

        if not is_spt:
            if self.spatio_temporal:
                self.patch_embedder = PatchEmbed3d(
                    in_img_size=in_img_size,
                    in_channels=in_channels,
                    in_num_frames=in_num_frames,
                    patch_height=patch_height,
                    patch_width=patch_width,
                    tubelet_size=tubelet_size,
                    embed_dim=embed_dim)
            else:
                self.patch_embedder = PatchEmbed2d(
                    in_img_size=in_img_size,
                    in_channels=in_channels,
                    patch_height=patch_height,
                    patch_width=patch_width,
                    embed_dim=embed_dim)
        else:
            assert patch_height == patch_width
            patch_size = patch_width
            if self.spatio_temporal:
                self.patch_embedder = ShiftedPatchEmbed3d(
                    in_img_size=in_img_size,
                    in_channels=in_channels,
                    in_num_frames=in_num_frames,
                    embed_dim=self.embed_dim,
                    patch_size=patch_size,
                    tubelet_size=tubelet_size)
            else:
                self.patch_embedder = ShiftedPatchEmbed2d(
                    in_img_size=in_img_size,
                    in_channels=in_channels,
                    embed_dim=self.embed_dim,
                    patch_size=patch_size)
        self.num_patches = self.patch_embedder.num_patches
        self.patch_dim = self.patch_embedder.patch_dim

        self.pos_embedding = nn.Parameter(
            torch.randn(1, self.num_patches + self.class_embed, self.embed_dim))
        if self.class_embed:
            self.class_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
        self.embed_dropout = nn.Dropout(pos_embed_dropout)

        self.transformer = Transformer(
            self.embed_dim,
            self.num_patches,
            depth,
            num_heads,
            head_dim,
            mlp_dim_ratio,
            dropout,
            stochastic_depth,
            is_lsa=is_lsa)
        self.norm = nn.LayerNorm(self.embed_dim)

    def forward(
            self,
            x,
            masked_indices=None):

        x = self.patch_embedder(x)
        b, n, d = x.shape

        if masked_indices is not None:
            x = torch.gather(
                x, dim=1, index=masked_indices.unsqueeze(-1).repeat(1, 1, d))

        if self.class_embed:
            class_tokens = repeat(self.class_token, '() n d -> b n d', b=b)
            x = torch.cat((class_tokens, x), dim=1)

        x = x + self.pos_embedding

        x = self.embed_dropout(x)
        x = self.transformer(x)
        x = self.norm(x)

        if self.class_embed:
            # remove cls token
            x = x[:, 1:, :]

        return x

class ViTDecoder(nn.Module):
    def __init__(
            self,
            *,
            in_latent_dim,
            in_num_patches,
        out_patch_dim,
            embed_dim,
            depth,
            num_heads,
            mlp_dim_ratio,
            head_dim=16,
            dropout=0.0,
            pos_embed_dropout=0.0,
            stochastic_depth=0.0,
            class_embed=False,
            use_masking=False,
            is_lsa=False):

        super().__init__()

        self.out_patch_dim = out_patch_dim
        self.in_latent_dim = in_latent_dim
        self.in_num_patches = in_num_patches

        self.input_dim = in_latent_dim
        self.embed_dim = embed_dim
        self.class_embed = bool(class_embed)
        self.use_masking = bool(use_masking)

        self.encoder_to_decoder = nn.Linear(
            self.input_dim, self.embed_dim, bias=True)

        if self.class_embed:
            self.class_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
        if self.use_masking:
            self.mask_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim))
        self.pos_embedding = nn.Parameter(
            torch.randn(1, self.in_num_patches + self.class_embed + self.use_masking, self.embed_dim))
        self.embed_dropout = nn.Dropout(pos_embed_dropout)

        self.transformer = Transformer(
            self.embed_dim,
            self.in_num_patches,
            depth,
            num_heads,
            head_dim,
            mlp_dim_ratio,
            dropout,
            stochastic_depth,
            is_lsa=is_lsa)

        self.norm = nn.LayerNorm(self.embed_dim)
        self.pred = nn.Linear(
            self.embed_dim, self.out_patch_dim, bias=True)  # decoder to patch

    def forward(
            self,
            x,
            unshuffle_indices=None):

        # [b, n, latent_dim]
        x = self.encoder_to_decoder(x)
        b, n, d = x.shape

        if self.use_masking:
            x = torch.cat((x, self.mask_token.expand(b, -1, -1)), dim=1)
            x = torch.gather(
                x, dim=1, index=unshuffle_indices.unsqueeze(-1).repeat(1, 1, d))  # unshuffle

        if self.class_embed:
            class_tokens = repeat(self.class_token, '() n d -> b n d', b=b)
            x = torch.cat((class_tokens, x), dim=1)

        x = x + self.pos_embedding
        x = self.embed_dropout(x)
        x = self.transformer(x)
        x = self.norm(x)  # [b, n, d]
        x = self.pred(x)  # [b, n, c * p0 * p1 * p2]

        if self.class_embed:
            # remove cls token
            x = x[:, 1:, :]

        return x

class ViTAutoEncoder(nn.Module):
    def __init__(
            self,
            *,
            in_img_size,
            in_channels=3,
            patch_size,
            spatio_temporal=False,
            in_num_frames=None,
            tubelet_size=None,
            encoder_embed_dim,
            encoder_depth,
            encoder_num_heads,
            decoder_embed_dim,
            decoder_depth,
            decoder_num_heads,
            mlp_dim_ratio,
            head_dim=16,
            dropout=0.0,
            pos_embed_dropout=0.0,
            stochastic_depth=0.0,
            class_embed=False,
            is_lsa=False,
            is_spt=False,
            use_masking=False):

        super().__init__()

        self.use_masking = use_masking
        if self.use_masking:
            self.mask_token = nn.Parameter(
                torch.zeros(1, 1, decoder_embed_dim))

        self.encoder = ViTEncoder(
            in_img_size=in_img_size,
            in_channels=in_channels,
            patch_size=patch_size,
            spatio_temporal=spatio_temporal,
            in_num_frames=in_num_frames,
            tubelet_size=tubelet_size,
            embed_dim=encoder_embed_dim,
            depth=encoder_depth,
            num_heads=encoder_num_heads,
            mlp_dim_ratio=mlp_dim_ratio,
            head_dim=head_dim,
            dropout=dropout,
            pos_embed_dropout=pos_embed_dropout,
            stochastic_depth=stochastic_depth,
            class_embed=class_embed,
            is_lsa=is_lsa,
            is_spt=is_spt)

        self.decoder = ViTDecoder(
            in_latent_dim=encoder_embed_dim,
            in_num_patches=self.encoder.num_patches,
            out_patch_dim=self.encoder.patch_dim,
            embed_dim=decoder_embed_dim,
            depth=decoder_depth,
            num_heads=decoder_num_heads,
            mlp_dim_ratio=mlp_dim_ratio,
            head_dim=head_dim,
            dropout=dropout,
            pos_embed_dropout=pos_embed_dropout,
            stochastic_depth=stochastic_depth,
            class_embed=class_embed,
            use_masking=use_masking,
            is_lsa=is_lsa)

    def forward(
            self,
            x,
            mask_info=None):

        if self.use_masking:
            _, _, unshuffle_indices, masked_indices = mask_info
        else:
            unshuffle_indices, masked_indices = None, None

        x_patched = self.encoder.patch_embedder.patchify(x)
        latent_patched = self.encoder(x, masked_indices=masked_indices)
        x_hat_patched = self.decoder(
            latent_patched, unshuffle_indices=unshuffle_indices)

        return latent_patched, x_patched, x_hat_patched

class ViTClassifierHead(nn.Module):
    def __init__(
            self,
            input_dim,
            num_classes):

        super().__init__()

        self.input_dim = input_dim
        self.num_classes = num_classes

        self.net = nn.Sequential(
            nn.LayerNorm(self.input_dim),
            nn.Linear(self.input_dim, self.num_classes)
        )

    def forward(
            self,
            x):

        return self.net(x)

class ViTClassifier(nn.Module):
    def __init__(
            self,
            *,
            in_img_size,
            in_channels=3,
            patch_size,
            spatio_temporal=False,
            in_num_frames=None,
            tubelet_size=None,
            num_classes,
            encoder_embed_dim,
            encoder_depth,
            encoder_num_heads,
            mlp_dim_ratio,
            head_dim=16,
            dropout=0.0,
            pos_embed_dropout=0.0,
            head_dropout=0.0,
            stochastic_depth=0.0,
            class_embed=False,
            is_lsa=False,
            is_spt=False,
            use_masking=False):

        super().__init__()

        self.use_masking = use_masking

        self.encoder = ViTEncoder(
            in_img_size=in_img_size,
            in_channels=in_channels,
            patch_size=patch_size,
            spatio_temporal=spatio_temporal,
            in_num_frames=in_num_frames,
            tubelet_size=tubelet_size,
            embed_dim=encoder_embed_dim,
            depth=encoder_depth,
            num_heads=encoder_num_heads,
            mlp_dim_ratio=mlp_dim_ratio,
            head_dim=head_dim,
            dropout=dropout,
            pos_embed_dropout=pos_embed_dropout,
            stochastic_depth=stochastic_depth,
            class_embed=class_embed,
            is_lsa=is_lsa,
            is_spt=is_spt)

        self.head_norm = nn.LayerNorm(encoder_embed_dim)
        self.head_dropout = nn.Dropout(head_dropout)
        self.head = ViTClassifierHead(
            input_dim=encoder_embed_dim,
            num_classes=num_classes)

    def forward(
            self,
            x,
            mask_info):

        if self.use_masking:
            _, _, _, masked_indices = mask_info
        else:
            masked_indices = None

        latent = self.encoder(x, masked_indices)

        latent = latent.mean(dim=1)  # patch-wise average pooling
        latent = self.head_norm(latent)
        latent = self.head_dropout(latent)

        pred = self.head(latent)

        return pred


import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.optim as optim
import torch.nn.functional as F
from model import Net

class Controller(nn.Module):

    def __init__(self, num_actions=10, hidden_size=64):
        super(Controller, self).__init__()

        self.cell = nn.GRUCell(
            input_size=num_actions,
            hidden_size=hidden_size
        )

        self.fc = nn.Linear(
            in_features=hidden_size,
            out_features=num_actions
        )

        self.num_actions = num_actions
        self.hidden_size = hidden_size
        self.epsilon = 0.8
        self.gamma = 1.0
        self.beta = 0.01
        self.max_depth = 6
        self.clip_norm = 0
        self.log_probs = []
        self.actions = []
        self.entropies = []
        self.reward = None

        self.index_to_action = {
            0: 1,
            1: 2,
            2: 4,
            3: 8,
            4: 16,
            5: 'Sigmoid',
            6: 'Tanh',
            7: 'ReLU',
            8: 'LeakyReLU',
            9: 'EOS'
        }

        self.optimizer = optim.Adam(self.parameters(), lr=1e-2)

    def forward(self, x, h):
        x = x.unsqueeze(dim=0)
        h = h.unsqueeze(dim=0)

        h = self.cell(x, h)
        x = self.fc(h)

        x = x.squeeze(dim=0)
        h = h.squeeze(dim=0)

        return x, h

    def step(self, state):
        logits, new_state = self(torch.zeros(self.num_actions), state)

        idx = torch.distributions.Categorical(logits=logits).sample()
        probs = F.softmax(logits, dim=-1)
        log_probs = torch.log(probs)

        action = self.index_to_action[int(idx)]
        self.actions.append(action)

        entropy = -(log_probs * probs).sum(dim=-1)
        self.entropies.append(entropy)

        if action == 'EOS' and len(self.actions) <= 2:
            self.reward -= 1
        elif len(self.actions) >= 2 and isinstance(self.actions[-1], int) and isinstance(self.actions[-2], int):
            self.reward -= 0.1
        elif len(self.actions) >= 2 and isinstance(self.actions[-1], str) and isinstance(self.actions[-2], str) and action != 'EOS':
            self.reward -= 0.1

        terminate = action == 'EOS' or len(self.actions) == self.max_depth

        return log_probs[idx], new_state, terminate

    def generate_rollout(self, iter_train, iter_dev, verbose=False):
        self.log_probs = []
        self.actions = []
        self.entropies = []
        self.reward = None

        state = torch.zeros(self.hidden_size)
        terminated = False
        self.reward = 0

        while not terminated:
            log_prob, state, terminated = self.step(state)
            self.log_probs.append(log_prob)

        if verbose:
            print('\nGenerated network:')
            print(self.actions)

        net = Net(self.actions)
        accuracy = net.fit(iter_train, iter_dev)
        self.reward += accuracy

        return self.reward

    def optimize(self):
        G = torch.ones(1) * self.reward
        loss = 0

        for i in reversed(range(len(self.log_probs))):
            G = self.gamma * G
            loss = loss - (self.log_probs[i]*Variable(G)) - self.beta * self.entropies[i]

        loss /= len(self.log_probs)

        self.optimizer.zero_grad()
        loss.backward()

        if self.clip_norm > 0:
            nn.utils.clip_grad_norm_(self.parameters(), self.clip_norm)

        self.optimizer.step()

        return float(loss.data.numpy())


import os
import abc

import matplotlib.pyplot as plt
import torch
import torchvision
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm

class GANTrainer:

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None,
                 ):
        self._generator = generator.to(device)
        self._discriminator = discriminator.to(device)
        self._device = device
        self._g_optimizer = g_optimizer
        self._d_optimizer = d_optimizer
        self._writer = writer
        self._model_path = model_path
        self._output_dir = output_dir
        self._n_latent = n_latent

        self._global_step = 0
        self._z_fixed = torch.autograd.Variable(torch.randn(64, self._n_latent)).to(self._device)

        self._writer = SummaryWriter(log_dir=self._model_path)

    @abc.abstractmethod
    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        raise NotImplementedError()

    @abc.abstractmethod
    def _g_loss(self, fake_logits):
        raise NotImplementedError()

    @abc.abstractmethod
    def _clip_weights(self):
        pass

    def train(self, train_loader, n_epochs=1, n_critic=8, interval=100):
        for epoch in range(n_epochs):
            d_loss = g_loss = 0.0
            for i, data in enumerate(tqdm(train_loader)):
                img = data.to(self._device)
                batch_size = img.shape[0]

                # Train D
                self._discriminator.train()
                self._discriminator.zero_grad()
                z = torch.autograd.Variable(torch.randn(batch_size, self._n_latent)).to(self._device)
                real_image = img
                real_logits = self._discriminator(real_image)

                fake_image = self._generator(z)
                fake_logits = self._discriminator(fake_image)

                d_loss = self._d_loss(real_logits, fake_logits, real_image, fake_image)

                d_loss.backward()
                self._d_optimizer.step()

                self._clip_weights()

                # Train G
                # train generator every n_critic steps
                if self._global_step % n_critic == 0:
                    self._generator.train()
                    z = torch.autograd.Variable(torch.randn(batch_size, self._n_latent)).to(self._device)
                    fake_image = self._generator(z)
                    fake_logits = self._discriminator(fake_image)
                    g_loss = self._g_loss(fake_logits)

                    self._generator.zero_grad()
                    g_loss.backward()
                    self._g_optimizer.step()

                if self._global_step % interval == 0:
                    self._generator.eval()
                    fake_images = (self._generator(self._z_fixed).data + 1) / 2
                    self._writer.add_scalar('d_loss', d_loss.item(), self._global_step)
                    self._writer.add_scalar('g_loss', g_loss.item(), self._global_step)
                    self._writer.add_image('fake_image', torchvision.utils.make_grid(fake_images), self._global_step)

                self._global_step += 1

            print(f'Discriminator Loss: {d_loss.data:.4f}, Generator Loss: {g_loss.data:.4f}')

            if self._model_path:
                torch.save(self._generator.state_dict(), os.path.join(self._model_path, 'generator.pth'))
                torch.save(self._discriminator.state_dict(), os.path.join(self._model_path, 'discriminator.pth'))

class VanillaGANTrainer(GANTrainer):

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer, criterion,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None,
                 ):
        super().__init__(generator, discriminator, g_optimizer, d_optimizer,
                         n_latent, device, writer, model_path, output_dir)
        self._criterion = criterion

    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        real_labels = torch.ones(real_logits).to(self._device)
        real_loss = self._criterion(real_logits, real_labels)
        fake_labels = torch.zeros(fake_logits).to(self._device)
        fake_loss = self._criterion(fake_logits, fake_labels)
        return real_loss + fake_loss

    def _g_loss(self, fake_logits):
        fake_labels = torch.ones(fake_logits).to(self._device)
        return self._criterion(fake_logits, fake_labels)

class WGANTrainer(GANTrainer):

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None, clip_value=0.01):
        super().__init__(generator, discriminator, g_optimizer, d_optimizer,
                         n_latent, device, writer, model_path, output_dir)
        self._clip_value = clip_value

    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        return -torch.mean(real_logits) + torch.mean(fake_logits)

    def _g_loss(self, fake_logits):
        return -torch.mean(fake_logits)

    def _clip_weights(self):
        for p in self._discriminator.parameters():
            p.data.clamp_(-self._clip_value, self._clip_value)

class GPGANTrainer(GANTrainer):

    def __init__(self, generator, discriminator, g_optimizer, d_optimizer,
                 n_latent=100, device='cpu', writer=None, model_path=None, output_dir=None, gp_weight=10,
                 ):
        super().__init__(generator, discriminator, g_optimizer, d_optimizer,
                         n_latent, device, writer, model_path, output_dir)
        self._gp_weight = gp_weight

    def _gradient_penalty(self, real, fake):
        batch_size = real.shape[0]
        # choose random interpolation point
        alpha = torch.rand(batch_size, 1, 1, 1).to(self._device)
        interpolates = (alpha * real + ((1 - alpha) * fake)).requires_grad_(True)
        # calculate probability of interpolates
        interpolates_prob = self._discriminator(interpolates)
        # calculate gradients of probabilities with respect to interpolates
        fake = torch.autograd.Variable(torch.ones(interpolates_prob.size()), requires_grad=True).to(self._device)
        gradients = torch.autograd.grad(outputs=interpolates_prob, inputs=interpolates,
                                        grad_outputs=fake, create_graph=True, retain_graph=True, only_inputs=True)[0]
        gradients = gradients.view(batch_size, -1)
        gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean()
        return gradient_penalty

    def _d_loss(self, real_logits, fake_logits, real_images, fake_images):
        gradient_penalty = self._gp_weight * self._gradient_penalty(real_images, fake_images)
        return -torch.mean(real_logits) + torch.mean(fake_logits) + gradient_penalty

    def _g_loss(self, fake_logits):
        return -torch.mean(fake_logits)


"""
test_dates_etl_job.py
~~~~~~~~~~~~~~~

This module contains unit tests for the transformation steps of the ETL
job defined in dates_etl_job.py. It makes use of a local version of PySpark
that is bundled with the PySpark package.
"""
import unittest

from datetime import datetime
from jobs.ETL.dates_etl_job import transform_data
import pyspark.sql.types as t
from tests.common import SparkETLTests
from tests.data import CleanedData

class DatesETLTests(SparkETLTests):
    """Test suite for transformation in dates_etl_job.py"""

    def test_transform_data(self):
        """Test data transformer.

        Using small chunks of input data and expected output data, we
        test the transformation step to make sure it's working as
        expected.
        """
        # assemble

        patent_cleaned_df = self.spark.createDataFrame(
            data=CleanedData.patent, schema=CleanedData.patent_schema
        )

        expected_data = [(datetime(2011, 1, 3), 2011), (datetime(2000, 9, 5), 2000)]
        expected_data_schema = t.StructType(
            [
                t.StructField("date", t.DateType(), False),
                t.StructField("year", t.IntegerType(), False),
            ]
        )

        expected_data_df = self.spark.createDataFrame(
            data=expected_data, schema=expected_data_schema
        )

        data_transformed = transform_data(patent_cleaned_df)

        self.check_schema(data_transformed, expected_data_df)
        self.check_data(data_transformed, expected_data_df)

if __name__ == "__main__":
    unittest.main()


import os
import socket
import netifaces
from queue import Queue
from threading import Thread
from scapy.all import srp, Ether, ARP, conf

# Get the IP address of the default gateway
gateway_ip = netifaces.gateways()['default'][netifaces.AF_INET][0]
print("_"*100+"\n")
print(f'Your gateway address is {gateway_ip}')
print("\n")

ip_range = '.'.join(gateway_ip.split('.')[:-1])

# Get the IP address of the local interface
interface_name = netifaces.gateways()['default'][netifaces.AF_INET][1]
interface_ip = netifaces.ifaddresses(interface_name)[netifaces.AF_INET][0]['addr']

# Print your IP address
print("_"*100+"\n")
print("Your IP address is:", interface_ip)
print("\n")

print("_"*100+"\n")
print("LIVE IP ADDRESSES ON YOUR SUBNET:")
print("\n")

def do_stuff(q):
    while True:
        ip=ip_range + "." + str(q.get())
        response = os.popen(f"ping  {ip} -c 1").read()
        if "1 packets received" in response:
            try:
                hostname = socket.gethostbyaddr(ip)[0]
                print(" {}ONLINE".format( hostname))
            except socket.error:
                hostname = "No HOST NAME "
                print(" {}{} ONLINE".format( hostname,ip))
        q.task_done()

q = Queue(maxsize=0)
num_threads = 100

for i in range(num_threads):
    worker = Thread(target=do_stuff, args=(q,))
    worker.setDaemon(True)
    worker.start()

for x in range(1,256):
    q.put(x)

q.join()

# Check to see if the queue is empty. if it is print 'Done!'
if q.qsize() == 0:
    print('Done!')


import medleydb as mdb
import numpy as np
import os, random, json, sys
from utils import log, binarize
import glob
import matplotlib
matplotlib.use('agg')
import librosa
import librosa.core as dsp
import librosa.display
import matplotlib.pyplot as plt
import sklearn
import re, csv
from scipy.signal import upfirdn
from scipy.ndimage import filters
from keras import backend as K
from model import model
from keras.models import Model
from keras.layers import BatchNormalization, Conv2D, Input, Flatten, Lambda
from predict_on_audio import load_model

FMIN = 32.7
HOP_LENGTH = 256
Fs = 22050#44100#22050
HARMONICS = [0.5, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
TASK = "MELODY2"
NCLASSES = 72

class DataSet(object):

    def __init__(self, inPath, outPath, targetDim):
        self.inputPath = inPath
        self.targetPath = outPath
        self.mtracks = mdb.load_all_multitracks(dataset_version = 'V1')
        self.trackList = mdb.TRACK_LIST_V1
        self.longest = 0
        if "rachel" in targetDim or "BASELINE" in targetDim or "MULTILABEL" in targetDim:
            self.deepModel = load_model('melody2')
            self.deepModel.predict(np.zeros((2,360,50,6))) # Test/Initialize ?
            # print(self.deepModel.summary())

    ### ---------------- CREATE DATASET OF FEATURES FROM AUDIO ---------------- ###
    def getFeature(self, dataSet, modelDim, outPath, binsPerOctave, nOctave, nHarmonics=1, homemade=False):

        log('Creating features from Audio')
        if os.path.isdir('/data2/anasynth_nonbp/laffitte'):
            audioPath = '/data2/anasynth_nonbp/laffitte/MedleyDB/Audio/'
            annotPath = '/data2/anasynth_nonbp/laffitte/MedleyDB/Annotations/Melody_Annotations/'+TASK
        else:
            annotPath = '/net/as-sdb/data/mir2/MedleyDB/Annotations/Melody_Annotations/'+TASK
            audioPath = '/net/as-sdb/data/mir2/MedleyDB/Audio/'
        self.trackList = []
        dirList = sorted(os.listdir(audioPath))
        dirList=[i for i in dirList if '.' not in i]
        H = []
        for j in dirList:  # loop over all audio files
            fileList = sorted(os.listdir(os.path.join(audioPath, j)))
            fileList = [k[:-8] for k in fileList if re.match('[^._].*?.wav', k)]
            tracks = [tr for tr in fileList if tr in dataSet]
            for i in tracks:
                prefix = i.replace('_MIX.wav', '')
                featPath = os.path.join(outPath, 'features')
                targPath = os.path.join(outPath, 'targets')
                if not os.path.isdir(featPath):
                    os.mkdir(featPath)
                if not os.path.isdir(targPath):
                    os.mkdir(targPath)
                inFile = os.path.join(featPath, '{}_mel2_input.npy'.format(prefix))
                outFile = os.path.join(targPath, '{}_mel2_target.npy'.format(prefix))
                if not os.path.exists(inFile) or not os.path.exists(outFile):
                    audioFile = os.path.join(audioPath, os.path.join(j, i+'_MIX.wav'))
                    annotFile = os.path.join(annotPath, i+'_'+TASK+'.csv')
                    if annotFile is not None and os.path.exists(annotFile):
                        ### Compute CQT
                        if not os.path.exists(inFile):
                            if homemade:
                                signal, fs = librosa.load(audioFile) # load signal
                                ''' Get CQT feature of whole signal according to the HCQT (Harmonic CQT) method described in "Deep Salience Representation for F0 Estimation in Polyphonic Music" by Rachel M. Bittner, Brian McFee, Justin Salamon, Peter Li, Juan P. Bello. The k-th CQT computes the CQT of the (k-1)-th harmonic of C1 '''
                                for k in range(nHarmonics): # loop over number of harmonics desired
                                    H.append(dsp.cqt(signal, Fs, fmin=(k+1)*FMIN, n_bins=binsPerOctave*nOctave, bins_per_octave=binsPerOctave, filter_scale=0.9, hop_length=HOP_LENGTH))
                            else: # use rachel's feature extraction
                                H = computeHcqt(audioFile, nHarmonics, 60, 360)
                            np.save(inFile, H.astype(np.float32))
                        if not os.path.exists(outFile):
                            ### Get labels
                            data = readAnnotation(annotFile)
                            annot = np.asarray(np.array(data).T[0])
                            annot.reshape((np.shape(annot)[0], np.shape(annot)[1]))
                            times = annot[:,0]
                            freqs = annot[:,1]
                            freq_grid = librosa.cqt_frequencies(
                                NCLASSES, FMIN, bins_per_octave=NCLASSES/nOctave
                                )
                            length = int(np.floor(len(times)*(Fs/44100)))
                            time_grid = librosa.core.frames_to_time(
                                range(int(length)), sr=Fs, hop_length=HOP_LENGTH
                            )
                            target = createAnnotation(freq_grid[:-1], time_grid, times, freqs)
                            np.save(outFile, target.astype(np.float32))
                self.trackList.append(i.replace('_MIX.wav', '')) # Add current track to track list
                self.inputPath = featPath # Set feature path
                self.targetPath = targPath # Set target path

    def partDataset(self):

        # data_splits_path = '/net/inavouable/u.anasynth/laffitte/Code/ismir2017-deepsalience/outputs/data_splits.json'
        data_splits_path = '/u/anasynth/laffitte/datasplits.json'
        with open(data_splits_path, 'r') as fhandle:
            data_splits = json.load(fhandle)
        ### Get test set from Rachel's paper data
        testSet = data_splits['test']
        trainSet = data_splits['train']
        validSet = data_splits['validation']
        ### GET TRAIN AND VALID DATASET FROM THE REST OF MedleyDB DATASET
        # trainSet, validSet = sklearn.model_selection.train_test_split(restSet, train_size=0.70)
        return trainSet, validSet, testSet

    def sizeDataset(self, dataset, batchSize, rnnBatch=16):

        nTracks = 0 # count number of tracks
        nSamples = 0 # count number individual training examples
        nBlocks = 0 # number of batches
        bucketList = self.bucketDataset(dataset, rnnBatch)
        for (s, subTracks) in enumerate(bucketList):
            self.mtracks = mdb.load_all_multitracks(dataset_version = 'V1')
            tracks = [tr.track_id for tr in self.mtracks if tr.track_id in self.trackList]
            tracks = [tr for tr in tracks if tr in subTracks[0]]
            longest, _ = self.findLongest(tracks)
            nSequences = int(np.floor(longest/batchSize))
            nBlocks += nSequences
            for song in tracks:
                length, _ = self.findLongest([song])
                nSamples += length
            nTracks += len(tracks)
        return [nSamples, nBlocks, nTracks]

    def findLongest(self, dataset):

        longest = 0
        name = ''
        songDic = {}
        for track in dataset:
            i = glob.glob(os.path.join(self.inputPath, '{}_mel2_input.npy'.format(track)))
            j = glob.glob(os.path.join(self.targetPath, '{}_mel2_target.npy'.format(track)))
            if i and j:
                curTarget = np.load(j[0])
                if curTarget.shape[-1] >= longest:
                    longest = curTarget.shape[-1]
                    name = track
                songDic[track] = curTarget.shape[-1]
        return longest, name

    def bucketDataset(self, dataset, size):

        sortedList = []
        bucketList = []
        dataList = dataset[:]
        for k in range(len(dataset)):
            L, longest = self.findLongest(dataList)
            if L != 0:
                sortedList.append(longest)
                dataList.remove(longest)
        for k in range(int(np.floor(len(dataset)/size))+1):
            bucketList.append([])
            bucketList[-1].append(sortedList[k*size: k*size+size])

        return bucketList

    def formatDataset(self, myModel, dataset, timeDepth, targetDim, batchSize, hopSize, fftSize, nHarmonics, binsPerOctave, nOctave, rnnBatch=16, stateFull=True):

        self.mtracks = mdb.load_all_multitracks(dataset_version = 'V1')
        tracks = [tr.track_id for tr in self.mtracks if tr.track_id in self.trackList]
        tracks = [tr for tr in tracks if tr in dataset]
        # tracks = dataset
        while 1:
            bucketList = self.bucketDataset(tracks, rnnBatch)
            for (s, subTracks) in enumerate(bucketList):
                if stateFull:
                    # log("Resetting model's states")
                    myModel.reset_states()
                if "SOFTMAX" in targetDim or "CATEGORICAL" in targetDim:
                    binary = True
                    voicing = True
                longest, _ = self.findLongest(subTracks[0])
                nSequences = int(np.floor(longest/batchSize))
                nOuts = NCLASSES
                offset = 0
                for b in range(nSequences): # Iterate over total number of batches and fill them up 1-b-1
                    if "BASELINE" in targetDim or "MULTILABEL" in targetDim or "RNN" in targetDim:
                        inputs = -1 * np.ones((rnnBatch, batchSize, fftSize))
                    else:
                        inputs = -1 * np.ones((rnnBatch, batchSize, fftSize, timeDepth, nHarmonics))
                    if "1D" in targetDim or "SOFTMAX" in targetDim or "BASELINE" in targetDim or "RNN" in targetDim:
                        targets = 0 * np.ones((rnnBatch, batchSize, nOuts))
                    elif "2D" in targetDim:
                        targets = 0 * np.ones((rnnBatch, batchSize, nOuts, timeDepth))
                    elif "MULTILABEL" in targetDim: ### USING SEPARATE LABEL FOR PITCH AND OCTAVE DETECTION
                        targets = -1 * np.ones((rnnBatch, batchSize, nOuts))
                        targetNote = -1 * np.ones((rnnBatch, batchSize, binsPerOctave, timeDepth))
                        targetOctave = -1 * np.ones((rnnBatch, batchSize, nOctave, timeDepth))
                    for (k, track) in enumerate(sorted(subTracks[0])):
                        i = glob.glob(os.path.join(self.inputPath, '{}_mel2_input.npy'.format(track)))
                        j = glob.glob(os.path.join(self.targetPath, '{}_mel2_target.npy'.format(track)))
                        if i and j:
                            curInput = np.load(i[0])
                            curTarget = np.load(j[0])
                            if curInput.shape[-1] != curTarget.shape[-1]:
                                lim = np.min((curInput.shape[-1], curTarget.shape[-1]))
                                curInput = curInput[:,:,:lim]
                                curTarget = curTarget[:,:lim]
                            # curInput = zero_pad(curInput, True, timeDepth)
                            if "1D" in targetDim or "CATEGORICAL" in targetDim:
                                if offset+batchSize+timeDepth < curInput.shape[-1]:
                                    for kk in range(0, batchSize, hopSize):
                                        if len(curInput.shape) == 2:
                                            temp = curInput[None, :, offset+kk:offset+kk+timeDepth]
                                        else:
                                            temp = curInput[:,:,offset+kk:offset+kk+timeDepth]
                                        inputs[k,kk] = temp.transpose(1,2,0)
                                    tar = curTarget[:,offset:offset+batchSize].transpose(1,0)
                            elif "BASELINE" in targetDim or "2D" in targetDim or "MULTILABEL" in targetDim:
                                if offset+batchSize < curTarget.shape[-1]:
                                    temp = curInput[:,:,offset:offset+batchSize]
                                    temp = temp.transpose(1, 2, 0)[None,:,:,:]
                                    if "BASELINE" in targetDim or "MULTILABEL" in targetDim:
                                        temp = self.deepModel.predict(temp)
                                    inputs[k,:,:] = temp[0,:,:].transpose(1,0)
                                    tar = curTarget[:,offset:offset+batchSize].transpose(1,0)
                            elif "RNN" in targetDim:
                                if offset+batchSize < curTarget.shape[-1]:
                                    inputs[k,:,:] = curInput[0,:,offset:offset+batchSize].transpose(1,0)
                                    tar = curTarget[:,offset:offset+batchSize].transpose(1,0)
                            targets[k,:,:] = tar
                    offset += batchSize
                    if "MULTILABEL" in targetDim:
                        targetNote, targetOctave, voicingArray = splitTarget(targets, nOctave, binsPerOctave, voicing)
                        yield inputs, [targetNote, targetOctave]
                    else:
                        yield inputs, targets

    def toyData(self, myModel, dataset, rnnBatch, batchSize, targetDim, fftSize, stateFull):
        while 1:
            bucketList = self.bucketDataset(dataset, rnnBatch)
            for (s, subTracks) in enumerate(bucketList):
                if stateFull:
                    log("Resetting model's states")
                    myModel.reset_states()
                if "SOFTMAX" in targetDim or "CATEGORICAL" in targetDim:
                    binary = True
                    voicing = True
                self.mtracks = mdb.load_all_multitracks(dataset_version = 'V1')
                tracks = [tr.track_id for tr in self.mtracks if tr.track_id in self.trackList]
                tracks = [tr for tr in tracks if tr in subTracks[0]]
                longest, _ = self.findLongest(tracks)
                nSequences = int(np.floor(longest/batchSize))
                offset = 0
                for b in range(nSequences):
                    labels = -1 * np.ones((rnnBatch, batchSize, fftSize))
                    inputs = -1 * np.ones((rnnBatch, batchSize, fftSize))
                    for l in range(2, rnnBatch):
                        inputs[l,:,:] = l * np.ones((batchSize, fftSize))
                        labels[l,:,:] = l * np.ones((batchSize, fftSize))

                    yield inputs, labels

def splitTarget(labels, nOctave, binsPerOctave, voicing):

    shape = labels.shape
    note = np.zeros((shape[0], shape[1], binsPerOctave))
    octave = np.zeros((shape[0], shape[1], nOctave))
    voicingArray = np.zeros((shape[0], shape[1], 1))
    for b in range(shape[0]):
        for f in range(shape[1]):
            if labels[b,f,:].nonzero()[0].any() and not any(labels[b,f,:]==-1):
                if not voicing:
                    index = (labels[b,f,:].nonzero()[0])
                else:
                    index = labels[b,f,:].nonzero()[0] - 1
                curOctave = int(np.floor(index/(binsPerOctave)))
                curNote = int(np.floor(index%binsPerOctave))
                octave[b,f,curOctave] = 1
                note[b,f,curNote] = 1
                voicingArray[b,f,:] = 1

    return note, octave, voicingArray

def mergeTarget(note, octave):
    shape = note.shape
    binsPerOctave = shape[2]
    nOctave = octave.shape[2]
    target = np.zeros((shape[0], shape[1], binsPerOctave*nOctave))
    for b in shape[0]:
        for f in shape[1]:
            if note[b,f,:].nonzero()[0].any():
                curNote = note[b,f,:].nonzero()[0]
                curOctave = octave[b,f,:].nonzero()[0]
                target[b,f,:] = octave*note

def getDeepSaliencePredictions(dataobj, dataset, params, modelDim, fftSize, rnnBatch):
    log('Getting labels and inputs for plotting and score calculation')
    batchSize = int(params['batchSize'])
    timeDepth = int(params['timeDepth'])
    nHarmonics = int(params['nHarmonics'])
    hopSize = int(params['hopSize'])
    stateFull = True if params['stateFull']=="True" else False
    trackList = []
    preds = []
    bucketList = dataobj.bucketDataset(dataset, rnnBatch)
    for (s, subTracks) in enumerate(bucketList):
        for (k, track) in enumerate(sorted(subTracks[0])):
            trackList.append(track)
    for track in trackList:
        print("Predicting on :", track)
        i = glob.glob(os.path.join(dataobj.inputPath, '{}_mel2_input.npy'.format(track)))
        curInput = np.load(i[0])
        input_hcqt = curInput.transpose(1, 2, 0)[np.newaxis, :, :, :]
        n_t = input_hcqt.shape[2]
        n_slices = 200
        t_slices = list(np.arange(0, n_t, n_slices))
        output_list = []
        for i, t in enumerate(t_slices):
            prediction = dataobj.deepModel.predict(input_hcqt[:, :, t:t+n_slices, :])
            output_list.append(prediction[0, :, :])
        preds.append(np.hstack(output_list))
    return preds, trackList

def getLabelMatrix(myModel, dataobj, dataset, params, modelDim, fftSize, rnnBatch):
    batchSize = int(params['batchSize'])
    timeDepth = int(params['timeDepth'])
    nHarmonics = int(params['nHarmonics'])
    hopSize = int(params['hopSize'])
    binsPerOctave = int(params['binsPerOctave'])
    nOctave = int(params['nOctave'])
    stateFull = True if params['stateFull']=="True" else False
    gen = dataobj.formatDataset(myModel, dataset, int(timeDepth), modelDim, batchSize, hopSize, fftSize, nHarmonics, binsPerOctave, nOctave, rnnBatch, stateFull)
    nSamples, size, length = dataobj.sizeDataset(dataset, batchSize, rnnBatch)
    trackList = []
    bucketList = dataobj.bucketDataset(dataset, rnnBatch)
    for (s, subTracks) in enumerate(bucketList):
        for (k, track) in enumerate(sorted(subTracks[0])):
            trackList.append(track)
    if nSamples != 0:
        if "MULTILABEL" in modelDim:
            labelNote = None
            labelOctave = None
        else:
            labels = None
        inputs = None
        for l in range(size):
            one, two = gen.__next__()
            if "MULTILABEL" in modelDim:
                if labelNote is None:
                    labelNote = two[0]
                    labelOctave = two[1]
                    inputs = one
                else:
                    labelNote = np.concatenate((labelNote, two[0]))
                    labelOctave = np.concatenate((labelOctave, two[1]))
                    inputs = np.concatenate((inputs, one))
            else:
                if labels is None:
                    labels = two
                    if not "1D" in modelDim:
                        inputs = one
                else:
                    labels = np.concatenate((labels, two))
                    if not "1D" in modelDim:
                        inputs = np.concatenate((inputs, one))
    if "MULTILABEL" in modelDim:
        return labelNote, labelOctave, inputs, trackList
    else:
        return labels, inputs, trackList

def createAnnotation(freq_grid, time_grid, annotation_times,
                             annotation_freqs):
    """ Create the binary annotation target labels from the frequency Annotations
    --- The target vector, of size 72, is built so that it covers a range of frequencies from 0Hz (non-melody) to 1864Hz
    """
    n_freqs = len(freq_grid)
    n_times = len(time_grid)
    idx = np.array(np.arange(0, len(annotation_times), 44100/Fs), dtype='int32')
    annotation_times = annotation_times[idx]
    annotation_freqs = annotation_freqs[idx]

    time_bins = grid_to_bins(time_grid, 0.0, time_grid[-1])
    freq_bins = grid_to_bins(freq_grid, 0.0, freq_grid[-1])
    annot_time_idx = np.digitize(annotation_times, time_grid) - 1
    annot_freq_idx = np.digitize(annotation_freqs, freq_grid)
    annotation_target = np.zeros((NCLASSES, n_times))
    annotation_target[annot_freq_idx, annot_time_idx] = 1

    return annotation_target

def computeVoicing(activations):

    # import pdb; pdb.set_trace()
    if len(activations.shape)==1:
        ind = np.where(activations)[0]
        voicingArray = np.zeros_like(activations)
        voicingArray = np.insert(voicingArray, -1, 0)
        if not any(activations):
            voicingArray[0] = 1
        else:
            voicingArray[ind+1] = 1
    else:
        for k in range(activations.shape[0]):
            # ind = np.where((activations[k,:]==0),0,0)
            voicingArray = activations
            voicingArray = np.insert(voicingArray, -1, 0)
            if not any(activations[k,:]):
                voicingArray[0] = 1

    return voicingArray

### ---------------- COMPUTE HCQT FEATURES ---------------- ###
def computeHcqt(audio_fpath, nHarmonics, binsPerOctave, n_bins):

    y, fs = librosa.load(audio_fpath, sr=Fs)
    cqt_list = []
    shapes = []
    harmonics = HARMONICS[0:nHarmonics]
    for h in harmonics:
        cqt = librosa.cqt(
            y, sr=Fs, hop_length=HOP_LENGTH, fmin=FMIN*float(h),
            n_bins=n_bins,
            bins_per_octave=binsPerOctave
        )
        cqt_list.append(cqt)
        shapes.append(cqt.shape)

    shapes_equal = [s == shapes[0] for s in shapes]
    if not all(shapes_equal):
        min_time = np.min([s[1] for s in shapes])
        new_cqt_list = []
        for i in range(len(cqt_list)):
            new_cqt_list.append(cqt_list[i][:, :min_time])
        cqt_list = new_cqt_list

    log_hcqt = ((1.0/80.0) * librosa.core.amplitude_to_db(
        np.abs(np.array(cqt_list)), ref=np.max)) + 1.0

    return log_hcqt

def turnToVoicing(targets):
    ### ---------------- TURN ANNOTATION TO BINARY VOICING LABELS ---------------- ###
    return

def readAnnotation(filePath, num_cols=None, header=False):
    if filePath is not None and os.path.exists(filePath):
        with open(filePath) as f_handle:
            annotation = []
            linereader = csv.reader(f_handle)

            # skip the headers for non csv files
            if header:
                header = next(linereader)
            else:
                header = []

            for line in linereader:
                if num_cols:
                    line = line[:num_cols]
                annotation.append([float(val) for val in line])
        return annotation, header
    else:
        return None, None

def grid_to_bins(grid, start_bin_val, end_bin_val):
    """Compute the bin numbers from a given grid
    """
    bin_centers = (grid[1:] + grid[:-1])/2.0
    bins = np.concatenate([[start_bin_val], bin_centers, [end_bin_val]])
    return bins

def bkld(y_true, y_pred):
    """KL Divergence where both y_true an y_pred are probabilities
    """
    y_true = K.clip(y_true, K.epsilon(), 1.0 - K.epsilon())
    y_pred = K.clip(y_pred, K.epsilon(), 1.0 - K.epsilon())
    return K.mean(K.mean(
        -1.0*y_true* K.log(y_pred) - (1.0 - y_true) * K.log(1.0 - y_pred),
        axis=-1), axis=-1)

def zero_pad(curInput, sym, size):

    shape = np.shape(curInput)
    newshape = np.asarray(shape)
    if len(shape)>=3:
        newshape[2] = newshape[2] + size
        newStructure = np.zeros(newshape)
        start = int(size / 2)
        stop = int(newshape[2] - (size / 2))
        newStructure[:, :, start : stop] = curInput
    else:
        newshape[1] = newshape[1] + size
        newStructure = np.zeros(newshape)
        start = int(size / 2)
        stop = int(newshape[1] - (size / 2))
        newStructure[:, start : stop] = curInput

    return newStructure


from torch.nn import Module
from torch import cat
import torch

from dupimage.helper import get_device, get_dataloader, ImageDataset

class BaseExtractor(Module):
    """
    Must override it

    model
    device
    root_dir
    dataset
    dataloader
    labels
    """
    def __init__(self, root_dir='./', suffix='jpg', batch_size=64, num_workers=1):
        super().__init__()
        self.device = get_device()
        self.root_dir = root_dir

        # Get Dataset and Dataloader objects from root_dir
        self.dataset = ImageDataset(root_dir=root_dir, suffix=suffix)
        self.dataloader = get_dataloader(self.dataset, batch_size, num_workers)

        # Get labels
        self.labels = [label.split('/')[-2] + '/' + label.split('/')[-1] for label in self.dataset.files]

    def get_feature_vectors(self, n_components=25088 + 6272):
        """

        """
        features = None

        # get features
        for batch in self.dataloader:
            if self.device != 'cpu':
                batch[0] = batch[0].to(self.device).float()
            output = self(batch[0])
            if features == None:
                features = output
            else:
                features = cat([features, output], dim=0)
            del batch
            torch.cuda.empty_cache()

        return features


"""
    ¿Con cuántas especies de pokémon puede procrear raichu? (2 Pokémon pueden
    procrear si están dentro del mismo egg group). Tu respuesta debe ser un número.
    Recuerda eliminar los duplicados.
"""
import requests

def number_species(name):
    no_egg_group = 1

    while no_egg_group < 16:

        url = 'http://pokeapi.co/api/v2/egg-group/' + str(no_egg_group)

        response = requests.get(url)

        if response.status_code == 200:
            payload = response.json()
            pokemons = payload['pokemon_species']

            for species in pokemons:

                if name in species['name']:
                    print(len(pokemons) - 1)
                    return

        no_egg_group += 1

if __name__ == '__main__':
    number_species(name='raichu')


from fastapi import Request
from fastapi.responses import JSONResponse

from .exceptions import BadRequest
from .exceptions import IncompatibleVersion

from ..controller import UnknownMethodException
from ..controller import InvalidMethodParamsException

__all__ = [
    "incompatible_version_exception_handler",
    "invalid_method_params_exception_handler",
    "unknown_method_exception_handler",
    "bad_request_exception_handler",
]

async def incompatible_version_exception_handler(request: Request, exc: IncompatibleVersion):
    return JSONResponse(
        content={
            "headers": {
                "status_code": 409,
                "detail": "Incompatible version of the widget."
            }
        }
    )

async def bad_request_exception_handler(request: Request, exc: BadRequest):
    return JSONResponse(
        content={
            "headers": {
                "status_code": 400,
                "detail": "Bad Request."
            }
        }
    )

async def invalid_method_params_exception_handler(request: Request, exc: InvalidMethodParamsException):
    return JSONResponse(
        content={
            "headers":
                {
                    "status_code": "410",
                    "detail": f"Invalid method parameters method: {exc.method}; params: {exc.params}"
                }
        },
    )

async def unknown_method_exception_handler(request: Request, exc: UnknownMethodException):
    return JSONResponse(
        content={
            "headers":
                {
                    "status_code": "404",
                    "detail": f"Unknown method: {exc.method}"
                }
        },
    )


from unittest import TestCase
import numpy as np
import matplotlib.pyplot as plt

class BaseTest(TestCase):
	# AssertAlmostEqual for lists.
	def assertItemsAlmostEqual(self, a, b, places=4):
		if np.isscalar(a):
			a = [a]
		else:
			a = self.mat_to_list(a)
		if np.isscalar(b):
			b = [b]
		else:
			b = self.mat_to_list(b)
		for i in range(len(a)):
			self.assertAlmostEqual(a[i], b[i], places)

	# Overriden method to assume lower accuracy.
	def assertAlmostEqual(self, a, b, places=4):
		super(BaseTest, self).assertAlmostEqual(a, b, places=places)

	def assertAlmostLeq(self, a, b, tol=1e-4):
		self.assertTrue(a - b <= tol)

	def assertAlmostGeq(self, a, b, tol=1e-4):
		self.assertTrue(a - b >= -tol)

	def mat_to_list(self, mat):
		"""Convert a numpy matrix to a list.
		"""
		if isinstance(mat, (np.matrix, np.ndarray)):
			return np.asarray(mat).flatten('F').tolist()
		else:
			return mat

	def plot_cdf(self, x, *args, **kwargs):
		x_sort = np.sort(x)
		prob = np.arange(1,len(x)+1)/len(x)
		handle = plt.plot(x_sort, prob, *args, **kwargs)
		plt.ylim(0,1)
		return handle

	def plot_abline(self, slope, intercept, *args, **kwargs):
		"""Plot a line from slope and intercept"""
		axes = plt.gca()
		x_vals = np.array(axes.get_xlim())
		y_vals = intercept + slope * x_vals
		plt.plot(x_vals, y_vals, *args, **kwargs)


import time
import datetime
from pyminder.pyminder import Pyminder

def main():
    break_start = datetime.datetime(2019,12,23)
    break_end = datetime.datetime(2019,12,28)
    enable_breaks(break_start, break_end)

def token():
    return open("token.secret").read()

def enable_breaks(break_start, break_end):
    pyminder = Pyminder(user='[your username - dummy field]', token=token())

    goals = pyminder.get_goals()

    print("Hello,")
    print("")
    print("Sadly, it is too late for me to enable a break manually.")
    print("I need to request a manual break override - from ", break_start.date(), "to", break_end.date(), "for following goals:")
    print("")
    for goal in goals:
        # Goal objects expose all API data as dynamic properties.
        # http://api.beeminder.com/#attributes-2

        # https://stackoverflow.com/a/46090618/4130619
        ep = datetime.datetime(1970,1,1,0,0,0)
        x = (break_end - ep).total_seconds()
        needed = goal.get_needed(x)
        if needed > 0:
            print(goal.slug)
            #print(goal.title)
            #print(goal.fineprint)
            #print(goal.losedate) # unix timestamp

        # Goal objects also implement a handful of helper functions.
        # Note: These functions probably contain bugs! Issues & pull requests welcome.
        # https://github.com/narthur/pyminder/blob/master/pyminder/goal.py
        now = time.time()
        sum_ = goal.get_data_sum(now)
        needed = goal.get_needed(now)

main()


from typing import Iterable

from dateutil.rrule import rrulestr
from django.db import models, transaction as db_transaction
from django.utils import timezone
from django.utils.translation import gettext_lazy as _

from ..models import BaseModel
from ..settings import Settings
from .validators import validate_rrule

class BaseTransaction(BaseModel):

    class Meta:
        abstract = True

    timestamp = models.DateField(_('timestamp'), default=timezone.now)
    amount = models.BigIntegerField(_('amount')) if Settings.USE_BIG_INTEGER else models.IntegerField(_('amount'))
    description = models.TextField(_('description'))

    # TODO: consider adding currency support (e.g. local_amount and local_currency)

    def __str__(self):
        return self.description

class RecurringTransaction(BaseTransaction):

    class Meta:
        verbose_name = _('recurring transaction')
        verbose_name_plural = _('recurring transactions')

    recurrence = models.TextField(_('recurrence'), validators=[validate_rrule])
    last_occurred_at = models.DateTimeField(blank=True)

    entity = models.ForeignKey(Settings.FINANCIAL_ENTITY_MODEL, verbose_name=_('entity'), related_name='recurring_transactions', on_delete=models.PROTECT)

    @property
    def recurrence_rule(self):
        # Parse recurrence rule
        return rrulestr(self.recurrence, cache=True)

    @staticmethod
    def generate_transactions(recurring_transactions: Iterable['RecurringTransaction'] = None):
        with db_transaction.atomic():
            # Find recurring transactions if not provided
            if not recurring_transactions:
                recurring_transactions = RecurringTransaction.objects.all()

            for recurring_transaction in recurring_transactions:
                # Parse recurrence rule and determine occurrences between the last occurrence and now
                if recurring_transaction.last_occurred_at:
                    occurrences = recurring_transaction.recurrence_rule.between(recurring_transaction.last_occurred_at, timezone.now(), inc=False)
                else:
                    occurrences = recurring_transaction.recurrence_rule.before(timezone.now(), inc=False)

                for occurrence in occurrences:
                    # TODO: replace placeholders in description (date, etc.)

                    # Create transaction
                    transaction = Transaction(timestamp=occurrence, amount=recurring_transaction.amount, description=recurring_transaction.description,
                                              entity=recurring_transaction.entity)
                    transaction.save()

                    # Update last occurrence
                    recurring_transaction.last_occurred_at = occurrence

                # Save last occurrence
                recurring_transaction.save()

class Transaction(BaseTransaction):

    class Meta:
        verbose_name = _('transaction')
        verbose_name_plural = _('transactions')

    entity = models.ForeignKey(Settings.FINANCIAL_ENTITY_MODEL, verbose_name=_('entity'), related_name='transactions', on_delete=models.PROTECT)
    recurring_transaction = models.ForeignKey(RecurringTransaction, verbose_name=_('recurring transaction'), related_name='transactions',
                                              blank=True, null=True, on_delete=models.SET_NULL)

    if Settings.TRANSACTION_SETTLEMENT_ENABLED:
        settled_by = models.ForeignKey('self', verbose_name=_('settled by'), related_name='settled', blank=True, null=True, on_delete=models.SET_NULL)

    @staticmethod
    def generate_settlements():
        with db_transaction.atomic():
            # Find entities and their balances
            entities = Settings.get_financial_entity_model().objects.with_balance().all()

            for entity in entities:
                # Find unsettled transactions
                transactions = entity.transactions.filter(settled_by__isnull=True, settled__isnull=True)

                # Sanity check for current state
                if entity.balance != sum([transaction.amount for transaction in transactions]):
                    raise Exception('Balance and sum of unsettled transactions are not equal.')

                # Check if there is a balance to settle
                if entity.balance == 0:
                    continue

                # Create settlement transaction
                settlement_transaction = Transaction(description=Settings.TRANSACTION_SETTLEMENT_DESCRIPTION, amount=-entity.balance, entity=entity)
                settlement_transaction.save()
                settlement_transaction.settled.set(transactions)


import torch
import torch.optim as optim
import numpy as np

device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(device)

torch.manual_seed(777)
if device == 'cuda':
    torch.cuda.manual_seed_all(777)

sentence = ("if you want to build a ship, don't drum up people together to "
            "collect wood and don't assign them tasks and work, but rather "
            "teach them to long for the endless immensity of the sea.")

char_set = list(set(sentence))
char_dic = {c:i for i, c in enumerate(char_set)}

dic_size = len(char_dic)
hiddend_size = len(char_dic)
sequence_length = 10
learning_rate = 0.1

x_data = []
y_data = []

for i in range(0, len(sentence) - sequence_length):
    x = sentence[i: i+sequence_length]
    y = sentence[i+1: i+sequence_length+1]

    x_data.append([char_dic[c] for c in x])
    y_data.append([char_dic[c] for c in y])

x_one_hot = [np.eye(dic_size)[x] for x in x_data]

X = torch.FloatTensor(x_one_hot).to(device)
Y = torch.LongTensor(y_data).to(device)

# RNN 모델
class Net(torch.nn.Module):
    def __init__(self, input_dim, hidden_dim, layers):
        super(Net, self).__init__()
        self.rnn = torch.nn.RNN(input_dim, hidden_dim, num_layers=layers, batch_first=True)
        # num_layers를 설정함으로써 RNN layer를 여러겹 쌓을 수 있다.
        self.fc = torch.nn.Linear(hidden_dim, hidden_dim, bias=True)

    def forward(self, x):
        x, _status = self.rnn(x)
        x = self.fc(x)
        return x

net = Net(dic_size, hiddend_size, 2).to(device)

criterion = torch.nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(net.parameters(), lr=learning_rate)

for i in range(100):
    optimizer.zero_grad()
    outputs = net(X)
    loss = criterion(outputs.view(-1,dic_size), Y.view(-1))
    loss.backward()
    optimizer.step()

    results = outputs.argmax(dim=2)
    predict_str = ''

    for j, result in enumerate(results):
        if j==0:
            predict_str += ''.join([char_set[t] for t in result])
        else:
            predict_str += char_set[result[-1]]

    print(predict_str)


# -*- coding: utf-8 -*-
import image
import scrapy
import re
from scrapy.shell import inspect_response  # for debugging
from scrapy.http import Request
import os, sys; sys.path.append(os.path.dirname(os.path.realpath('webdriver.py')))
from . import webdriver

class restaurantInfo:
    def __init__(self, basic_data, advanced_data):
        self.basic_d = basic_data
        self.advanced_d = advanced_data

class TalabatbotSpider(scrapy.Spider):
    name = 'talabatBot'
    mainDomain: str = 'https://www.talabat.com'
    start_urls = ["https://www.talabat.com/qatar/restaurants"]
    default_location = "al-mansoura"
    custom_settings = {
        'FEED_URI': 'talabat.json'
    }
    item_num = 0
    possibleLocations = ['ain-khaled?aid=1740', 'onaiza?aid=1700']
    restaurant_advanced_info = []
    dataList = []

    # Data processing methods
    def fix_data_file(self, data):
        data['type'] = data['@type']
        data['context'] = data['@context']
        del data['@type']
        del data['@context']
        del data['@id']
        del data['image']

    def fix_JSON_format(self, obj):
        new_json = {}   # put menu section after it was modified in here
        new_json['menuSections'] = []
        new_json['menu_id'] = obj['result']['menu']['id']

        menuSection = obj['result']['menu']['menuSection'] # a list of menu sections
        for section in menuSection:
            new_json['menuSections'].append({'sectionName': section['nm'], 'items': self.fix_item_list(section['itm']) })

        return new_json

    def fix_item_list(self, item_list):
        new_item_list = []
        for item in item_list:
            new_item = {'name'   : item['nm'],
                        'itemID' : item['id'],
                        'rating' : item['rt'],
                        'price'  : item['pr'],
                        'image'  : item['img'],
                        'description' : item['dsc']}
            new_item_list.append(new_item)

        return new_item_list

    # Parse is called whenever the spider successfully crawls a URL
    # Response object is automatically filled with page info and passed here
    def parse(self, response):
        """
        Step 1: Main parse callback, crawling starts here "talabat/qatar/allRestaurants"
        Gather: None
        Fetch : All restaurant links
        """

        restaurant_list = webdriver.runScrollDriver(self.start_urls[0])
        for restaurant in restaurant_list:
            yield scrapy.Request(f"{self.mainDomain}/qatar/{restaurant}", callback=self.parse_restaurant_page)

    def parse_restaurant_page(self, response):
        """
        Step 2: Parsing individual restaurant pages "talabat/qatar/specificRestaurant"
        Gather: basic info (e.g Restaurant Name)
        Fetch : Link to Menu page
        """

        # 1) Get the json file containing all restaurant information
        try:
            # Get the basic data of restaurant
            self.data = self.get_JSON_File(response)[0]
            if (self.data['@type'] != 'Restaurant' and self.data['@type'] != 'restaurant'):
                return {response.url : 'Not restaurant'}
            self.fix_data_file(self.data)
            self.dataList.append(self.data)
            restaurant_ID = re.findall("bid:\d\d\d\d\d", response.css("script").extract()[22])[0][4:]
            valid_locations = re.findall("Al \w+", self.data['description'])
            if (len(valid_locations) == 0):
                valid_locations = re.search("Umm \w+", self.data['description']).group()

            # 2) search for the mainModel JSON
            text = response.xpath("//script[@type='text/javascript']").extract()[8]
            result = re.search('id:\d\d\d\d,cid:\d\d,cn:"\w+",an:"\D+"', text).group()
            result = result.split('"')
            valid_location = result[-2].replace(" ", "-")
            aid = result[0][3:7]
            valid_location = valid_locations[0].rstrip().replace(" ", "-")
        except:
            self.item_num += 1
            while (len(self.restaurant_advanced_info) < len(self.dataList)):
                self.restaurant_advanced_info.append(None)
            return {response.url : self.data}

        # 3) Get URL to menu page
        menu_url = f"https://www.talabat.com/qatar/restaurant/{restaurant_ID}/{valid_location}?aid={aid}"
        return scrapy.Request(menu_url, callback=self.parse_menu_page)

    def get_JSON_File(self, response):
        """ Method to get JSON file from a url """
        items_list = webdriver.runWebDriverJSON(response.url)
        return items_list

    # Step 3: Main parse, parse information from the restaurant
    # Parse Menu items / prices / working hours / etc
    def parse_menu_page(self, response):
        # Skip page if we were redirected to homepage from previous step
        if response.url == 'https://www.talabat.com/qatar':
            self.item_num += 1
            return {f"could not parse {self.dataList[-1]['url']}": None}
        self.restaurant_advanced_info.append(webdriver.runWebDriverMenuPage(response.url))
        self.restaurant_advanced_info[-1] = self.fix_JSON_format(self.restaurant_advanced_info[-1])
        self.item_num += 1
        if (self.restaurant_advanced_info[self.item_num-1] and self.dataList[self.item_num-1]):
            return { str(self.item_num-1): {**self.dataList[self.item_num-1], **self.restaurant_advanced_info[self.item_num-1]} }


from flask import Flask,request,render_template,jsonify
from ainize import *
from src.data import *
# 남이 만든 모듈을 내가 가져다 사용
# from model import predictMainStream

app = Flask(__name__)

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/short', methods=['POST'])
def short():
    res = {
        'msg': decoding_summary_text(mySummary_text_ids(request.form['msg']))
    }
    return jsonify(res)

@app.route('/analisys',methods=['POST'])
def analisys():
    res ={
        'msg' : getName(0)
    }
    return jsonify(res)

if __name__=='__main__':
    app.run(debug=True)


from collections import Counter

class Dictionary:
    def __init__(self, word_list: list[str]) -> None:
        self.word_list = [word.lower() for word in word_list]

    def compute_anagrams(
        self, input_str: str, target_length: int = None, reusable: str = None
    ) -> set[str]:
        """Computes all anagrams of the input string

        Args:
            input_str: A string, which could be a word or jumbled set of letters,
            anagrams should be computed for. Use * for wildcard characters.
            target_length: Specifies the target length of the output anagrams.
                Default to None which results in anagrams that match the length of
                input_str.

        Raises:
            TypeError: If the input is not a string.

        Returns:
            A set of anagrams which are contained within the input string.
        """
        if not isinstance(input_str, str):
            raise TypeError(f"input_str: {input_str} is not a string.")

        input_str = self._clean_input(input_str)

        if target_length is None:
            target_length = len(input_str)

        if reusable is not None:
            reusable = self._clean_input(reusable)
            input_str = self._reusable_letters(input_str, target_length, reusable)

        output = set()
        wilcard_temp = Counter({"*": 1})

        for word in self.word_list:
            if len(word) != target_length:
                continue
            input_counts = Counter(input_str)
            check = []
            for letter in word:
                if input_counts[letter] > 0:
                    check.append(True)
                    input_counts = input_counts - Counter(letter)
                elif input_counts["*"] > 0:
                    check.append(True)
                    input_counts = input_counts - wilcard_temp
                else:
                    check.append(False)
                    break

            if all(check):
                output.add(word)

        return output

    @staticmethod
    def _reusable_letters(input_str: str, target_length: int, reusable: str) -> str:
        """Extends the input string with the a set of reusable letters. Each
        letter is added to the string so that the total number of that letter
        is equal to target_length.

        Args:
            input_str: A string, which could be a word or jumbled set of letters,
            target_length: Number of times the reusable letters will appear in
                the output string.
            reusable: A string containing the letters to repeat

        Raises:
            ValueError: If the letters specified in "reusable" cannot be found in
                "input_str".

        Returns:
            Modified string.
        """
        reusable = set(reusable)
        unique_letters = set(input_str)
        all_letters = list(input_str)
        letter_counter = Counter(input_str)

        excess_letters = reusable - unique_letters
        if excess_letters:
            raise ValueError(
                "The letters specified as reusable do not all appear in the input string."
            )

        for letter in reusable:
            count = letter_counter[letter]
            for i in range(count, target_length):
                all_letters.append(letter)

        return "".join(sorted(all_letters))

    @staticmethod
    def _clean_input(string):
        string = string.lower()
        return "".join(e for e in string if e.isalpha() or e == "*")

class AnagramResult:
    def __init__(self, anagrams: set):
        self.anagrams = anagrams

    def sort(self, method=None):
        return sorted(list(self.anagrams), key=method)


from torchvision import transforms
from torchvision.datasets import CIFAR10, SVHN

DATASETS = ['cifar10', 'svhn']

root = '../data'

def get_dataset(dataset):
    if dataset == 'cifar10':
        transform_train = transforms.Compose([
            transforms.RandomCrop(32, padding=4),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
        ])
        transform_test = transforms.Compose([
            transforms.ToTensor(),
        ])

        trainset = CIFAR10(
            root=root, train=True, download=True, transform=transform_train)
        testset = CIFAR10(
            root=root, train=False, download=True, transform=transform_test)

        return trainset, testset, transform_test
    elif dataset == 'svhn':
        transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        ])
        transform_test = transforms.Compose([
        transforms.ToTensor(),
        ])

        trainset = SVHN(
            root=root, split='train', download=True, transform=transform_train)
        testset = SVHN(
            root=root, split='test', download=True, transform=transform_test)

        return trainset, testset, transform_test


import discord
import aiohttp
import asyncio
import json
from discord.utils import get
from discord.ext import commands
from discord import guild
from discord.channel import DMChannel

rolesF = json.loads(open("config/roles.json", "r").read())

devLeadID = rolesF["devLead"]
devID = rolesF["dev"]

# Role managment functions.
def hasRole(member, id):
        for role in member.roles:
                if id == role.id:
                        return True
        return False

# Actual COG and command.
class DevRole(commands.Cog,name="BOT Dev"):
        def __init__(self,bot):
                self.bot = bot

        @commands.command(description="Toggles bot-dev role to someone.", usage="", hidden=True)
        async def botdev(self, ctx, member: discord.Member):

                devRole = ctx.guild.get_role(devID)

                # Check if the user has the requiered role to issue the command. (DEV LEAD)
                if (hasRole(ctx.author, devLeadID) and not (hasRole(member, devID))):
                        await member.add_roles(devRole)
                        await ctx.send("Welcome on the BOT Dev team, " + member.mention + "!")
                elif (hasRole(ctx.author, devLeadID) and (hasRole(member, devID))):
                        await member.remove_roles(devRole)
                        await ctx.send(member.mention + " left the BOT Dev team!")
                else:
                        await ctx.send("Sorry, " + ctx.author.mention + " but you do not have the permission to do that.")

def setup(bot):
        bot.add_cog(DevRole(bot))