import matplotlib.pyplot as plt

# recall_RPN = [0.986, 0.986, 0.983, 0.979, 0.970, 0.954, 0.933]
# recall_NRPN = [0.999, 0.998, 0.997, 0.996, 0.988, 0.983, 0.971]
# x = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7]
recall_RPN = [0.970, 0.965, 0.958, 0.948, 0.933, 0.911, 0.874]
recall_NRPN = [0.993, 0.990, 0.986, 0.978, 0.975, 0.963, 0.936]
x = [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8]

data = ['RPN', 'N-RPN']
plt.rcParams['font.sans-serif'] = ['SimHei']  # 中文显示
# plt.rcParams['savefig.dpi'] = 300  # 图片像素
# plt.rcParams['figure.dpi'] = 300  # 分辨率
# plt.style.use('bmh')  # ggplot
# plt.figure(figsize=(10, 5))
# plt.title('不同backbone下FPS')
plt.xlabel("IoU")
plt.ylabel("Recall")
plt.plot(x, recall_RPN, '-o', linestyle='-', markersize=5, color='blue', label=data[0])
plt.plot(x, recall_NRPN, '-o', linestyle='-', markersize=5, color='red', label=data[1])

# plt.xlim(0, 21)
plt.ylim(0.86, 1)
plt.legend(loc=1)
plt.grid(True, linestyle='--')
plt.savefig('plot_recall_RPN.png')
plt.show()


import pygame
import numpy as np

width = 1200 #//2
height = 800 #//2
max_iteracion = 20
factorColor = 255//max_iteracion
dw = dh = 20

def iteraMandelbrot(x0,y0):
    global max_iteracion
    x = 0
    y = 0

    iteracion = 0

    while x*x + y*y < 4 and iteracion < max_iteracion:
        xtemp = x*x - y*y + x0
        y = 2*x*y + y0

        x = xtemp

        iteracion += 1
    return iteracion

def getX0Y0(i,j):
    x0 = (float(i)/width)*3.5 - 2.1
    y0 = (float(j)/height)*2.4 - 1.2
    return x0,y0

def getMandel(i,j):
    global mandel
    if mandel[i,j] == 0:
        x0, y0 = getX0Y0(i,j)
        iteracion = iteraMandelbrot(x0,y0)
        mandel[i,j] = iteracion
    else:
        iteracion = mandel[i,j]

    return iteracion

def addRect(x,y,dx,dy):
    global pendientes
    if not (x,y,dx,dy) in pendientes:
        pendientes.append((x,y,dx,dy))

def drawSquare(i,j,dx,dy):
    global fractal, factorColor
    if dx >= 2 and dy >= 2:
        iteracion00   = getMandel(i,   j)
        iteraciondx0  = getMandel(i+dx,j)
        iteracion0dy  = getMandel(i,   j+dy)
        iteraciondxdy = getMandel(i+dx,j+dy)

        if iteracion00 == iteraciondxdy and  iteracion00 == iteracion0dy and  iteracion00 == iteraciondx0 and dx <20:
            color = iteracion00 * factorColor
            if dx>50:
                print(f'{iteracion00} {dx} {dy} ')
        else:
            dx2 = dx//2
            dy2 = dy//2
            if dx2>0 and dy2>0:
                if iteracion00 != iteraciondx0:
                    addRect(i,     j,     dx2,dy2)
                    addRect(i+dx2, j,     dx2,dy2)
                if iteraciondx0 != iteraciondxdy:
                    addRect(i+dx2, j,     dx2,dy2)
                    addRect(i+dx2, j+dy2, dx2,dy2)
                if iteracion00 != iteracion0dy:
                    addRect(i,     j,     dx2,dy2)
                    addRect(i,     j+dy2, dx2,dy2)
                if iteracion0dy != iteraciondxdy:
                    addRect(i,     j+dy2, dx2,dy2)
                    addRect(i+dx2, j+dy2, dx2,dy2)
            else:
                print(f'{dx2} {dy2}')
            color = (iteracion00 + iteraciondxdy + iteracion0dy + iteraciondx0) * factorColor // 4
    else:
        iteracion = getMandel(i,j)
        color =iteracion * factorColor
    pygame.draw.rect(fractal,(color,color,color),(i,j,dx+1,dy+1))

pygame.init()

screen = pygame.display.set_mode((width,height),pygame.DOUBLEBUF | pygame.HWSURFACE)
pygame.display.set_caption("Mandelbrot Fractal")

fractal = screen.copy()

fractal.fill((0,0,0))

mandel = np.zeros((width + dw, height + dh))

pendientes = []
pendientesOld = []

for i in range(0,width,dw):
    for j in range(0,height,dh):
        drawSquare(i,j,dw,dh)

screen.blit(fractal, (0,0))
pygame.display.flip()

contador = 0
while len(pendientes) > 0:

    pendientesOld = pendientes
    pendientes = []
    # print(f'pendientes {len(pendientesOld)}')
    while len(pendientesOld)>0:
         x,y,dx,dy = pendientesOld.pop()

         contador += 1
         drawSquare(x,y,dx,dy)
         if contador%100 == 0:
             screen.blit(fractal, (0,0))
             pygame.display.flip()
             print(f'pendientes: {len(pendientesOld)}')

print(f'hemos terminado {len(pendientes)} {len(pendientesOld)}')
screen.blit(fractal, (0,0))
pygame.display.flip()
running = True

while running:

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_ESCAPE:
                running = False

pygame.quit()


"""
This file contais the views for the project's endpoints.
All classes and methods have their own docstring documentation with a
brief description of how they work. For detailed documentation and examples
of how the API deals with invalid or bad requests, please refer to the
Postman documentation, linked in the repository README.md file.
"""

from django.shortcuts import get_object_or_404
from django.db import transaction
from rest_framework import status, generics
from rest_framework.response import Response

from .models import Client, Referral
from .serializers import ClientSerializer, ReferralSerializer
from .utils import delete_referrals_older_than_30_days

import logging
logger = logging.getLogger(__name__)

class MainPage(generics.ListAPIView):
    """
    Description of API endpoints according to their function.
    """
    """
    This is only so that the local host endpoint gives information on
    the API endpoints, which are more detailed on the documentation.
    """

    def get(self, request):
        urls = {'User detail and update': 'user/<str:cpf>/',
                'List of all referrals registered': 'all-referrals/',
                'List of all referrals performed by an user registered': 'all-referrals/<str:cpf>/',
                'Information on specific referral': 'referral/<str:cpf>/',
                'Create new referral': 'create-referral/',
                'Accept specific referral': 'accept-referral/<str:cpf>/',
                }
        logger.info("Received request to get the main page.")
        return Response(urls, status=status.HTTP_200_OK)

class CreateUserView(generics.ListCreateAPIView):
    """
    Creates a new client, with the requested data
    """

    queryset = Client.objects.all()
    serializer_class = ClientSerializer

    def get(self, request):
        """
        It expects:
        - GET as http method;

        It returns:
        - HTTP status = 200;
        - A message like this:
            [
                "waiting on client creation"
            ]
        """

        logger.info("Waiting for user to create new client.")
        return Response(["waiting on client creation"], status=status.HTTP_200_OK)

    def post(self, request):
        """
        Creates a new user

        It expects:
            - POST as http method;
            - A JSON like this:
            {
                "cpf": "94353687433",
                "name": "José Coelho",
                "phone": "11956555877",
                "email": "jose.coelho@gmail.com"
            }

        It returns:
             - HTTP status = 201;
             - A JSON like this:
               {
                    "Created user:": {
                        "cpf": "94353687433",
                        "name": "José Coelho",
                        "phone": "11956555877",
                        "email": "jose.coelho@gmail.com",
                        "created_at": "2022-01-02T18:23:36.749961-03:00",
                        "updated_at": "2022-01-02T18:23:36.750023-03:00",
                        "points": 0
                    }
                }
        """

        request_data = request.data
        logger.info(
            "Received a request to create a new client with the following data: %s:", request_data)
        serializer = self.serializer_class(data=request.data)

        if serializer.is_valid():
            if request.data['cpf'].isalnum():
                serializer.save()
                logger.info(
                    "Requested data is valid, created the user and returning 201!")
                return Response({'Created user:': serializer.data}, status=status.HTTP_201_CREATED)

            logger.info("CPF is not all numeric, returning 400.")
            return Response({'Error': 'Please enter CPF just with numbers.'}, status=status.HTTP_400_BAD_REQUEST)

        logger.warning(
            "Received data is invalid, returning 400 and the errors.")
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class UpdateUserView(generics.RetrieveUpdateAPIView):
    """
    Gets and/or change the data of a specific user.
    """

    queryset = Client.objects.all()
    serializer_class = ClientSerializer
    lookup_field = 'cpf'

    def get(self, request, cpf):
        """
        Gets the data of a specific user.

        It expects:
        - GET as http method;
        - The CPF specified on the url;

        It returns:
        - HTTP status = 200;
        - A JSON like this:
            {
                "cpf": "11987098390",
                "name": "Luisa Souza",
                "phone": "31998877554",
                "email": "luisa@gmail.com",
                "created_at": "2021-12-22T18:31:48.327319-03:00",
                "updated_at": "2021-12-22T18:39:51.509125-03:00",
                "points": 0
            }
        """

        logger.info("Received a request to fetch a specific User")

        user = get_object_or_404(Client, cpf=cpf)
        serializer = ClientSerializer(user)
        return Response(serializer.data, status=status.HTTP_200_OK)

    def put(self, request, cpf):
        """
        Saves the changes made for the requested user

        It expects:
        - PUT as http method;
        - The CPF specified on the url;
        - A JSON like this:
            {
                "cpf": "11987098390",
                "name": "Luisa Souza",
                "phone": "31998877554",
                "email": "luisa_souza@gmail.com"
            }

        It returns:
        - HTTP status = 200;
        - A JSON like this:
            {
                  "Updated user:": {
                    "cpf": "11987098390",
                    "name": "Luisa Souza",
                    "phone": "31998877554",
                    "email": "luisa_souza@gmail.com",
                    "created_at": "2021-12-22T18:31:48.327319-03:00",
                    "updated_at": "2021-12-24T15:42:27.480610-03:00",
                    "points": 0
                }
            }
        """

        request_data = request.data
        logger.info(
            "Received a request to update a specific User, with the following data: %s", request_data)

        user = Client.objects.get(cpf=cpf)
        serializer = ClientSerializer(user, data=request.data, partial=True)

        if serializer.is_valid():
            if cpf == request.data['cpf']:
                serializer.save()
                logger.info(
                    "Requested data is valid, updating the user and returning 200!")
                return Response({'Updated user:': serializer.data}, status=status.HTTP_200_OK)
            else:
                logger.warning(
                    "User is trying to change their CPF, returning 400.")
                return Response({"error": "cannot change user CPF"}, status=status.HTTP_400_BAD_REQUEST)

        logger.warning("Received data is invalid, returning 400.")
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class GetReferralsView(generics.ListAPIView):
    """
    Gets the data of all referrals on database.
    """
    """
    It expects:
    - GET as http method;

    It returns:
    - HTTP status = 200;
    - A JSON like this:
        [
            {
                "id": 1,
                "source_cpf": "12631049675",
                "target_cpf": "51805510649",
                "created_at": "2021-12-21T15:22:23.097487-03:00",
                "updated_at": "2021-12-23T15:04:34.881831-03:00",
                "status": true
            },
            ...
        ]
    """

    logger.info("Received a request to fetch a list of all Referrals")

    delete_referrals_older_than_30_days()
    queryset = Referral.objects.all()
    serializer_class = ReferralSerializer

class GetUserReferralsView(generics.RetrieveAPIView):
    """
    Gets the data of all referrals on database made by specific user.
    """

    queryset = Referral.objects.all()
    serializer_class = ReferralSerializer
    lookup_field = 'source_cpf'

    def get(self, request, cpf):
        """
        Returns a list of referrals performed by specific user.

        It expects:
        - GET as http method;
        - The CPF specified on the url;

        It returns:
        - HTTP status = 200;
        - A JSON like this:
            [
            {
                "id": 3,
                "source_cpf": "52768135070",
                "target_cpf": "58874265786",
                "created_at": "2021-12-21T18:50:30.355478-03:00",
                "updated_at": "2021-12-21T18:50:30.355534-03:00",
                "status": false
            },
            ...
        ]
        """

        logger.info(
            "Received a request to fetch a list of all Referrals made by user: %s", cpf)

        delete_referrals_older_than_30_days()
        is_client_on_db = Client.objects.filter(cpf=cpf).exists()

        if is_client_on_db:
            if Referral.objects.filter(source_cpf=cpf).exists():
                referrals = Referral.objects.filter(source_cpf=cpf)
                serializer = ReferralSerializer(referrals, many=True)

                logger.info("Data checks, returning referrals and 200!")
                return Response(serializer.data, status=status.HTTP_200_OK)
            else:

                logger.warning("User doesn't have referrals, returning 404.")
                return Response(["error: User doesn't have registered referrals"], status=status.HTTP_404_NOT_FOUND)
        else:
            logger.warning("User is not registered, returning 404.")
            return Response(["error: User not on database"], status=status.HTTP_404_NOT_FOUND)

class GetReferralView(generics.RetrieveAPIView):
    """
    Gets the data of a specific referral on database by the cpf of referred person.
    """

    queryset = Referral.objects.all()
    serializer_class = ReferralSerializer
    lookup_field = 'target_cpf'

    def get(self, request, cpf):
        """
        Returns a specific referral made towards the person with the given cpf.

        It expects:
        - GET as http method;
        - The CPF specified on the url;

        It returns:
        - HTTP status = 200;
        - A JSON like this:
            {
                "id": 1,
                "source_cpf": "12631049675",
                "target_cpf": "51805510649",
                "created_at": "2021-12-21T15:22:23.097487-03:00",
                "updated_at": "2021-12-23T15:04:34.881831-03:00",
                "status": true
            }
        """

        logger.info("Received a request to fetch a specific Referral")
        delete_referrals_older_than_30_days()
        if Referral.objects.filter(target_cpf=cpf).exists():
            referrals = Referral.objects.filter(target_cpf=cpf)
            serializer = ReferralSerializer(referrals, many=True)

            logger.info("Data checks, returning referral and 200!")
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            logger.warning(
                "This person doesn't have active referrals, returning 404.")
            return Response(["error: No active referral towards this person."], status=status.HTTP_404_NOT_FOUND)

class CreateReferralView(generics.ListCreateAPIView):
    """
    Creates a Referral
    """

    queryset = Referral.objects.all()
    serializer_class = ReferralSerializer

    def get(self, request):
        """
        It expects:
        - GET as http method;

        It returns:
        - HTTP status = 200;
        - A message like this:
            [
                "waiting on referral creation"
            ]
        """

        logger.info("Waiting for user to create a referral.")
        return Response(["waiting on referral creation"], status=status.HTTP_200_OK)

    def post(self, request):
        """
        Creates a referral.

        It expects:
        - POST as http method;
        - A JSON like this:
        {
            "source_cpf": "12631049675",
            "target_cpf": "12262411239",
            "status": false
        }

        It returns:
        - HTTP status = 201;
        - A JSON like this:
            {
                "Referral registered": {
                    "id": 8,
                    "source_cpf": "12631049675",
                    "target_cpf": "12262411239",
                    "created_at": "2021-12-24T19:46:17.978403-03:00",
                    "updated_at": "2021-12-24T19:46:17.978446-03:00",
                    "status": false
                }
            }
        """

        request_data = request.data
        logger.info(
            "Received a request to create a referral with the following data: %s:", request_data)
        delete_referrals_older_than_30_days()
        serializer = self.serializer_class(data=request.data)

        if serializer.is_valid():
            if Client.objects.filter(cpf=request.data['source_cpf']).exists():
                if request.data['source_cpf'] == request.data['target_cpf']:

                    logger.warning(
                        "User is trying to refer themselves, returning 400.")
                    return Response(["error: User cannot refer themselves"], status=status.HTTP_400_BAD_REQUEST)

                else:
                    if Client.objects.filter(cpf=request.data['target_cpf']).exists():

                        logger.warning(
                            "User is trying to refer someone who is already on database, returning 400.")
                        return Response(["error: Referred person is already registered"],
                                        status=status.HTTP_400_BAD_REQUEST)

                    else:
                        serializer.save()

                        logger.info(
                            "Data checks, creating referral and returning 201!")
                        return Response({"Referral registered": serializer.data}, status=status.HTTP_201_CREATED)

            else:
                logger.warning(
                    "Non-registered user is trying to refer someone, returning 400.")
                return Response(["error: User must be registered to make a referral"], status=status.HTTP_404_NOT_FOUND)

        elif Referral.objects.filter(target_cpf=request.data['target_cpf']).exists():
            logger.warning(
                "User is trying to refer someone with an active referral, returning 400.")
            return Response("error: This person was already referred.",
                            status=status.HTTP_400_BAD_REQUEST)

        else:
            logger.warning("Requested data is invalid, returnin 400")
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class AcceptReferralView(generics.RetrieveUpdateAPIView):
    """
    Gets and updates referral, allowing its acceptance.
    """

    queryset = Referral.objects.all()
    serializer_class = ReferralSerializer
    lookup_field = 'target_cpf'

    def get(self, request, cpf):
        """
        Gets a specific referral, allowing its acceptance. The referred
        person's CPF is passed on the URL path.

        It expects:
        - GET as http method;
        - The CPF specified on the url;

        It returns:
        - HTTP status = 200;
        - A JSON like this:
            {
                "id": 5,
                "source_cpf": "12631049675",
                "target_cpf": "10370335317",
                "created_at": "2021-12-21T23:36:47.608175-03:00",
                "updated_at": "2021-12-22T23:17:14.602338-03:00",
                "status": false
            }
        """

        logger.info("Received a request to fetch a specific Referral")
        delete_referrals_older_than_30_days()
        if Referral.objects.filter(target_cpf=cpf).exists():
            referrals = Referral.objects.get(target_cpf=cpf)
            serializer = ReferralSerializer(referrals)
            logger.info("Data checks, returning referral and 200!")
            return Response(serializer.data, status=status.HTTP_200_OK)
        else:
            logger.warning("No referrals with this CPF, returning 404")
            return Response({"error": "No active referral registered for this CPF"}, status=status.HTTP_404_NOT_FOUND)

    def put(self, request, cpf):
        """
        Updates the specified referral.

        It expects:
        - PUT as http method;
        - The CPF of reffered person specified on the url;
        - A JSON like this:
            {
                "id": 5,
                "source_cpf": "12631049675",
                "target_cpf": "10370335317",
                "status": true
            }

        It returns:
        - HTTP status = 200;
        - A JSON like this:
            {
                "Updated referral:": {
                    "id": 5,
                    "source_cpf": "12631049675",
                    "target_cpf": "10370335317",
                    "created_at": "2021-12-21T23:36:47.608175-03:00",
                    "updated_at": "2021-12-24T20:14:46.355914-03:00",
                    "status": true
                }
            }

        """

        request_data = request.data
        logger.info(
            "Received a request to update a specific User, with the following data: %s", request_data)

        referral = Referral.objects.get(target_cpf=cpf)
        serializer = ReferralSerializer(
            referral, data=request.data, partial=True)
        referrent = Client.objects.get(cpf=referral.source_cpf)
        updated_status = request.data['status']

        if serializer.is_valid():
            if request.data['target_cpf'] == cpf and request.data['source_cpf'] == referral.source_cpf:
                if updated_status:
                    with transaction.atomic():
                        """
                        Using atomic to ensure both actions will happen, or neither of them.
                        The number of points can be changed to be consistent with the existing point system
                        """
                        referrent.points += 10
                        referrent.save()
                        serializer.save()

                    logger.info(
                        "User accepted the referral! Giving points to referrer and returning 200!")
                    return Response({'Updated referral:': serializer.data}, status=status.HTTP_200_OK)

                else:
                    serializer.save()
                    logger.info("User didn't accept referral, returning 200.")
                    return Response({'Updated referral:': serializer.data}, status=status.HTTP_200_OK)

            else:
                logger.warning("User is trying to change CPFs, returning 400.")
                return Response({"error": "cannot change users CPF"}, status=status.HTTP_400_BAD_REQUEST)

        logger.warning("Requested data is invalid, returning 400.")
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)


#!/usr/bin/env python3

import threading
import serial
import sys

def read_from_port(ser):
    while True:
        data = ser.readline().decode('ascii')
        print(f'{data[:-1]}')

def write_to_port(ser):
    while True:
        data = input().encode('ascii')
        data += b'\n'
        ser.write(data)

def main():

    retval = 0

    if 1 < len(sys.argv):
        try:
            ser = serial.Serial(port = sys.argv[1],
                                baudrate = 9600,
                                timeout = None,
                                parity = serial.PARITY_NONE,
                                stopbits = serial.STOPBITS_ONE)

            print(f"\nUsing serial port {ser.name}\n")

            t1 = threading.Thread(target=read_from_port, args=(ser,))
            t2 = threading.Thread(target=write_to_port, args=(ser,))

            t1.start()
            t2.start()

        except serial.SerialException as e:
            print(f"\nSerial port error: {e}")
            retval = 1
        except KeyboardInterrupt:
            print('\nExiting...')
            ser.close()
            retval = 0
    else:
        print("\nNo port argument given.")
        retval = 1

    return retval

if __name__ == "__main__":
    sys.exit(main())


##How to injest things from notion based on the git here: https://github.com/hwchase17/notion-qa

"""This is the logic for ingesting Notion Data into LangChain."""
from pathlib import Path
from langchain.text_splitter import CharacterTextSplitter
import faiss
from langchain.vectorstores import FAISS
from langchain.embeddings import OpenAIEmbeddings
import pickle
from dotenv import load_dotenv, dotenv_values
#loading the keys from python-dotenv
load_dotenv()
config = dotenv_values(".env.local") #or .env for shared code

def injest_notion():
    # Here we load in the data in the format that Notion exports it in.

    #need an automatic way to get the data directly from notion.
    ps = list(Path("Notion_DB/").glob("**/*.md"))
    print (len(ps))

    data = []
    sources = []
    for p in ps:
        with open(p) as f:
            data.append(f.read())
        sources.append(p)

    # Here we split the documents, as needed, into smaller chunks.
    # We do this due to the context limits of the LLMs.
    text_splitter = CharacterTextSplitter(chunk_size=1500, separator="\n")
    docs = []
    metadatas = []
    for i, d in enumerate(data):
        splits = text_splitter.split_text(d)
        docs.extend(splits)
        metadatas.extend([{"source": sources[i]}] * len(splits))

    # Here we create a vector store from the documents and save it to disk.
    openai_api_key=""
    store = FAISS.from_texts(docs, OpenAIEmbeddings(), metadatas=metadatas)
    faiss.write_index(store.index, "docs.index")
    store.index = None
    with open("faiss_store.pkl", "wb") as f:
        pickle.dump(store, f)
    return None

"""
#Testing the code locally
if __name__ == "__main__":
    injest_notion()
"""


import numpy as np
from typing import List

def trap(height: List[int]) -> int:
    if len(height) == 0:
        return 0
    # First get the max height
    max_ht=max(height); area = 0
    potential = np.zeros(max_ht); max_h_so_far = 0
    prev_ht = 0
    ## Now start collecting rain.
    for h in height:
        for j in range(h, max_h_so_far):
            potential[j]+=1

        for j in range(prev_ht,h):
            area += potential[j]
            potential[j] = 0

        if h > max_h_so_far:
            max_h_so_far = h

        prev_ht = h
    return int(area)

def trap_w_stack(height: List[int])-> int:
    st = []; area = 0; prev_ht=0
    accu_up = 0
    for i in range(len(height)):
        h = height[i]
        if h<prev_ht:
            h_del = (prev_ht-h)
            st.append([h_del,i-1])
            print(st)
            accu_up = 0
        elif h>prev_ht:
            h_del = (h-prev_ht); accu_up=0
            slack=0
            while len(st)>0 and accu_up<h_del:
                up_move = min(st[len(st)-1][0],h_del-accu_up)
                area += (i-st[len(st)-1][1]-1)*up_move
                print((i-st[len(st)-1][1]-1)*up_move)
                accu_up += up_move
                if slack+st[len(st)-1][0]<=accu_up:
                    slack+=st[len(st)-1][0]
                    st.pop()
                else:
                    st[len(st)-1][0]-=up_move
        prev_ht = h
    return area

def trap_w_stack_leetcode(height: List[int])-> int:
    ans = 0; current = 0
    st=[]
    while (current < len(height)):
        while (len(st)>0 and height[current] > height[st[len(st)-1]]):
            top = st[len(st)-1]
            st.pop()
            if len(st)==0:
                break
            distance = current - st[len(st)-1] - 1
            bounded_height = min(height[current], height[st[len(st)-1]]) \
                            - height[top]
            print(str(current)+","+str(st[len(st)-1])+","+str(top))
            ans += distance * bounded_height
        st.append(current)
        current+=1
    return ans

def trap_w_dp_leetcode(height: List[int])-> int:
    left = 0; right = len(height) - 1
    ans = 0
    left_max = 0; right_max = 0
    while (left < right):
        if (height[left] < height[right]):
            if height[left] >= left_max:
                left_max = height[left]
            else:
                ans += (left_max - height[left])
            left+=1
        else:
            if height[right] >= right_max:
                right_max = height[right]
            else:
                ans += (right_max - height[right])
            right-=1
    return ans

h = [4,2,0,3,2,5]
trap_w_dp_leetcode(h)

# h = [3,1,1,0,1,3]
# trap_w_stack(h)

# h = [4,2,0,3,2,5]
# trap(h)

# h = [0,1,0,2,1,0,1,3,2,1,2,1]
# trap_w_stack(h)

h = [0,1,0,4,1,0,1,3,2,1,2,1]
trap_w_dp_leetcode(h)


import matplotlib as mpl
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import random

mpl.rcParams['legend.fontsize'] = 10

fig = plt.figure()
ax = fig.gca(projection='3d')

xyz = []
cur = [0, 0, 0]

for _ in range(888):
    axis = random.randrange(0, 3)
    cur[axis] += random.choice([-1, 1])
    xyz.append(cur[:])

x, y, z = zip(*xyz)
ax.plot(x, y, z,c='darkslategrey',alpha=0.6, label='Random walk')
ax.scatter(x[0], y[0], z[0], c='deepskyblue',s=55,alpha=0.6, marker='o',label='Start point')
ax.scatter(x[-1], y[-1], z[-1], c='crimson',s=55,alpha=0.6, marker='<',label='End point')   # End point
ax.legend()
plt.title('3D lattice random walk')
plt.show()


from django.db.models import Q
from django.shortcuts import render, get_object_or_404, redirect

from carts.models import CartItem
from orders.models import OrderProduct
from .forms import ReviewForm
from .models import Product, ReviewRating, ProductGallery
from category.models import Category
from carts.views import _cart_id
from django.core.paginator import EmptyPage, PageNotAnInteger, Paginator
from django.contrib import messages

# Create your views here.

def store(request, category_slug=None):  # category_slug=None is used to make the category_slug optional
    categories = None  # categories is used to store the category_slug
    products = None  # products is used to store the products

    if category_slug != None:  # if category_slug is not None, then the category_slug is stored in categories, iT MEANS THAT IF THE CATEGORY IS SELECTED THEN THE PRODUCTS OF THAT CATEGORY WILL BE DISPLAYED OR FILTERED
        categories = get_object_or_404(Category,
                                       slug=category_slug)  # get_object_or_404() is used to get the object of the Category model with the slug=category_slug
        products = Product.objects.filter(category=categories,
                                          is_available=True)  # products is used to store the products of the category_slug
        paginator = Paginator(products, 6)  # 6 is the number of products per page
        page = request.GET.get('page')  # get the page number
        paged_products = paginator.get_page(page)  # get the products for the page number
        products_count = products.count()
    else:  # if category_slug is None, then all the products are stored in products
        products = Product.objects.all().filter(is_available=True).order_by("id") # Order_by(id) because we remove warning in the console "UnorderedObjectListWarning: Pagination may yield inconsistent results with an unordered object_list: <class 'store.models.Product'> QuerySet."
        paginator = Paginator(products, 6)  # 6 is the number of products per page
        page = request.GET.get('page')  # get the page number
        paged_products = paginator.get_page(page)  # get the products for the page number
        products_count = products.count()

    context = {
        'products': paged_products,  # products is passed as a context
        'products_count': products_count,
    }
    return render(request, 'store\store.html', context)

def product_detail(request, category_slug, product_slug):  # category_slug and product_slug are passed as parameters
    try:
        single_product = Product.objects.get(category__slug=category_slug,
                                             slug=product_slug)  # get the product with the category_slug and product_slug and category__slug, double underscore use because of foreign key and get slug from other model
        in_cart = CartItem.objects.filter(cart__cart_id=_cart_id(
            request),
            product=single_product).exists()  # check if the product is in the cart or not and return True or False

    except Exception as e:
        raise e
    if request.user.is_authenticated: # check if the user is authenticated or not
        try: # check the orderproduct table if the user already owns the product or not, then it allows the user to submit the review
            orderproduct = OrderProduct.objects.filter(user=request.user, product_id=single_product.id).exists()
        except OrderProduct.DoesNotExist:
            orderproduct = None
    else: # if the user is not authenticated then orderproduct will be None
        orderproduct = None
    # Get the reviews
    reviews = ReviewRating.objects.filter(product_id=single_product.id, status=True)

    # Get the product gallery
    product_gallery = ProductGallery.objects.filter(product_id=single_product.id)

    context = {
        'single_product': single_product,
        'in_cart': in_cart,
        'orderproduct': orderproduct,
        'reviews': reviews,
        'product_gallery': product_gallery,
    }
    return render(request, 'store\product_detail.html', context)

def search(request):
    if 'keyword' in request.GET:
        keyword = request.GET['keyword']
        if keyword:
            products = Product.objects.order_by(
                '-created_date').filter(Q(description__icontains=keyword) | Q(product_name__icontains=keyword))
            products_count = products.count()
    context = {
        'products': products,
        'products_count': products_count,
    }
    return render(request, 'store\store.html',context)

def submit_review(request,product_id):
    url = request.META.get('HTTP_REFERER') # get the last url
    if request.method == 'POST':
        try:
            reviews = ReviewRating.objects.get(user__id=request.user.id, product__id=product_id) # get the review of the user and product
            form = ReviewForm(request.POST, instance=reviews) # instance=reviews is used to update the review, if the review is already submitted
            form.save()
            messages.success(request, 'Thank you! Your review has been updated.')
            return redirect(url)

        except ReviewRating.DoesNotExist: # if the review is not submitted then create a new review
            form = ReviewForm(request.POST) # create a form instance
            if form.is_valid(): # check if the form is valid or not
                data = ReviewRating() # create an object
                data.subject = form.cleaned_data['subject']
                data.rating = form.cleaned_data['rating']
                data.review = form.cleaned_data['review']
                data.ip = request.META.get('REMOTE_ADDR') # get the ip address
                data.product_id = product_id
                data.user_id = request.user.id
                data.save() # save the data
                messages.success(request, 'Thank you! Your review has been submitted.')
                return redirect(url)


from numpy import size
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision.models import resnet50
from .network_utils import *
from .xception import xception
import pdb
class Segception_small(nn.Module):
    def __init__(self, num_classes, input_shape=(None, None, 3), in_channels=3, weights='imagenet', **kwargs):
        super(Segception_small, self).__init__(**kwargs)
        # load pretrained Xception

        self.base_output = {}

# bn2(147)
# block1.rep[4](74)
# 2(37)
# 12(19)
# bn4(10)
        self.base_model = xception(num_classes=1000, pretrained='imagenet')
        self.base_model.bn2.register_forward_hook(get_features(self.base_output, 'bn2'))
        self.base_model.block1.rep[4].register_forward_hook(get_features(self.base_output,'block1_sepconv2_bn'))
        self.base_model.block2.rep[4].register_forward_hook(get_features(self.base_output,'block2_sepconv2_bn'))
        self.base_model.block12.rep[4].register_forward_hook(get_features(self.base_output,'block12_sepconv2_bn'))
        self.base_model.bn4.register_forward_hook(get_features(self.base_output,'bn4'))

        # TODO get the each layer output here

        # Decoder
        self.adap_encoder_1 = EncoderAdaption(in_channels=2048, filters=128, kernel_size=3, dilation_rate=1)
        self.adap_encoder_2 = EncoderAdaption(in_channels=1024, filters=128, kernel_size=3, dilation_rate=1)
        self.adap_encoder_3 = EncoderAdaption(in_channels=256, filters=128, kernel_size=3, dilation_rate=1)
        self.adap_encoder_4 = EncoderAdaption(in_channels=128, filters=64, kernel_size=3, dilation_rate=1)
        self.adap_encoder_5 = EncoderAdaption(in_channels=64, filters=32, kernel_size=3, dilation_rate=1)

        self.decoder_conv_1 = FeatureGeneration(in_channels=128, filters=128, kernel_size=3, dilation_rate=1, blocks=3)
        self.decoder_conv_2 = FeatureGeneration(in_channels=128, filters=64, kernel_size=3, dilation_rate=1, blocks=3)
        self.decoder_conv_3 = FeatureGeneration(in_channels=64, filters=32, kernel_size=3, dilation_rate=1, blocks=3)
        self.decoder_conv_4 = FeatureGeneration(in_channels=32, filters=32, kernel_size=3, dilation_rate=1, blocks=1)
        self.aspp = ASPP_2(in_channels=32, filters=32, kernel_size=3)

        self.conv_logits = conv(in_channels=32, out_channels=num_classes, kernel_size=1, strides=1, use_bias=True)

    def forward(self, inputs, training=None, mask=None, aux_loss=False):

        # outputs = self.model_output(inputs)
        # emb1=self.model_output(inputs)
        self.base_model(inputs)
        outputs = [self.base_output['bn4'], #2048*10*10
                   self.base_output['block12_sepconv2_bn'], #1024*19*19
                   self.base_output['block2_sepconv2_bn'], #256*74*74
                   self.base_output['block1_sepconv2_bn'], #128*147*147
                   self.base_output['bn2']] #64*147*147
        # add activations to the ourputs of the model
        for i in range(len(outputs)):
            outputs[i] = nn.LeakyReLU(negative_slope=0.3)(outputs[i])

        x = self.adap_encoder_1(outputs[0])
        x = upsampling(x, scale=2)
        # TODO: why here the author's comment is 512??
        x += reshape_into(self.adap_encoder_2(outputs[1]), x)  # 512
        x = self.decoder_conv_1(x)  # 256

        x = upsampling(x, scale=2)
        x += reshape_into(self.adap_encoder_3(outputs[2]), x)  # 256
        x = self.decoder_conv_2(x)  # 256

        x = upsampling(x, scale=2)
        x += reshape_into(self.adap_encoder_4(outputs[3]), x)  # 128
        x = self.decoder_conv_3(x)  # 128

        x = self.aspp(x, operation='sum')  # 128

        x = upsampling(x, scale=2)
        x += reshape_into(self.adap_encoder_5(outputs[4]), x)  # 64
        x = self.decoder_conv_4(x)  # 64
        x = self.conv_logits(x)
        x = upsampling(x, scale=2)

        if aux_loss:
            return x, x
        else:
            # TODO: ?why
            return x

class RefineNet(nn.Module):
    def __init__(self, num_classes, base_model, **kwargs):
        super(RefineNet, self).__init__(**kwargs)
        # mirar si cabe en memoria RefineNet
        # usar esta pero con restore self.model_output.variables y el forward es con rgb y segmentacion
        # poner que justo esas a no optimizar, solo las nuevas
        self.base_model = base_model

        # Decoder
        self.conv = FeatureGeneration(in_channels=3, filters=32, kernel_size=3, dilation_rate=1, blocks=1)
        self.conv_logits = conv(filters=num_classes, kernel_size=1, strides=1, use_bias=True)

    def forward(self, inputs, training=None, mask=None, iterations=1):
        # get non refined segmentation
        segmentation = self.base_model(inputs)

        for i in range(iterations):
            x = torch.cat([inputs, segmentation], -1)
            x = self.conv(x)
            segmentation = self.conv_logits(x)

        return segmentation

class Conv_BN(nn.Module):
    def __init__(self, in_channels, filters, kernel_size, strides=1) -> None:
        super(Conv_BN, self).__init__()

        self.filters = filters
        self.kernel_size = kernel_size
        self.strides = strides

        self.conv = conv(in_channels=in_channels, out_channels=filters, kernel_size=kernel_size, strides=strides)
        # TODO: what is num_features
        self.bn = nn.BatchNorm2d(num_features=filters, eps=1e-3, momentum=0.993)

    def forward(self, inputs, training=None, activation=True):
        x = self.conv(inputs) # (batch_size, 128, 147, 147)
        x = self.bn(x) # (batch_sizze, 128, 147, 147)
        if activation:
            x = nn.LeakyReLU(negative_slope=0.3)(x)
        return x

class DepthwiseConv_BN(nn.Module):
    def __init__(self, in_channels, filters, kernel_size, strides=1, dilation_rate=1):
        super(DepthwiseConv_BN, self).__init__()

        self.filters = filters
        self.kernel_size = kernel_size
        self.strides = strides

        self.conv = SeparableConv2d(in_channels=in_channels, out_channels=filters, kernel_size=kernel_size)
        self.bn = nn.BatchNorm2d(num_features=filters, eps=1e-3, momentum=0.993)

    def forward(self, inputs, training=None):
        x = self.conv(inputs)
        x = self.bn(x)
        x = nn.LeakyReLU(negative_slope=0.3)(x)

        return x

class Transpose_Conv_BN(nn.Module):
    def __init__(self, in_channels, filters, kernel_size, strides=1):
        super(Transpose_Conv_BN, self).__init__()

        self.filters = filters
        self.kernel_size = kernel_size
        self.strides = strides

        self.conv = transposeConv(filters=filters, kernel_size=kernel_size, strides=strides)
        self.bn = nn.BatchNorm2d(num_features=filters, eps=1e-3, momentum=0.993)

    def forward(self, inputs, training=None):
        x = self.conv(inputs)
        x = self.bn(x)
        x = nn.LeakyReLU(negative_slope=0.3)(x)

        return x

class ShatheBlock(nn.Module):
    def __init__(self, in_channels, filters, kernel_size,  dilation_rate=1, bottleneck=2) -> None:
        super(ShatheBlock, self).__init__()

        self.filters = filters * bottleneck
        self.kernel_size = kernel_size

        self.conv = DepthwiseConv_BN(in_channels, self.filters, kernel_size=kernel_size, dilation_rate=dilation_rate)
        self.conv1 = DepthwiseConv_BN(self.filters, self.filters, kernel_size=kernel_size, dilation_rate=dilation_rate)
        self.conv2 = DepthwiseConv_BN(self.filters, self.filters, kernel_size=kernel_size, dilation_rate=dilation_rate)
        # TODO: why use filters not self.filters?
        self.conv3 = Conv_BN(self.filters, filters, kernel_size=1)

    def forward(self, inputs, training=None):
        x = self.conv(inputs)
        x = self.conv1(x)
        x = self.conv2(x)
        x = self.conv3(x)
        return x + inputs

class ASPP(nn.Module):
    def __init__(self, in_channels, filters, kernel_size):
        super(ASPP, self).__init__()

        self.conv1 = DepthwiseConv_BN(in_channels, filters, kernel_size=1, dilation_rate=1)
        self.conv2 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=4)
        self.conv3 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=8)
        self.conv4 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=16)
        self.conv5 = Conv_BN(in_channels, filters, kernel_size=1)

    def forward(self, inputs, training=None, operation='concat'):
        # TODO: check if the inputs are tensor?
        feature_map_size = inputs.size()
        image_features = torch.mean(inputs, [1, 2], keepdim=True)
        image_features = self.conv1(image_features)
        image_features = F.interpolate(image_features, size=(feature_map_size[1], feature_map_size[2]), mode='bilinear')
        x1 = self.conv2(inputs)
        x2 = self.conv3(inputs)
        x3 = self.conv4(inputs)
        if 'concat' in operation:
            x = self.conv5(torch.cat((image_features, x1, x2, x3, inputs), axis=3))
        else:
            x = image_features + x1 + x2 + x3 + inputs

        return x

class ASPP_2(nn.Module):
    def __init__(self, in_channels, filters, kernel_size):
        super(ASPP_2, self).__init__()

        self.conv1 = DepthwiseConv_BN(in_channels, filters, kernel_size=1, dilation_rate=1)
        self.conv2 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=4)
        self.conv3 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=8)
        self.conv4 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=16)
        self.conv6 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=(2, 8))
        self.conv7 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=(6, 3))
        self.conv8 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=(8, 2))
        self.conv9 = DepthwiseConv_BN(in_channels, filters, kernel_size=kernel_size, dilation_rate=(3, 6))
        self.conv5 = Conv_BN(in_channels, filters, kernel_size=1)

    def forward(self, inputs, training=None, operation='concat'):
        feature_map_size = inputs.size()
        # TODO: compare to tf version, which dim to choose?
        image_features = torch.mean(inputs, dim=(2, 3), keepdim=True)
        image_features = self.conv1(image_features)
        image_features = F.interpolate(image_features, size=(feature_map_size[2], feature_map_size[3]), mode='bilinear')
        x1 = self.conv2(inputs)
        x2 = self.conv3(inputs)
        x3 = self.conv4(inputs)
        x4 = self.conv6(inputs)
        x5 = self.conv7(inputs)
        x4 = self.conv8(inputs) + x4
        x5 = self.conv9(inputs) + x5
        if 'concat' in operation:
            x = self.conv5(torch.cat((image_features, x1, x2, x3,x4,x5, inputs), axis=3))
        else:
            x = self.conv5(image_features + x1 + x2 + x3+x5+x4) + inputs

        return x

class EncoderAdaption(nn.Module):
    def __init__(self, in_channels, filters, kernel_size, dilation_rate=1) -> None:
        super(EncoderAdaption, self).__init__()
        self.filters = filters
        self.kernel_size = kernel_size

        self.conv1 = Conv_BN(in_channels, filters, kernel_size=1)
        self.conv2 = ShatheBlock(filters, filters, kernel_size=kernel_size, dilation_rate=dilation_rate)

    def forward(self, inputs, training=None):
        x = self.conv1(inputs)
        x = self.conv2(x)
        return x

class FeatureGeneration(nn.Module):
    def __init__(self, in_channels, filters, kernel_size, dilation_rate=1, blocks=3):
        super(FeatureGeneration, self).__init__()

        self.filters = filters
        self.kernel_size = kernel_size

        self.conv0 = Conv_BN(in_channels, self.filters, kernel_size=1)
        self.blocks = []
        for n in range(blocks):
            self.blocks = self.blocks + [
                ShatheBlock(self.filters, self.filters, kernel_size=kernel_size, dilation_rate=dilation_rate)]

    def forward(self, inputs, training=None):

        x = self.conv0(inputs)
        for block in self.blocks:
            x = block(x)

        return x

class SeparableConv2d(nn.Module):

    def __init__(self, in_channels, out_channels, kernel_size, bias=False):
        super(SeparableConv2d, self).__init__()
        self.depthwise = nn.Conv2d(in_channels, in_channels, kernel_size=kernel_size,
                                groups=in_channels, bias=bias, padding=1)
        self.pointwise = nn.Conv2d(in_channels, out_channels,
                                kernel_size=1, bias=bias)

    def forward(self, x):
        out = self.depthwise(x)
        out = self.pointwise(out)
        return out


import numpy as np
import matplotlib.pyplot as plt
import networkx as nx
from tqdm import tqdm
import random

#读取数据, 这里先不考虑节点的特征X，单纯从邻接矩阵进行无监督学习，得到游走序列后用w2v进行训练，最后用Kmeans评估分类结果
A = np.fromfile(r'D:\coding\dataset\cora数据集\预处理\A.bin', dtype=np.int32).reshape(2708,2708)

#接下来用随机游走生成2*len(A)条序列供w2v训练，每一条序列限制长度为20
iter_num = 2 #每个节点迭代10次
seq_len = 20

#先定义一个类, 该类接收邻接矩阵A的一条边 w = A[i]，并由w产生跳转节点， LeetCode 528原题
class jump(object):
    def __init__(self, w):
        """
        :type w: List[int]
        """
        self.w = w
    def binary_search(self, s, r):
        #在前缀和数组s中查找满足s[i]<r<=s[i+1]的i
        left, right = 0, len(s)-1
        while(left<right):
            mid = (left+right)//2
            if(s[mid]>=r):
                #说明可能位置在左边或者就是该位置
                right = mid
            else:
                #说明可能位置在右边
                left = mid+1
        return right-1
    def pickIndex(self):
        """
        :rtype: int

        计算权值的前缀和s, s[i]表示前i个数的和, s[0]=0
        生成一个位于1~s[-1]之间的随机数r
        找到一个位置i, 满足s[i-1]<r<=s[i], 这里用二分查找
        于是此时选择的数即为第i个数

        复杂度分析：时间O(n), 空间O(n)
        """
        s = [0]*(len(self.w)+1)
        for i in range(len(self.w)): s[i+1] = s[i]+self.w[i]
        n = len(self.w)
        r = random.randint(1,s[-1])
        return self.binary_search(s, r)

#产生训练数据s
def gernerate_train_data(A, iter_num, seq_len):
    '''
    params:
        A:邻接矩阵
        iter_num:迭代次数
        seq_len:游走序列的长度

    return:
        句子列表 [sentence1, sentence2,...] shape:[len(A)*iter_num, seq_len]
    '''
    all_sentence = []
    for epoch in range(iter_num):
        for i in tqdm(range(len(A))):
            sentence = [] #存放当前游走的序列
            index = i #当前游走的节点下标
            for k in range(seq_len):
                sentence.append(str(index))
                J = jump(A[index])
                index = J.pickIndex() #下一个跳转的节点
            all_sentence.append(sentence)
    return all_sentence

all_sentence = gernerate_train_data(A, iter_num, seq_len) #生成训练数据

from gensim.models import Word2Vec
w2v = Word2Vec(
    sentences = all_sentence,
    size = 32,
    min_count = 0,
    workers = -1
)

#创建归一化向量字典
w2v_dict = {}
for i in range(len(A)):
    w2v_dict[i] = w2v.wv.__getitem__(str(i))/np.linalg.norm(w2v.wv.__getitem__(str(i)), ord=2)

#供Kmeans训练的数据
from sklearn.cluster import KMeans
kmeans = KMeans(n_clusters = 7)

X = np.array([
    w2v_dict[i] for i in range(len(A))
])

kmeans.fit(X)

#接下来评估DeepWalk+Kmeans的性能, 用nx作图并打印, 和原图进行比较
#为每一个节点指定颜色, 定义颜色编号
id2color = {
    0:'r',
    1:'g',
    2:'b',
    3:'m',
    4:'orange',
    5:'k',
    6:'c'
}

#需要得到图的边信息
edges = []
for i in range(len(A)):
    for j in range(len(A[i])):
        if A[i][j] != 0:
            edges.append([i,j])

all_nodes = np.array(list(range(len(A))))
#首先绘制原图
Y = np.fromfile(r'D:\coding\dataset\cora数据集\预处理\Y.bin', dtype=np.int32)
colormap1 = [id2color[i] for i in Y]
G1 = nx.Graph()
G1.add_nodes_from(all_nodes)
G1.add_edges_from(np.array(edges))

#绘制无监督学习的图
colormap2 = [id2color[i] for i in kmeans.labels_]
G2 = nx.Graph()
G2.add_nodes_from(all_nodes)
G2.add_edges_from(edges)

#画图
plt.subplot(1,2,1)
nx.draw(G1, with_labels=False, edge_color='y', node_color=colormap1, node_size=10)
plt.subplot(1,2,2)
nx.draw(G2, with_labels=False, edge_color='y', node_color=colormap2, node_size=10)
plt.show()


from pathlib import Path
from typing import Any, Dict, Tuple

from xz import XZFile

_IntegrationCase = Tuple[Path, Dict[str, Any]]

def test_read_all(integration_case: _IntegrationCase, data_pattern: bytes) -> None:
    xz_path, metadata = integration_case
    with XZFile(xz_path) as xzfile:
        streams_items = list(
            xzfile._fileobjs.items()  # pylint: disable=protected-access
        )
        assert len(streams_items) == len(metadata["streams"])
        pos = 0
        stream_boundaries = []
        block_boundaries = []
        for stream_item, metadata_stream in zip(streams_items, metadata["streams"]):
            stream_boundaries.append(pos)
            stream_pos, stream = stream_item
            assert stream_pos == pos
            assert stream.check == metadata_stream["check"]
            block_items = list(
                stream._fileobjs.items()  # pylint: disable=protected-access
            )
            assert len(block_items) == len(metadata_stream["blocks"])
            for block_item, metadata_block in zip(
                block_items, metadata_stream["blocks"]
            ):
                block_boundaries.append(pos)
                block_pos, block = block_item
                assert block_pos == pos - stream_pos
                assert len(block) == metadata_block["length"]
                pos += metadata_block["length"]
            assert len(stream) == pos - stream_pos
        assert xzfile.stream_boundaries == stream_boundaries
        assert xzfile.block_boundaries == block_boundaries
        assert xzfile.read() == data_pattern

def test_read_reversed(integration_case: _IntegrationCase, data_pattern: bytes) -> None:
    xz_path, _ = integration_case
    with XZFile(xz_path) as xzfile:
        # we are testing the worst possible case (lots of negative seeking)
        # limit the time to test by reading in chunks instead of 1 byte at a time
        read_size = 37
        for pos in reversed(range(0, len(data_pattern), read_size)):
            xzfile.seek(pos)
            assert xzfile.read(read_size) == data_pattern[pos : pos + read_size]


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.ticker as mtick
import sqlite3
import seaborn as sns
from matplotlib.offsetbox import OffsetImage, AnnotationBbox
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import requests
import shutil
import datetime
from scipy.stats import norm
from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder
import os
import winsound
os.chdir('C:\\Users\\Travis\\OneDrive\\Data Science\\Personal_Projects\\Sports\\NBA_Prediction_V3_1\\notebooks_v3_1\\Apps\\Team_Play_Type_Project')

from bs4 import BeautifulSoup
import streamlit as st
import plotly.express as px
import plotly.graph_objects as go

home = 'C:\\Users\\Travis\\OneDrive\\Data Science\\Personal_Projects\\Sports\\NBA_Prediction_V3_1\\'

today = datetime.date.today()
today = today.strftime('%Y-%m-%d')

iso = home + 'data/team/Playtypes/isolation_offense' + today + '.csv'
if os.path.exists(iso):
    st.write('Data Appears to be up to date')
    pass
else:
    os.system('C:\\Users\\Travis\\OneDrive\\Data Science\\Personal_Projects\\Sports\\NBA_Prediction_V3_1\\notebooks_v3_1\\Apps\\Team_Play_Type_Project\\team_playtype_scrape.py')

st.sidebar.title('NBA Team Play Type Analysis')

#second = st.sidebar.checkbox('Add Second Team?')

    # load data where the data SHOULD be
isolation = pd.read_csv(home + 'data/team/Playtypes/isolation_offense' + today + '.csv')
postup = pd.read_csv(home + 'data/team/Playtypes/postup_offense' + today + '.csv')
prb = pd.read_csv(home + 'data/team/Playtypes/prb_offense' + today + '.csv')
pnr = pd.read_csv(home + 'data/team/Playtypes/pnr_offense' + today + '.csv')
spotup = pd.read_csv(home + 'data/team/Playtypes/spotup_offense' + today + '.csv')
handoff = pd.read_csv(home + 'data/team/Playtypes/handoff_offense' + today + '.csv')
cut = pd.read_csv(home + 'data/team/Playtypes/cut_offense' + today + '.csv')
offscreen = pd.read_csv(home + 'data/team/Playtypes/offscreen_offense' + today + '.csv')
transition = pd.read_csv(home + 'data/team/Playtypes/transition_offense' + today + '.csv')
putbacks = pd.read_csv(home + 'data/team/Playtypes/putbacks_offense' + today + '.csv')
misc = pd.read_csv(home + 'data/team/Playtypes/misc_offense' + today + '.csv')

# defensive data
isolation_def = pd.read_csv(home + 'data/team/Playtypes/isolation_defense' + today + '.csv')
postup_def = pd.read_csv(home + 'data/team/Playtypes/postup_defense' + today + '.csv')
prb_def = pd.read_csv(home + 'data/team/Playtypes/prb_defense' + today + '.csv')
pnr_def = pd.read_csv(home + 'data/team/Playtypes/pnr_defense' + today + '.csv')
spotup_def = pd.read_csv(home + 'data/team/Playtypes/spotup_defense' + today + '.csv')
handoff_def = pd.read_csv(home + 'data/team/Playtypes/handoff_defense' + today + '.csv')
cut_def = pd.read_csv(home + 'data/team/Playtypes/cut_defense' + today + '.csv')
offscreen_def = pd.read_csv(home + 'data/team/Playtypes/offscreen_defense' + today + '.csv')
transition_def = pd.read_csv(home + 'data/team/Playtypes/transition_defense' + today + '.csv')
putbacks_def = pd.read_csv(home + 'data/team/Playtypes/putbacks_defense' + today + '.csv')
misc_def = pd.read_csv(home + 'data/team/Playtypes/misc_defense' + today + '.csv')

# all offensive / defensive data
the_files = os.listdir(home + 'data/team/Playtypes')
off_files = [f for f in the_files if 'offense' in f]
off_files = [f for f in off_files if today in f]
def_files = [f for f in the_files if 'defense' in f]
def_files = [f for f in def_files if today in f]

playtype_list = ['isolation', 'postup', 'prb', 'pnr', 'spotup', 'handoff', 'cut', 'offscreen', 'transition', 'putbacks', 'misc']

teams = isolation['TEAM'].unique()
# drop nan
teams = teams[~pd.isnull(teams)]

pt = st.sidebar.selectbox('Choose Playtype to view NBA Team Scatter Plot', playtype_list)

if pt == 'isolation':
    df = isolation
    df_def = isolation_def
elif pt == 'postup':
    df = postup
    df_def = postup_def
elif pt == 'prb':
    df = prb
    df_def = prb_def
elif pt == 'pnr':
    df = pnr
    df_def = pnr_def
elif pt == 'spotup':
    df = spotup
    df_def = spotup_def
elif pt == 'handoff':
    df = handoff
    df_def = handoff_def
elif pt == 'cut':
    df = cut
    df_def = cut_def
elif pt == 'offscreen':
    df = offscreen
    df_def = offscreen_def
elif pt == 'transition':
    df = transition
    df_def = transition_def
elif pt == 'putbacks':
    df = putbacks
    df_def = putbacks_def
elif pt == 'misc':
    df = misc
    df_def = misc_def

st.subheader('NBA Team Offensive Playtype Scatter Plot')
st.write('Select the playtype you would like to view from the sidebar. It automatically defaults to isolation.')

# plotly scatter plot of team playtype PPP

fig = px.scatter(df, x = 'FREQ%', y = 'PPP', color = 'PPP',
                    hover_name = 'TEAM', text= 'TEAM', title = pt + ' PPP')
fig.update_layout(
    xaxis_title = 'Frequency of Playtype',
    yaxis_title = 'Points Per Play',
    font = dict(
        family = 'Courier New, monospace',
        size = 14,
        color = '#7f7f7f'
    )
)
fig.update_traces(textposition='top center')

st.plotly_chart(fig)

selected_team = st.sidebar.selectbox('Choose Team to Analyze', teams)
st.subheader('Selected Team: ' + selected_team + ' Offense')

#if second == True:
   # team2 = st.sidebar.selectbox('Select Second Team', options = teams)

data = home + 'data/team/Playtypes/offensive_playtypes/' + today + '.csv'
df1 = pd.read_csv(data)
team_offensive_playtype = df1[df1['TEAM'] == selected_team]
# move the 'playtype' column to the front
cols = team_offensive_playtype.columns.tolist()
cols.remove('play_type')
new_cols = ['play_type'] + cols
team_offensive_playtype = team_offensive_playtype[new_cols]
team_offensive_playtype = team_offensive_playtype.sort_values(by = 'FREQ%', ascending = False).reset_index(drop = True)

# drop the misc playtype
team_offensive_playtype = team_offensive_playtype[team_offensive_playtype['play_type'] != 'misc']

# plot the selected team's playtype PPP vs FREQ%
fig = px.scatter(team_offensive_playtype, x = 'play_type', y = 'PPP', color = 'FREQ%', size = 'FREQ%',
                    hover_name = 'play_type', title = selected_team + ' Playtype PPP')
fig.update_layout(
    xaxis_title = 'Playtype',
    yaxis_title = 'Points Per Play',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

see_df2 = st.checkbox('Show Offensive Data')
if see_df2:
    st.dataframe(team_offensive_playtype)

#################     DEFENSIVE DATA      #################################

st.subheader('Selected Team: ' + selected_team + ' Defense')
data = home + 'data/team/Playtypes/defensive_playtypes/' + today + '.csv'
df2 = pd.read_csv(data)
team_defensive_playtype = df2[df2['TEAM'] == selected_team]
# move the 'playtype' column to the front
cols = team_defensive_playtype.columns.tolist()
cols.remove('play_type')
new_cols = ['play_type'] + cols
team_defensive_playtype = team_defensive_playtype[new_cols]
team_defensive_playtype = team_defensive_playtype.sort_values(by = 'FREQ%', ascending = False).reset_index(drop = True)
# DROP THE MISC PLAYTYPE
team_defensive_playtype = team_defensive_playtype[team_defensive_playtype['play_type'] != 'misc']

# plot barchart of teams defensive playtype PPP
fig = px.scatter(team_defensive_playtype, x = 'play_type', y = 'PPP', color = 'FREQ%', size = 'FREQ%',
                    hover_name = 'play_type', title = selected_team + ' Defensive Playtype PPP')
fig.update_layout(
    xaxis_title = 'Points Per Play',
    yaxis_title = '',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

see_df = st.checkbox('Show Defensive Data')
if see_df:
    st.dataframe(team_defensive_playtype)

#################   DATA COMPARISON   #################################

# get average PPP for each playtype on offense
avg_ppp = df1.groupby('play_type')['PPP'].mean().reset_index()
avg_ppp = avg_ppp.sort_values(by = 'PPP', ascending = False).reset_index(drop = True)
#st.dataframe(avg_ppp)

avg_cut = avg_ppp[avg_ppp['play_type'] == 'cut']['PPP'].values[0]
avg_handoff = avg_ppp[avg_ppp['play_type'] == 'handoff']['PPP'].values[0]
avg_isolation = avg_ppp[avg_ppp['play_type'] == 'isolation']['PPP'].values[0]
avg_offscreen = avg_ppp[avg_ppp['play_type'] == 'offscreen']['PPP'].values[0]
avg_postup = avg_ppp[avg_ppp['play_type'] == 'postup']['PPP'].values[0]
avg_prb = avg_ppp[avg_ppp['play_type'] == 'prb']['PPP'].values[0]
avg_pnr = avg_ppp[avg_ppp['play_type'] == 'pnr']['PPP'].values[0]
avg_putbacks = avg_ppp[avg_ppp['play_type'] == 'putbacks']['PPP'].values[0]
avg_spotup = avg_ppp[avg_ppp['play_type'] == 'spotup']['PPP'].values[0]
avg_transition = avg_ppp[avg_ppp['play_type'] == 'transition']['PPP'].values[0]

# now defense
avg_ppp_def = df2.groupby('play_type')['PPP'].mean().reset_index()
avg_ppp_def = avg_ppp_def.sort_values(by = 'PPP', ascending = False).reset_index(drop = True)
#st.dataframe(avg_ppp_def)

avg_cut_def = avg_ppp_def[avg_ppp_def['play_type'] == 'cut']['PPP'].values[0]
avg_handoff_def = avg_ppp_def[avg_ppp_def['play_type'] == 'handoff']['PPP'].values[0]
avg_isolation_def = avg_ppp_def[avg_ppp_def['play_type'] == 'isolation']['PPP'].values[0]
avg_offscreen_def = avg_ppp_def[avg_ppp_def['play_type'] == 'offscreen']['PPP'].values[0]
avg_postup_def = avg_ppp_def[avg_ppp_def['play_type'] == 'postup']['PPP'].values[0]
avg_prb_def = avg_ppp_def[avg_ppp_def['play_type'] == 'prb']['PPP'].values[0]
avg_pnr_def = avg_ppp_def[avg_ppp_def['play_type'] == 'pnr']['PPP'].values[0]
avg_putbacks_def = avg_ppp_def[avg_ppp_def['play_type'] == 'putbacks']['PPP'].values[0]
avg_spotup_def = avg_ppp_def[avg_ppp_def['play_type'] == 'spotup']['PPP'].values[0]
avg_transition_def = avg_ppp_def[avg_ppp_def['play_type'] == 'transition']['PPP'].values[0]

# plot team defensive PPP vs average PPP as a shadow bar
fig = go.Figure()
fig.add_trace(go.Bar(x = team_defensive_playtype['play_type'], y = team_defensive_playtype['PPP'],
                    name = 'Team PPP', marker_color = 'rgb(180, 83, 109)'))
fig.add_trace(go.Bar(x = team_defensive_playtype['play_type'], y = [avg_cut_def, avg_handoff_def, avg_isolation_def, avg_offscreen_def, avg_postup_def, avg_prb_def, avg_pnr_def, avg_putbacks_def, avg_spotup_def, avg_transition_def],
                    name = 'Avg PPP', marker_color = 'rgb(169, 169, 169)'))
fig.update_layout(
    title = selected_team + ' Defensive PPP vs League Avg',
    xaxis_title = 'Playtype',
    yaxis_title = 'Points Per Play',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

# plot team PPP as a bar and average PPP as a shadow bar
fig = go.Figure()
fig.add_trace(go.Bar(x = team_offensive_playtype['play_type'], y = team_offensive_playtype['PPP'],
                    name = 'Team PPP', marker_color = 'rgb(180, 83, 109)'))
fig.add_trace(go.Bar(x = team_offensive_playtype['play_type'], y = [avg_cut, avg_handoff, avg_isolation, avg_offscreen, avg_postup, avg_prb, avg_pnr, avg_putbacks, avg_spotup, avg_transition],
                    name = 'Average PPP', marker_color = 'rgb(169, 169, 169)'))
fig.update_layout(
    title= selected_team + ' Offensive PPP vs League Avg',
    xaxis_title = 'Play Type',
    yaxis_title = 'Points Per Play',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

# calculate net difference between team PPP and average PPP
top= team_offensive_playtype[team_offensive_playtype['play_type'] != 'misc']
# add league average PPP to dataframe
top['avg_ppp'] = [avg_cut, avg_handoff, avg_isolation, avg_offscreen, avg_postup, avg_prb, avg_pnr, avg_putbacks, avg_spotup, avg_transition]
# calculate net difference
top['net_ppp_diff'] = top['PPP'] - top['avg_ppp']
# sort by net difference
top = top.sort_values(by = 'net_ppp_diff', ascending = False).reset_index(drop = True)

# add average FREQ% from league
avg_freq = df1.groupby('play_type')['FREQ%'].mean().reset_index()
avg_freq = avg_freq.sort_values(by = 'FREQ%', ascending = False).reset_index(drop = True)
# add average FREQ% to dataframe
top['avg_freq'] = [avg_freq[avg_freq['play_type'] == 'cut']['FREQ%'].values[0],
                     avg_freq[avg_freq['play_type'] == 'handoff']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'isolation']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'offscreen']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'postup']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'prb']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'pnr']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'putbacks']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'spotup']['FREQ%'].values[0],
                        avg_freq[avg_freq['play_type'] == 'transition']['FREQ%'].values[0]]
# calculate net difference
top['net_freq_diff'] = top['FREQ%'] - top['avg_freq']
# sort by net difference
top = top.sort_values(by = 'net_freq_diff', ascending = False).reset_index(drop = True)

# plot team PPP as a bar and average PPP as a shadow bar
st.subheader('Points Per Play Differential vs. League Average (OFF)')
fig = go.Figure()
fig.add_trace(go.Bar(x = top['play_type'], y = top['net_ppp_diff'],
                    name = 'Net PPP Difference', marker_color = top['net_ppp_diff']))
fig.update_layout(
    xaxis_title = 'Play Type',
    yaxis_title = 'Points Per Play',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

# FREQUENCY DIFFERENTIAL
st.subheader('Frequency Differential vs. League Average (OFF)')
fig = go.Figure()
fig.add_trace(go.Bar(x = top['play_type'], y = top['net_freq_diff'],
                    name = 'Net Frequency Difference', marker_color = top['net_freq_diff']))
fig.update_layout(
    xaxis_title = 'Play Type',
    yaxis_title = 'Frequency',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

# calculate net difference between team defensive PPP and average defensive PPP
top_def= team_defensive_playtype[team_defensive_playtype['play_type'] != 'misc']
# add league average PPP to dataframe
top_def['avg_ppp'] = [avg_cut_def, avg_handoff_def, avg_isolation_def, avg_offscreen_def, avg_postup_def, avg_prb_def, avg_pnr_def, avg_putbacks_def, avg_spotup_def, avg_transition_def]
# calculate net difference
top_def['net_ppp_diff'] = top_def['PPP'] - top_def['avg_ppp']
# sort by net difference
top_def = top_def.sort_values(by = 'net_ppp_diff', ascending = False).reset_index(drop = True)

# add average FREQ% from league
avg_freq_def = df1.groupby('play_type')['FREQ%'].mean().reset_index()
avg_freq_def = avg_freq_def.sort_values(by = 'FREQ%', ascending = False).reset_index(drop = True)
# add average FREQ% to dataframe
top_def['avg_freq'] = [avg_freq_def[avg_freq_def['play_type'] == 'cut']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'handoff']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'isolation']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'offscreen']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'postup']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'prb']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'pnr']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'putbacks']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'spotup']['FREQ%'].values[0],
                        avg_freq_def[avg_freq_def['play_type'] == 'transition']['FREQ%'].values[0]]
# calculate net difference
top_def['net_freq_diff'] = top_def['FREQ%'] - top_def['avg_freq']
# sort by net difference
top_def = top_def.sort_values(by = 'net_freq_diff', ascending = False).reset_index(drop = True)

# plot team PPP as a bar and average PPP as a shadow bar
st.subheader('Points Per Play Differential vs. League Average (DEF)')
fig = go.Figure()
fig.add_trace(go.Bar(x = top_def['play_type'], y = top_def['net_ppp_diff'],
                    name = 'Net PPP Difference', marker_color = top_def['net_ppp_diff']))
fig.update_layout(
    xaxis_title = 'Play Type',
    yaxis_title = 'Points Per Play',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

# FREQUENCY DIFFERENTIAL
st.subheader('Frequency Differential vs. League Average (DEF)')
fig = go.Figure()
fig.add_trace(go.Bar(x = top_def['play_type'], y = top_def['net_freq_diff'],
                    name = 'Net Frequency Difference', marker_color = top_def['net_freq_diff']))
fig.update_layout(
    xaxis_title = 'Play Type',
    yaxis_title = 'Frequency',
    font = dict(
        family = 'Courier New, monospace',
        size = 18,
        color = '#7f7f7f'
    )
)
st.plotly_chart(fig)

# TEAM 2 - OFFENSE
#st.subheader('Team 2 - Offense')

#df_t2 = pd.read_csv(data)
#team2_offensive_playtype = df_t2[df_t2['TEAM'] == team2]
# move the 'playtype' column to the front
#cols2 = team2_offensive_playtype.columns.tolist()
#cols2.remove('play_type')
#new_cols2 = ['play_type'] + cols2
#team2_offensive_playtype = team2_offensive_playtype[new_cols2]
#team2_offensive_playtype = team2_offensive_playtype.sort_values(by = 'FREQ%', ascending = False).reset_index(drop = True)

# Violin plot of all teams playtype PPP
#st.subheader('Team 2 - Points Per Play')

#st.subheader('All Teams')

# add league average PPP and FREQ% to df1
#df1['avg_ppp'] = df1.groupby('play_type')['PPP'].transform('mean')
#df1['avg_freq'] = df1.groupby('play_type')['FREQ%'].transform('mean')

#play_selection = st.select_slider('Select a Play Type', options = ['cut', 'handoff', 'isolation', 'offscreen', 'postup', 'prb', 'pnr', 'putbacks', 'spotup', 'transition'])

# plot PPP for all teams
#fig = px.bar(df1[df1['play_type'] == play_selection], y = 'TEAM', x = 'PPP', orientation= 'h',
#                                                        hover_data = df1.columns)

#fig.update_layout(
#    xaxis_title = 'Team',
#    yaxis_title = 'Points Per Play',
#    font = dict(
#        family = 'Courier New, monospace',
#        size = 12,
#        color = '#7f7f7f'
#    )
#)
#st.plotly_chart(fig)


import os
import sys
import random
from common import TestCase
from annoy import AnnoyIndex

class IndexTest(TestCase):
    def test_not_found_tree(self):
        i = AnnoyIndex(10, 'angular')
        self.assertRaises(IOError, i.load, 'nonexists.tree')

    def test_binary_compatibility(self):
        i = AnnoyIndex(10, 'angular')
        i.load('test/test.tree')

        # This might change in the future if we change the search algorithm, but in that case let's update the test
        self.assertEqual(i.get_nns_by_item(0, 10), [0, 85, 42, 11, 54, 38, 53, 66, 19, 31])

    def test_load_unload(self):
        # Issue #108
        i = AnnoyIndex(10, 'angular')
        for x in range(100000):
            i.load('test/test.tree')
            i.unload()

    def test_construct_load_destruct(self):
        for x in range(100000):
            i = AnnoyIndex(10, 'angular')
            i.load('test/test.tree')

    def test_construct_destruct(self):
        for x in range(100000):
            i = AnnoyIndex(10, 'angular')
            i.add_item(1000, [random.gauss(0, 1) for z in range(10)])

    def test_save_twice(self):
        # Issue #100
        t = AnnoyIndex(10, 'angular')
        for i in range(100):
            t.add_item(i, [random.gauss(0, 1) for z in range(10)])
        t.build(10)
        t.save('t1.ann')
        t.save('t2.ann')

    def test_load_save(self):
        # Issue #61
        i = AnnoyIndex(10, 'angular')
        i.load('test/test.tree')
        u = i.get_item_vector(99)
        i.save('i.tree')
        v = i.get_item_vector(99)
        self.assertEqual(u, v)
        j = AnnoyIndex(10, 'angular')
        j.load('test/test.tree')
        w = i.get_item_vector(99)
        self.assertEqual(u, w)
        # Ensure specifying if prefault is allowed does not impact result
        j.save('j.tree', True)
        k = AnnoyIndex(10, 'angular')
        k.load('j.tree', True)
        x = k.get_item_vector(99)
        self.assertEqual(u, x)
        k.save('k.tree', False)
        l = AnnoyIndex(10, 'angular')
        l.load('k.tree', False)
        y = l.get_item_vector(99)
        self.assertEqual(u, y)

    def test_save_without_build(self):
        t = AnnoyIndex(10, 'angular')
        for i in range(100):
            t.add_item(i, [random.gauss(0, 1) for z in range(10)])
        # Note: in earlier version, this was allowed (see eg #61)
        self.assertRaises(Exception, t.save, 'x.tree')

    def test_unbuild_with_loaded_tree(self):
        i = AnnoyIndex(10, 'angular')
        i.load('test/test.tree')
        self.assertRaises(Exception, i.unbuild)

    def test_seed(self):
        i = AnnoyIndex(10, 'angular')
        i.load('test/test.tree')
        i.set_seed(42)

    def test_unknown_distance(self):
        self.assertRaises(Exception, AnnoyIndex, 10, 'banana')

    def test_metric_kwarg(self):
        # Issue 211
        i = AnnoyIndex(2, metric='euclidean')
        i.add_item(0, [1, 0])
        i.add_item(1, [9, 0])
        self.assertAlmostEqual(i.get_distance(0, 1), 8)
        self.assertEqual(i.f, 2)

    def test_metric_f_kwargs(self):
        i = AnnoyIndex(f=3, metric='euclidean')

    def test_item_vector_after_save(self):
        # Issue #279
        a = AnnoyIndex(3, 'angular')
        a.verbose(True)
        a.add_item(1, [1, 0, 0])
        a.add_item(2, [0, 1, 0])
        a.add_item(3, [0, 0, 1])
        a.build(-1)
        self.assertEqual(a.get_n_items(), 4)
        self.assertEqual(a.get_item_vector(3), [0, 0, 1])
        self.assertEqual(set(a.get_nns_by_item(1, 999)), set([1, 2, 3]))
        a.save('something.annoy')
        self.assertEqual(a.get_n_items(), 4)
        self.assertEqual(a.get_item_vector(3), [0, 0, 1])
        self.assertEqual(set(a.get_nns_by_item(1, 999)), set([1, 2, 3]))

    def test_prefault(self):
        i = AnnoyIndex(10, 'angular')
        i.load('test/test.tree', prefault=True)
        self.assertEqual(i.get_nns_by_item(0, 10), [0, 85, 42, 11, 54, 38, 53, 66, 19, 31])

    def test_fail_save(self):
        t = AnnoyIndex(40, 'angular')
        with self.assertRaises(IOError):
            t.save('')

    def test_overwrite_index(self):
        # Issue #335
        f = 40

        # Build the initial index
        t = AnnoyIndex(f, 'angular')
        for i in range(1000):
            v = [random.gauss(0, 1) for z in range(f)]
            t.add_item(i, v)
        t.build(10)
        t.save('test.ann')

        # Load index file
        t2 = AnnoyIndex(f, 'angular')
        t2.load('test.ann')

        # Overwrite index file
        t3 = AnnoyIndex(f, 'angular')
        for i in range(500):
            v = [random.gauss(0, 1) for z in range(f)]
            t3.add_item(i, v)
        t3.build(10)
        if os.name == 'nt':
            # Can't overwrite on Windows
            with self.assertRaises(IOError):
                t3.save('test.ann')
        else:
            t3.save('test.ann')
            # Get nearest neighbors
            v = [random.gauss(0, 1) for z in range(f)]
            nns = t2.get_nns_by_vector(v, 1000)  # Should not crash

    def test_get_n_trees(self):
        i = AnnoyIndex(10, 'angular')
        i.load('test/test.tree')
        self.assertEqual(i.get_n_trees(), 10)

    def test_write_failed(self):
        f = 40

        # Build the initial index
        t = AnnoyIndex(f, 'angular')
        t.verbose(True)
        for i in range(1000):
            v = [random.gauss(0, 1) for z in range(f)]
            t.add_item(i, v)
        t.build(10)

        if os.name == 'nt':
            path = 'Z:\\xyz.annoy'
        else:
            path = '/x/y/z.annoy'
        self.assertRaises(Exception, t.save, path)

    def test_dimension_mismatch(self):
        t = AnnoyIndex(100, 'angular')
        for i in range(1000):
            t.add_item(i, [random.gauss(0, 1) for z in range(100)])
        t.build(10)
        t.save('test.annoy')

        u = AnnoyIndex(200, 'angular')
        self.assertRaises(IOError, u.load, 'test.annoy')
        u = AnnoyIndex(50, 'angular')
        self.assertRaises(IOError, u.load, 'test.annoy')

    def test_add_after_save(self):
        # 398
        t = AnnoyIndex(100, 'angular')
        for i in range(1000):
            t.add_item(i, [random.gauss(0, 1) for z in range(100)])
        t.build(10)
        t.save('test.annoy')

        # Used to segfault:
        v = [random.gauss(0, 1) for z in range(100)]
        self.assertRaises(Exception, t.add_item, i, v)

    def test_build_twice(self):
        # 420
        t = AnnoyIndex(100, 'angular')
        for i in range(1000):
            t.add_item(i, [random.gauss(0, 1) for z in range(100)])
        t.build(10)
        # Used to segfault:
        self.assertRaises(Exception, t.build, 10)

    def test_very_large_index(self):
        # 388
        f = 3
        dangerous_size = 2**31
        size_per_vector = 4*(f+3)
        n_vectors = int(dangerous_size / size_per_vector)
        m = AnnoyIndex(3, 'angular')
        m.verbose(True)
        for i in range(100):
            m.add_item(n_vectors+i, [random.gauss(0, 1) for z in range(f)])
        n_trees = 10
        m.build(n_trees)
        path = 'test_big.annoy'
        m.save(path)  # Raises on Windows

        # Sanity check size of index
        self.assertGreaterEqual(os.path.getsize(path), dangerous_size)
        self.assertLess(os.path.getsize(path), dangerous_size + 100e3)

        # Sanity check number of trees
        self.assertEquals(m.get_n_trees(), n_trees)


from collections import OrderedDict
from typing import Tuple, Union

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import clip

from .attention import MultiheadAttention

def gelu(x):
    return x * torch.sigmoid(1.702 * x)

class LayerNorm(nn.LayerNorm):
    """Subclass torch's LayerNorm to handle fp16."""

    def forward(self, x: torch.Tensor):
        orig_type = x.dtype
        ret = super().forward(x.type(torch.float32))
        return ret.type(orig_type)

class QuickGELU(nn.Module):
    def forward(self, x: torch.Tensor):
        return gelu(x)

class ResidualAttentionBlock(nn.Module):
    def __init__(self, d_model: int, n_head: int, clip_patch_size: int, need_masks_embed: bool):
        super().__init__()

        self.n_head = n_head
        self.clip_patch_size = clip_patch_size
        self.attn = MultiheadAttention(d_model, n_head, need_masks_embed=need_masks_embed)
        self.ln_1 = LayerNorm(d_model)
        self.mlp = nn.Sequential(OrderedDict([
            ("c_fc", nn.Linear(d_model, d_model * 4)),
            ("gelu", QuickGELU()),
            ("c_proj", nn.Linear(d_model * 4, d_model))
        ]))
        self.ln_2 = LayerNorm(d_model)

    def attention(self, x: torch.Tensor, attn_mask, masks_embed):
        x, _, masks_res = self.attn(x, x, x, need_weights=False, attn_mask=attn_mask, masks_embed=masks_embed)
        return x, masks_res

    def forward(self, x: torch.Tensor, masks: torch.Tensor, masks_embed: torch.Tensor = None):
        l, b, d = x.shape
        _, q, _, _ = masks.shape
        masks = (masks.sigmoid() >= 0.5).float()
        masks = F.max_pool2d(masks, self.clip_patch_size).flatten(2)

        attn_mask = torch.empty((b, l, l), device=x.device, dtype=torch.bool)
        attn_mask[:, :, :] = False
        attn_mask[:, :, :q] = True
        attn_mask[:, :q, q+1:] = masks == 0.

        attn_mask = torch.repeat_interleave(attn_mask, self.n_head, dim=0)

        x_res, masks_res = self.attention(self.ln_1(x), attn_mask=attn_mask, masks_embed=masks_embed)
        x = x + x_res
        x = x + self.mlp(self.ln_2(x))
        return x, masks_res

class Transformer(nn.Module):
    def __init__(
            self, width: int, layers: int, heads: int,
            clip_input_resolution, clip_patch_size, clip_width, clip_layers, clip_heads
        ):
        super().__init__()
        self.width = width
        self.layers = layers
        self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, clip_patch_size, (i + 1) % 6 == 0)
            for i in range(layers)]
        )

        self.clip_input_resolution = clip_input_resolution
        self.clip_patch_size = clip_patch_size
        self.clip_width = clip_width
        self.clip_layers = clip_layers
        self.clip_heads = clip_heads
        self.clip_num_patches_dim = clip_input_resolution // clip_patch_size
        self.clip_num_patches = self.clip_num_patches_dim ** 2

        self.idxs = [5, 11, 17, 23]
        self.conv1_added_params = nn.Sequential(
            *[nn.Conv2d(1, clip_width, clip_patch_size, clip_patch_size, bias=True)
                for i in range(len(self.idxs))]
        )
        self.conv3_added_params = nn.Sequential(
            *[nn.Conv2d(clip_width, clip_patch_size ** 2, 1, 1, bias=True)
                for i in range(len(self.idxs))]
        )
        self.apply(self.init_weights)

    def init_weights(self, m):
        if isinstance(m, nn.Conv2d):
            nn.init.kaiming_normal_(m.weight)
            nn.init.uniform_(m.bias)

    def forward(self, x: torch.Tensor, masks: torch.Tensor):

        masks_list = []
        for i, block in enumerate(list(self.resblocks.modules())[0]):

            if i in self.idxs:
                masks_embed = self.conv1_added_params[i//6](masks.tanh().unsqueeze(2).flatten(0, 1)) #(Bxq)x1xhxw
                masks_embed = masks_embed.reshape(
                    x.shape[1], masks.shape[1], self.clip_width, self.clip_num_patches
                ).permute(1, 3, 0, 2)
                x, masks_res = block(x, masks, masks_embed)  # masks_res: Bx100x256x1024

                masks_res = masks_res.permute(0, 1, 3, 2).reshape(
                    x.shape[1], 100, self.clip_width, self.clip_num_patches_dim, self.clip_num_patches_dim
                ).flatten(0, 1)
                masks_res = gelu(masks_res)
                masks_res = self.conv3_added_params[i//6](masks_res)
                masks_res = masks_res.reshape(
                    -1, self.clip_patch_size, self.clip_patch_size, self.clip_num_patches_dim, self.clip_num_patches_dim
                ).permute(0,3,1,4,2)
                masks_res = masks_res.flatten(1, 2).flatten(2, 3)
                masks_res = masks_res.squeeze(1).reshape(
                    x.shape[1], 100, self.clip_input_resolution, self.clip_input_resolution
                )

                masks += masks_res
                masks_list.append(masks)
            else:
                x, _ = block(x, masks)

        return x, masks_list

class VisionTransformer(nn.Module):
    def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int):
        super().__init__()
        self.input_resolution = input_resolution
        self.output_dim = output_dim
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False)

        scale = width ** -0.5
        self.class_embedding = nn.Parameter(scale * torch.randn(width))
        self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width))
        self.ln_pre = LayerNorm(width)

        self.transformer = Transformer(width, layers, heads, input_resolution, patch_size, width, layers, heads)

        self.ln_post = LayerNorm(width)
        self.proj = nn.Parameter(scale * torch.randn(width, output_dim))

    def forward(self, x: torch.Tensor, masks: torch.Tensor):
        q = masks.shape[1]

        x = self.conv1(x)  # shape = [*, width, grid, grid]
        x = x.reshape(x.shape[0], x.shape[1], -1)  # shape = [*, width, grid ** 2]
        x = x.permute(0, 2, 1)  # shape = [*, grid ** 2, width]
        x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1)  # shape = [*, grid ** 2 + 1, width]
        x = x + self.positional_embedding.to(x.dtype)
        x = self.ln_pre(x)

        x = x.permute(1, 0, 2)  # NLD -> LND
        cls_embed = x[0:1]
        cls_embed = cls_embed.repeat(q, 1, 1)
        x = torch.cat([cls_embed, x], dim=0)

        x, masks_list = self.transformer(x, masks)
        x = x.permute(1, 0, 2)  # LND -> NLD

        x = self.ln_post(x[:, :q, :])

        if self.proj is not None:
            x = x @ self.proj

        return x, masks_list

class MaskCLIP(nn.Module):
    def __init__(
            self,
            # initialize CLIP
            clip_model_name,
            input_resolution,
            patch_size,
            width,
            layers,
            heads,
            output_dim,
            temperature
        ):
        super().__init__()

        self.temperature = temperature

        self.visual = VisionTransformer(
                input_resolution=input_resolution,
                patch_size=patch_size,
                width=width,
                layers=layers,
                heads=heads,
                output_dim=output_dim
            )

        clip_, _ = clip.load(clip_model_name, device='cpu')
        self.visual.load_state_dict(clip_.visual.state_dict(), strict=False)

        del clip_

    def forward(self, x, masks, txt_embed):
        outputs = {}

        img_fet, masks_list = self.visual(x, masks)

        logits = torch.einsum('b q c, n c -> b q n', img_fet / img_fet.norm(dim=-1, keepdim=True),
            txt_embed.to(img_fet.device)) / self.temperature
        outputs['pred_logits'] = logits
        outputs['pred_masks'] = masks_list[-1]
        outputs['aux_outputs'] = []
        for i in range(len(masks_list) - 1):
            outputs['aux_outputs'].append({'pred_logits': logits, 'pred_masks': masks_list[i]})

        return outputs


from freezegun import freeze_time
from datetime import datetime, timedelta
from unittest.mock import ANY

from django.test import TestCase
from rest_framework.test import RequestsClient

from ...models import Referral, Client
from ..utils import create_user, generate_valid_cpf

class TestAcceptReferralView(TestCase):
    """
    Testing the methods on the 'accept-referral/' endpoint.
    """

    def setUp(self):
        """
        Initializing the RequestsClient for all tests, as well as creating
        an user and a referral.
        """

        self.creation_time = datetime.now().astimezone().isoformat()
        # this is the Datetime format used by Django
        self.target_cpf = generate_valid_cpf()
        with freeze_time(self.creation_time):
            create_user()
            Referral.objects.create(
                source_cpf="11987098390",
                target_cpf=self.target_cpf,
                status=False
            )

        self.client = RequestsClient()

    def test_should_return_200_and_specific_referral(self):
        """
        Testing if the GET method on endpoint.
        """

        URL = f'http://127.0.0.1:8000/accept-referral/{self.target_cpf}/'
        response = self.client.get(URL)
        json_response = response.json()

        expected_json_response = {
            'id': 1, 'source_cpf': '11987098390',
            'target_cpf': self.target_cpf, 'created_at': self.creation_time,
            'updated_at': self.creation_time, 'status': False}

        self.assertEqual(response.status_code, 200)
        self.assertEqual(json_response, expected_json_response)

    def test_should_return_404_for_cpf_without_referral(self):
        """
        Testing if GET method on 'accept-referral/<str:cpf>/' endpoint
        returns a 404 response if the user with the cpf on url path
        isn't connected to a valid referral on database.
        """

        unreferred_cpf = generate_valid_cpf()
        URL = f'http://127.0.0.1:8000/accept-referral/{unreferred_cpf}/'
        response = self.client.get(URL)
        json_response = response.json()

        expected_json_response = {
            'error': 'No active referral registered for this CPF'}

        self.assertEqual(response.status_code, 404)
        self.assertEqual(json_response, expected_json_response)

    def test_should_post_referral_with_201(self):
        """
        Testing if the PUT method on endpoint successfully updates the
        referral in order to accept it (status = True) and that the user
        recieves points for the referral approval.
        """

        URL = f'http://127.0.0.1:8000/accept-referral/{self.target_cpf}/'
        update_time = datetime.now().astimezone().isoformat()
        body = {
            'source_cpf': '11987098390',
            'target_cpf': self.target_cpf,
            'status': True
        }

        with freeze_time(update_time):
            response = self.client.put(URL, body)
            json_response = response.json()

        expected_json_response = {
            "Updated referral:": {
                "id": 1,
                "source_cpf": "11987098390",
                "target_cpf": self.target_cpf,
                "created_at": self.creation_time,
                "updated_at": update_time,
                "status": True
            }
        }

        client = Client.objects.get(cpf=11987098390)

        self.assertEqual(response.status_code, 200)
        self.assertEqual(Referral.objects.count(), 1)
        self.assertEqual(json_response, expected_json_response)
        self.assertEqual(client.points, 10)

    def test_should_return_400_if_updating_To_invalid_cpf(self):
        """
        Testing if the PUT method on endpoint returns a 400 response if
        user is trying to update the referent cpf with an invalid number.
        """

        URL = f'http://127.0.0.1:8000/accept-referral/{self.target_cpf}/'
        update_time = datetime.now().astimezone().isoformat()
        body = {
            'source_cpf': '11987098390',
            'target_cpf': '11111111111',
            'status': True
        }

        response = self.client.put(URL, body)
        json_response = response.json()

        expected_json_response = {"target_cpf": ["Invalid CPF number."]}

        client = Client.objects.get(cpf=11987098390)

        self.assertEqual(response.status_code, 400)
        self.assertEqual(Referral.objects.count(), 1)
        self.assertEqual(json_response, expected_json_response)
        self.assertEqual(client.points, 0)

    def test_should_return_400_if_changing_referent_cpf(self):
        """
        Testing if the PUT method on endpoint returns a 400 response if
        user is trying to update the referent cpf.
        """

        URL = f'http://127.0.0.1:8000/accept-referral/{self.target_cpf}/'
        update_time = datetime.now().astimezone().isoformat()
        body = {
            'source_cpf': '11987098390',
            'target_cpf': generate_valid_cpf(),
            'status': True
        }

        response = self.client.put(URL, body)
        json_response = response.json()

        expected_json_response = {'error': 'cannot change users CPF'}

        client = Client.objects.get(cpf=11987098390)

        self.assertEqual(response.status_code, 400)
        self.assertEqual(Referral.objects.count(), 1)
        self.assertEqual(json_response, expected_json_response)
        self.assertEqual(client.points, 0)

    def test_should_not_return_expired_referrals(self):
        """
        Testing if expired referrals are actually deleted from the database.
        """

        target_cpf = generate_valid_cpf()
        expired_date = datetime.now() - timedelta(days=30)
        expired_date_formatted = expired_date.astimezone().isoformat()

        with freeze_time(expired_date_formatted):
            Referral.objects.create(
                source_cpf="11987098390",
                target_cpf=target_cpf,
                status=False
            )

        self.assertEqual(Referral.objects.count(), 2)
        # before calling a method that deletes expired referrals

        URL = f'http://127.0.0.1:8000/accept-referral/{target_cpf}/'
        response = self.client.get(URL)
        json_response = response.json()

        expected_json_response = {
            'error': 'No active referral registered for this CPF'}

        self.assertEqual(response.status_code, 404)
        self.assertEqual(json_response, expected_json_response)

        self.assertEqual(Referral.objects.count(), 1)
        # after calling a method that deletes expired referrals