# Emote Trainer Wrapper / Mass Analyzer
# Command line interface for mass analysis for Emote, and to train Emote's classification system using TextBlob's API
# Note: Training features have not yet been implemented yet
# All that's really working is mass analyzing text files through the CLI

#!/usr/bin/python
# -*- coding: utf-8-*-
encoding = "utf-8"
import os
import sys
from emote import emote
import time
import gc
import re
import csv
import os
from nltk.tokenize import sent_tokenize

data = ""
trainingFile = ""
trainingData = []
testData = {}

openingCSV = False
openingText = False
openingPDF = False

csvOutputData = False

csvData = []
csvTextData = []
csvResults = []
csvFile = {}

massResults = []
sentences = []

def startInterface():
    global openingCSV
    global openingText
    global openingPDF
    print("\n\tNow starting Emote Mass Analyzer..")
    option = input("\n\tTo analyze a file with Emote, enter in the type name ('PDF', 'Text', or 'CSV'), or enter 'Train' or 'Test': ")
    option = option.lower()
    if option == 'pdf':
        # print("PDF file input.")
        openingPDF = True
        openPDF(path, data)
    elif option == 'text':
        # print("Text file input.")
        openingText = True
        openFile(path, data)
    elif option == 'csv':
        # print("CSV file input.")
        openingCSV = True
        parseCSV(path, csvData, csvTextData)
    elif option == 'train':
        addToDatabase(trainingFile, trainingData)
    elif option == 'test':
        testEmote(testData)
    else:
        print("\n\tBad command entered. Please try again.")
        startInterface()

def openFile(path, data):
    global csvOutputData
    path = input("\n\tEnter the name of the text and extension of the text, CSV, or PDF file (has to be in directory 'texts') to be mass analyzed in Emote: ")
    try:
        p = os.getcwd()
        p = os.path.join(p, 'texts', path)
        print(p)
        file = open(p, 'r')
        data = file.read()
        text = data
        split_into_sentences(text)
        return data
    except IOError as err:
        print("Error opening path to file.")
        # self.openFile(path, data)
            # print("\n\tI/O error({0}): {1}".format(errno, strerror))
    option = input("\n\tOutput classification results to CSV file? (Yes / No)")
    option = option.lower()
    if option == 'yes':
        csvOutputData = True
    elif option == 'no':
        csvOutputData = False
    else:
        ("Commad not understood!")
        startInterface()
    return

def parseText(path):
    csvData = []
    csvTextData = []
    file = open(path, 'r')
    csv_file = csv.reader(file, delimiter = ",")
    for row in csv_file:
        csvData.append(row[0])
        csvTextData.append(row[1])
    file.close()
    analyzeCSV(csvData, csvTextData, massResults, csvFile)
    print("\n\t", csvData)
    print("\n\t", csvTextData)
    return csvData, csvTextData

def analyzeText(csvData, csvTextData, csvFile):
    print("\n\t",csvTextData)
    print("\n\t", csvData)
    global massResults
    massResults = []
    csvResults = {}
    csvFile = {}
    for i in range(len(csvTextData)):
        emote.getInput(csvTextData[i])
        # print(emote.normalizedProbValues)
        massResults.append(emote.normalizedProbValues)
    csvFile = open('static/results.csv', 'w', newline='')
    for i in range(len(massResults)):
        # with open('static/results.csv', 'w', newline='') as csvFile:
        csvIndRowList = []
        csvResults = csv.writer(csvFile, delimiter = ',')
        csvIndRowList.append(csvData[i])
        csvIndRowList.append(csvTextData[i])
        csvIndRowList.append(massResults[i][0])
        csvIndRowList.append(massResults[i][1])
        csvIndRowList.append(massResults[i][2])
        csvIndRowList.append(massResults[i][3])
        csvIndRowList.append(massResults[i][4])
        csvIndRowList.append(massResults[i][5])
        print("\n\tROW LIST", csvIndRowList)
        csvResults.writerow(csvIndRowList)
    csvFile.close()
    return csvResults, csvFile

def openPDF(path, data):
    return

# def openCSV(path, data):
#     path = input("\n\tEnter the name of the text and extension of the text, CSV, or PDF file (has to be in same directory) to be mass analyzed in Emote: ")
#     try:
#         with open(path) as csvfile:
#             reader = csv.reader(csvfile, dialect='excel', quoting=csv.QUOTE_NONNUMERIC)
#             datalist = []
#             datalist = list(reader)
#             return datalist
#     except IOError as err:
#         print("Error opening path to file.")
#     startInterface()
#     return

def split_into_sentences(text):
    print("CLASSIFYING MULTIPLE SENTENCES")
    global sentences
    sentences = []
    # Code below splits sentences without NLTK Tokenizer
    # START
    # caps = "([A-Z])"
    # prefixes = "(Mr|St|Mrs|Ms|Dr)[.]"
    # suffixes = "(Inc|Ltd|Jr|Sr|Co)"
    # starters = "(Mr|Mrs|Ms|Dr|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
    # acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
    # websites = "[.](com|net|org|io|gov)"
    # digits = "([0-9])"
    # print("\n\tTaking input file, converting to text, and splitting it up into sentences..")
    # text = " " + text + "  "
    # text = text.replace("\n"," ")
    # text = re.sub(prefixes,"\\1<prd>",text)
    # text = re.sub(websites,"<prd>\\1",text)
    # if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>")
    # text = re.sub("\s" + caps + "[.] "," \\1<prd> ",text)
    # text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text)
    # text = re.sub(caps + "[.]" + caps + "[.]" + caps + "[.]","\\1<prd>\\2<prd>\\3<prd>",text)
    # text = re.sub(caps + "[.]" + caps + "[.]","\\1<prd>\\2<prd>",text)
    # text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text)
    # text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text)
    # text = re.sub(" " + caps + "[.]"," \\1<prd>",text)
    # text = re.sub(digits + "[.]" + digits,"\\1<prd>\\2",text)
    # if '"' in text: text = text.replace('."','".')
    # if "\"" in text: text = text.replace(".\"","\".")
    # if "!" in text: text = text.replace("!\"","\"!")
    # if "?" in text: text = text.replace("?\"","\"?")
    # text = text.replace(".",".<stop>")
    # text = text.replace("?","?<stop>")
    # text = text.replace("!","!<stop>")
    # text = text.replace("<prd>",".")
    # sentences = text.split("<stop>")
    # sentences = sentences[:-1]
    # sentences = [s.strip() for s in sentences]
    # END
    sentences = sent_tokenize(text)
    print("\n\tSENTENCES", sentences)
    sendToEmote()
    # return sentences

def sendToEmote():
    # pickledData = pickle.load(open("base_corpus.pickle", "wb"))
    # print(emote.objectToPickle)
    # print(sentences)
    global sentences
    global massResults
    massResults = []
    print("""\n\n\tEach sentence found in the text will now be output with the strongest classification associated with it on screen.""")
    print("""\n\tEmote will classify: """ + str(len(sentences)) + " detected sentences.")
    countSentences = int(len(sentences))
    print("COUNT: ", countSentences)
    # sentences = []
    # for num, sent in enumerate(sentences):
    for m in sentences:
        print("\n\t", m)
        # print("\n\tMESSAGE, message")
        # gc.disable()
        # print("\n\tI", i)
        emote.getInput(m)
        # print("\n")
        # print("\n\tH1", message)
        # print("\n\tH2", emote.normalizedProbValues)
        # print("\n"+emote.pre_result)
        # gc.enable()
        # return emote.normalizedProbVals
        massResults.append(emote.normalizedProbValues)
    # massResult.append(emote.result)
    print("\n\tMASS:" , massResults)
    # returnAsCSV(massresults)
    # timeTotal = time.time() - timeToClassify
    # m, s = divmod(s,  60)
    # timeTotal = m + " :" + s + " "
    # print("""\n\n\n\tEmote took """ + str(timeTotal) + " time to classify the text of " + sentences.length + " detected sentences.")
    # outputOption(massResults)
    # return massResults

# Code below is not functional
def outputOption(massResults):
    option = input("\n\tOutput results into data for 'training' or 'spreadsheet' (enter one): ")
    option = option.lower()
    if option == 'training':
        print("\n\tOutputting data into traininable format.")
        outputTraining(massResults)
    elif option == 'spreadsheet':
        print("\n\tOutputting data into CSV with full results.")
        outputSpreadsheet(massResults)
    else:
        print("\n\tCommand not understood!")
        outputOption(massResults)

def outputTraining(massResults):
    startInterfaCe()

def outputSpreadsheet(massResults):
    startInterfaCe()

# Code below is not functional
def addToDatabase(trainingFile, trainingData):

    # USING SHELF
    try:
        pickledData = shelve.open('base_corpus.db', writeback = True)
        train = pickledData["base"]
        # train = pickle.load(open("base_corpus.pickle", "rb" ) )
        # cl = NaiveBayesClassifier(train)
        print("\n\tLoaded pickled default database corpus.")
        # pickledData.close()
    except IOError as err:
        # print("\n\tI/O error({0}): {1}".format(errno, strerror))
        print("\n\tError training pickle file.. system will exit. Go into the directory, delete the corrupt pickle file, and retry this script to train a new copy.")
        # sys.exit()
    trainingFile = raw_input("\n\tEnter the directory / name of the text file to train (with extension): ")
    with open(trainingFile, "rb") as fp:
        for i in fp.readlines():
            # tmp = i.decode('string_escape')
            # tmp = i.decode('utf-8').strip()
            tmp = i.strip()
            # tmp = i.replace('\r', ' ').replace('\n', '')
            # tmp = i.replace('\\', ' ').replace('\'', '')
            # tmp = i.replace('\\', ' ').replace('\"', '')
            # i = i.strip()
            tmp = tmp.split(",")
            try:
                trainingData.append((str(tmp[0]), str(tmp[1])))
            except:
                pass
    for data in trainingData:
        try:
            pickledData["base"].append(data)
        except:
            pass
    pickledData.sync()
    pickledData.close()
    print("\n\tTraining data added to the default database corpus.")
    startInterface()

# Code below is not functional
def testEmote(testData):
    testFileLoc = input("Enter the file name of the text you would like to test against Emote for accuracy (Must be in the same directory and labelled as .txt):")
    try:
        print(str(testFileLoc))
    except:
        print("Did not find the text file.")
        startInterface()
    # for line in open (str(testFileLoc)):
        # print(str(testFileLoc))
        # testDataTemp = line.split()
        # testDataTemp = str(testDataTemp)
        # print(testDataTemp)
        # testData.append(testDataTemp)
        # print(testData)
    # print(testData)
    with open(str(testFileLoc)) as testData:
        print(testData)
        emote.testAccuracy(testData)
    print("\n\n\t\tTesting done.")
    startInterfaCe()

        # print("\n\tNo test data file found.")
        # startInterface()

if __name__ == '__main__':
    path = ""
    data = ""
    trainingFile = ""
    trainingData = []
    csvData = []
    csvTextData = []
    csvResults = []
    csvIndRowList = []
    testData = {}
    csvFile = {}
    massResults = []
    sentences = []
    startInterface()
else:
    path = ""
    data = ""
    trainingFile = ""
    trainingData = []
    csvData = []
    csvTextData = []
    csvResults = []
    csvIndRowList = []
    testData = {}
    csvFile = {}
    massResults = []
    sentences = []


import pandas as pd
import plotly.graph_objects as go
import plotly.subplots as sp
import webbrowser
import os

# Path to the dataset
file_path_ufos = "/Users/maz/Desktop/UFOs/ufo_sighting_data.csv"

# Read the dataset
df = pd.read_csv(file_path_ufos, low_memory=False)

# Convert the 'latitude' column to numeric
df['latitude'] = pd.to_numeric(df['latitude'], errors='coerce')

# Filter the dataset for triangle UFO sightings
triangle_ufos = df[df['UFO_shape'] == 'triangle']

# Create the base map
fig = sp.make_subplots(rows=1, cols=1, subplot_titles=["Triangle UFO Sightings by Latitude"])

# Add the triangle UFO sightings to the map
fig.add_trace(go.Scattergeo(
    lon=triangle_ufos['longitude'],
    lat=triangle_ufos['latitude'],
    mode='markers',
    marker=dict(color='yellow', opacity=0.5),
    hoverinfo='text',
    hovertext=triangle_ufos['city'] + ', ' + triangle_ufos['state/province'] + '
' + triangle_ufos['date_documented'] + '
' + triangle_ufos['description']
))

# Add a shaded region for the predominant latitudinal range
fig.add_trace(go.Scattergeo(
    lon=[-180, 180, 180, -180, -180],
    lat=[39, 39, 53, 53, 39],
    mode='none',
    fill='toself',
    fillcolor='blue',
    opacity=0.5
))

# Update the layout to show a world map with country borders
fig.update_geos(
    showcoastlines=True,
    coastlinecolor="White",
    showland=True,
    landcolor="Black",
    showcountries=True,
    countrycolor="Grey",
    projection_type="natural earth"
)

fig.update_layout(title_text='Triangle UFO Sightings by Latitude', template='plotly_dark')

# Save the figure as an HTML file
file_path = "/Users/maz/Desktop/UFOs/triangle_ufo_map.html"
fig.write_html(file_path)

# Open the saved HTML file in the default web browser
webbrowser.open('file://' + os.path.realpath(file_path))


from django.shortcuts import render

# Create your views here.
from re import I
from django.shortcuts import render

# Create your views here.
import json
from os import stat
from urllib import response
from django.shortcuts import render

import facilities

# Create your views here.
from .serializers import *
from rest_framework.permissions import IsAuthenticated

from authen.models import User
from .models import *
from .serializers import *
from rest_framework import generics
from rest_framework.permissions import IsAuthenticated
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status
from django.shortcuts import get_object_or_404
from facilities.models import Facility

class CountryView(APIView):
    permission_classes = (IsAuthenticated,)

    def post(self, request):
        query_set=CountryConfig.objects.all()
        print(len(query_set))
        if(len(query_set)>0):
            return Response("Country exist",status=status.HTTP_400_BAD_REQUEST)
        else:
            serializer =   countrySerializer(data=request.data)
            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data, status=status.HTTP_201_CREATED)
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def get(self, request):
        country = CountryConfig.objects.all()
        serializer =  countrySerializer(country, many=True)
        return Response(serializer.data)

    def put(self, request, ):
        id=request.data["id"]
        country = get_object_or_404(CountryConfig, id=id)
        serializer =  countrySerializer(country, data=request.data)
        if serializer.is_valid():
            serializer.save()
            code=serializer.data["codecountry"]
            fac=Facility.objects.filter(id=1)[0]
            fac.code=code+"0100001"
            fac.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, *args, **kwargs):
        id=request.data["id"]
        country = get_object_or_404(CountryConfig, id=id)
        country.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)
class LevelView(APIView):
    permission_classes = (IsAuthenticated,)

    def post(self, request):
        level=0
        country=CountryConfig.objects.all()
        #check country levels
        for i in country:
            x=countrySerializer(i,many=False)
            level=x.data["levels"]
        count=len(LevelConfig.objects.all())
        if(count==level):
            return Response("Level is greater than country level",status=status.HTTP_400_BAD_REQUEST)

        else:
            serializer =   levelSerializer(data=request.data)
            if serializer.is_valid():
                serializer.save()
                return Response(serializer.data, status=status.HTTP_201_CREATED)
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def get(self, request):
        level = LevelConfig.objects.all()
        serializer =  levelSerializer(level, many=True)
        x=sorted(serializer.data, key=lambda k: k['id'])
        return Response(x)

    def put(self, request, *args, **kwargs):
        # rows=request.data["rows"]
        # print(rows)
        res=[]
        for x in request.data:
            # print(x)
            level = get_object_or_404(LevelConfig, id=x["id"])
            serializer =  levelSerializer(level, data=x)
            if serializer.is_valid():
                serializer.save()
                res.append(serializer.data)
            else:
                return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
        return Response(res,status=status.HTTP_200_OK)

    def delete(self, request, *args, **kwargs):
        id=request.data["id"]
        level = get_object_or_404(LevelConfig, id=id)
        level.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)


import json
import os, signal
import psutil
import time
from cilantro_ee.cli.utils import restart
from cilantro_ee.cli.start import start_mongo

def check_pid(pid):
    """ Check For the existence of a unix pid. """
    try:
        #signal 0 just checks for pid status
        os.kill(pid, 0)
    except OSError:
        return False
    else:
        return True

def find_pid(name):
    try:
        PNAME = name
        for proc in psutil.process_iter():
            # check whether the process name matches
            if proc.name() == PNAME:
                return proc.pid
    except BaseException as err:
        print("Node Setup not finished")
        print("Error: {}".format(err))
        return -1

def main():

    mpid = find_pid(name='mongod')
    cpid = find_pid(name='cil')

    if -1 in {mpid, cpid}:
        return

    while (True):
        time.sleep(10)
        m_status = check_pid(mpid)
        c_status = check_pid(cpid)

        if False in {c_status, m_status}:
            try:
                os.kill(mpid, signal.SIGTERM)
                os.kill(cpid, signal.SIGTERM)
            except BaseException as err:
                print("Error: {}".format(err))

            # start mongo
            start_mongo()

            # restart node from config
            restart()

if __name__ == "__main__":
    # execute only if run as a script
    main()


import numpy as np
import torch
import torch.nn as nn

from copy import deepcopy
from typing import Dict, Union, Tuple
from offlinerlmoup.policy import BasePolicy
from torch.nn import functional as F
from offlinerlmoup.dynamics import BaseDynamics

deta = 10000
class SACPolicy(BasePolicy):

    def __init__(
        self,
        actor: nn.Module,
        critic1: nn.Module,
        critic2: nn.Module,
        dynamics: BaseDynamics,
        behavior_policy: nn.Module,
        actor_optim: torch.optim.Optimizer,
        critic1_optim: torch.optim.Optimizer,
        critic2_optim: torch.optim.Optimizer,
        behavior_policy_optim: torch.optim.Optimizer,
        tau: float = 0.005,
        gamma: float  = 0.99,
        alpha: Union[float, Tuple[float, torch.Tensor, torch.optim.Optimizer]] = 0.2,
        num_samples_mmd_match: int = 4
    ) -> None:
        super().__init__()

        self.actor = actor
        self.critic1, self.critic1_old = critic1, deepcopy(critic1)
        self.critic1_old.eval()
        self.critic2, self.critic2_old = critic2, deepcopy(critic2)
        self.critic2_old.eval()
        self.behavior_policy = behavior_policy

        self.actor_optim = actor_optim
        self.critic1_optim = critic1_optim
        self.critic2_optim = critic2_optim
        self.behavior_policy_optim = behavior_policy_optim

        self._tau = tau
        self._gamma = gamma
        self.num_samples_mmd_match = num_samples_mmd_match
        self.dynamics = dynamics

        self._is_auto_alpha = False
        if isinstance(alpha, tuple):
            self._is_auto_alpha = True
            self._target_entropy, self._log_alpha, self.alpha_optim = alpha
            self._alpha = self._log_alpha.detach().exp()
        else:
            self._alpha = alpha

    def train(self) -> None:
        self.actor.train()
        self.critic1.train()
        self.critic2.train()

    def eval(self) -> None:
        self.actor.eval()
        self.critic1.eval()
        self.critic2.eval()

    def _sync_weight(self) -> None:
        for o, n in zip(self.critic1_old.parameters(), self.critic1.parameters()):
            o.data.copy_(o.data * (1.0 - self._tau) + n.data * self._tau)
        for o, n in zip(self.critic2_old.parameters(), self.critic2.parameters()):
            o.data.copy_(o.data * (1.0 - self._tau) + n.data * self._tau)

    def actforward_multy(
        self,
        obs: torch.Tensor,
        nums: int,
        deterministic: bool = False
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        dist = self.actor(obs)
        res = []
        for i in range(nums):
            squashed_action, raw_action = dist.rsample()
            res.append(raw_action)
        return torch.stack(res)

    def actforward(
        self,
        obs: torch.Tensor,
        deterministic: bool = False
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        dist = self.actor(obs)
        if deterministic:
            squashed_action, raw_action = dist.mode()
        else:
            squashed_action, raw_action = dist.rsample()
        log_prob = dist.log_prob(squashed_action, raw_action)
        return squashed_action, log_prob

    def select_action(
        self,
        obs: np.ndarray,
        deterministic: bool = False
    ) -> np.ndarray:
        with torch.no_grad():
            action, _ = self.actforward(obs, deterministic)
        return action.cpu().numpy()

    def mmd_loss_laplacian(self, samples1, samples2, sigma=20):
        diff_x_x = samples1.unsqueeze(2) - samples1.unsqueeze(1)
        diff_x_x = torch.mean((-(diff_x_x.abs()).sum(-1)/(2.0 * sigma)).exp(), dim=(1,2))

        diff_x_y = samples1.unsqueeze(2) - samples2.unsqueeze(1)
        diff_x_y = torch.mean((-(diff_x_y.abs()).sum(-1)/(2.0 * sigma)).exp(), dim=(1, 2))

        diff_y_y = samples2.unsqueeze(2) - samples2.unsqueeze(1)
        diff_y_y = torch.mean((-(diff_y_y.abs()).sum(-1)/(2.0 * sigma)).exp(), dim=(1,2))

        overall_loss = (diff_x_x + diff_y_y - 2.0 * diff_x_y + 1e-6).sqrt()
        return overall_loss

    def learn(self, real_batch, fake_batch, batch: Dict) -> Dict[str, float]:
        obss, actions, next_obss, rewards, terminals = batch["observations"], batch["actions"], \
            batch["next_observations"], batch["rewards"], batch["terminals"]

        # global deta
        # deta += 1
        # real_obs = real_batch["observations"]
        # real_act = real_batch["actions"]
        # real_next_obs = real_batch["next_observations"]
        # fake_obs = fake_batch["observations"]
        # fake_act = fake_batch["actions"]
        # fake_next_obs = fake_batch["next_observations"]
        # real_obs_act = torch.cat((real_obs, real_act), axis=-1)
        # fack_obs_act = torch.cat((fake_obs, fake_act), axis=-1)
        # ksi = torch.zeros(real_obs.shape).to(self.actor.device)
        # nn.init.normal_(tensor=ksi, mean=0.0, std=0.1)

        # recon, mean, std = self.behavior_policy(real_obs_act, real_next_obs)
        # recon_loss = F.mse_loss(recon, real_next_obs)
        # kl_loss = -0.5 * (1 + torch.log(std.pow(2)) - mean.pow(2) - std.pow(2)).mean()
        # vae_loss = recon_loss + 0.5 * kl_loss
        # self.behavior_policy_optim.zero_grad()
        # vae_loss.backward()
        # self.behavior_policy_optim.step()

        # with torch.no_grad():
        #     _, fack_raw_sampled_obs = self.behavior_policy.decode_multiple(fack_obs_act, num_decode=self.num_samples_mmd_match)
        #     fake_raw_next_obs = fake_next_obs.unsqueeze(1).repeat(1,4,1)
        #     mmd_loss = self.mmd_loss_laplacian(fake_raw_next_obs, fack_raw_sampled_obs)
        #     test_mmd = (mmd_loss-0.07).mean()

        #     actor_fake_action, _ = self.actforward(fake_obs)
        #     actor_fack_obs_act = torch.cat((fake_obs, actor_fake_action), axis=-1)
        #     _, actor_fack_sampled_obs = self.behavior_policy.decode_multiple(actor_fack_obs_act, num_decode=2)
        #     real_o = [real_obs]
        #     real_o.append(real_next_obs)
        #     real_raw_obs = torch.stack(real_o, dim = 1)
        #     actor_mmd_loss = self.mmd_loss_laplacian(actor_fack_sampled_obs, real_raw_obs)
        #     actor_test_mmd = (actor_mmd_loss-0.07).mean()

        #     loss_deta = 10000/deta
        #     mmd_q_loss = loss_deta *test_mmd * actor_test_mmd
        global deta
        deta += 1
        real_obs = real_batch["observations"]
        real_next_obs = real_batch["next_observations"]
        fake_obs = fake_batch["observations"]
        ksi = torch.zeros(real_obs.shape).to(self.actor.device)
        nn.init.normal_(tensor=ksi, mean=0.0, std=0.1)

        recon, mean, std = self.behavior_policy(real_obs, real_next_obs)
        recon_loss = F.mse_loss(recon, real_next_obs)
        kl_loss = -0.5 * (1 + torch.log(std.pow(2)) - mean.pow(2) - std.pow(2)).mean()
        vae_loss = recon_loss + 0.5 * kl_loss
        self.behavior_policy_optim.zero_grad()
        vae_loss.backward()
        self.behavior_policy_optim.step()

        with torch.no_grad():
            _, fack_raw_sampled_obs = self.behavior_policy.decode_multiple(fake_obs, num_decode=self.num_samples_mmd_match)
            actor_fake_action, _ = self.actforward(fake_obs)
            fake_obs_cpu = fake_obs.cpu().numpy()
            actor_fake_action_cpu = actor_fake_action.cpu().numpy()
            fake_next_obs, _, _, _= self.dynamics.step(fake_obs_cpu, actor_fake_action_cpu)
            fake_raw_next_obs = torch.tensor(fake_next_obs).unsqueeze(1).repeat(1,4,1).to(self.actor.device)
            mmd_loss = self.mmd_loss_laplacian(fake_raw_next_obs, fack_raw_sampled_obs)
            test_mmd = (mmd_loss-0.07).mean()
            loss_deta = 10000/deta
            mmd_q_loss = test_mmd

        # global deta
        # deta += 1
        # real_obs = real_batch["observations"]
        # real_next_obs = real_batch["next_observations"]
        # fake_obs = fake_batch["observations"]
        # ksi = torch.zeros(real_obs.shape).to(self.actor.device)
        # nn.init.normal_(tensor=ksi, mean=0.0, std=0.1)

        # # recon, mean, std = self.behavior_policy(real_obs, real_next_obs)
        # # recon_loss = F.mse_loss(recon, real_next_obs)
        # # kl_loss = -0.5 * (1 + torch.log(std.pow(2)) - mean.pow(2) - std.pow(2)).mean()
        # # vae_loss = recon_loss + 0.5 * kl_loss
        # # self.behavior_policy_optim.zero_grad()
        # # vae_loss.backward()
        # # self.behavior_policy_optim.step()

        # with torch.no_grad():
        #     # _, fack_raw_sampled_obs = self.behavior_policy.decode_multiple(fake_obs, num_decode=self.num_samples_mmd_match)
        #     actor_fake_action, _ = self.actforward(fake_obs)
        #     fake_obs_cpu = fake_obs.cpu().numpy()
        #     actor_fake_action_cpu = actor_fake_action.cpu().numpy()
        #     fake_next_obs, _, _, _= self.dynamics.step(fake_obs_cpu, actor_fake_action_cpu)
        #     fake_raw_next_obs = torch.tensor(fake_next_obs).unsqueeze(1).repeat(1,2,1).to(self.actor.device)
        #     real_o = [real_obs]
        #     real_o.append(real_next_obs)
        #     real_raw_obs = torch.stack(real_o, dim = 1)
        #     mmd_loss = self.mmd_loss_laplacian(fake_raw_next_obs, real_raw_obs)
        #     test_mmd = (mmd_loss-0.07).mean()
        #     loss_deta = 10000/deta
        #     mmd_q_loss = test_mmd
        # update critic
        q1, q2 = self.critic1(obss, actions), self.critic2(obss, actions)
        with torch.no_grad():
            next_actions, next_log_probs = self.actforward(next_obss)
            next_q = torch.min(
                self.critic1_old(next_obss, next_actions), self.critic2_old(next_obss, next_actions)
            ) - self._alpha * next_log_probs
            next_q = next_q #- mmd_q_loss
            target_q = rewards + self._gamma * (1 - terminals) * next_q

        critic1_loss = ((q1 - target_q).pow(2)).mean() #+ mmd_q_loss
        self.critic1_optim.zero_grad()
        critic1_loss.backward()
        self.critic1_optim.step()

        critic2_loss = ((q2 - target_q).pow(2)).mean()# + mmd_q_loss
        self.critic2_optim.zero_grad()
        critic2_loss.backward()
        self.critic2_optim.step()

        a, log_probs = self.actforward(obss)
        q1a, q2a = self.critic1(obss, a), self.critic2(obss, a)

        actor_loss = - torch.min(q1a, q2a).mean() + self._alpha * log_probs.mean() + 10*mmd_q_loss#- loss_deta * actor_test_mmd
        self.actor_optim.zero_grad()
        actor_loss.backward()
        self.actor_optim.step()

        if self._is_auto_alpha:
            log_probs = log_probs.detach() + self._target_entropy
            alpha_loss = -(self._log_alpha * log_probs).mean()
            self.alpha_optim.zero_grad()
            alpha_loss.backward()
            self.alpha_optim.step()
            self._alpha = torch.clamp(self._log_alpha.detach().exp(), 0.0, 1.0)

        self._sync_weight()

        result = {
            "loss/actor": actor_loss.item(),
            "loss/critic1": critic1_loss.item(),
            "loss/critic2": critic2_loss.item(),
            "loss/mmd": mmd_q_loss.item(),
            # "loss/actor_mmd": actor_test_mmd.item(),
            "loss/fack_mmd": test_mmd.item(),
            # "loss/vae_loss": vae_loss.item(),
        }

        if self._is_auto_alpha:
            result["loss/alpha"] = alpha_loss.item()
            result["alpha"] = self._alpha.item()

        return result


import json
import paho.mqtt.client as mqtt
from flask import Flask, request, render_template, jsonify
import sqlite3 as sql

#create the format for the table
TableSchema="""
drop table if exists Wind_Turbine_Data ;
create table Wind_Turbine_Data (
  DayNumber integer primary key autoincrement,
  WindSpeed text,
  EnergyEfficiency text,
  Temperature text
);
"""

#create/access the database
conn = sql.connect('database.db', check_same_thread=0)
curs = conn.cursor()

#create the tables
sql.complete_statement(TableSchema)
curs.executescript(TableSchema)

#flask name
app = Flask(__name__)

#callback function to retrieve message
def on_message(client, userdata, message):
    print("Data recieved: "+ str(message.payload.decode("utf-8")))
    print("message topic=", message.topic)
    print("adding to database")
    Wind_Turbine_Data_Handler(message.payload)

#callback function to connect to the broker
def on_connect(client, userdata, flags, rc):
    print("connected ok!")
    client.subscribe("Data")

#handles the incoming data from the mqtt broker
def Wind_Turbine_Data_Handler(jsonData):
    # Parse Data
    json_Dict = json.loads(jsonData)
    Windspeed = json_Dict['Windspeed']
    EnergyEfficiency = json_Dict['EnergyEfficiency']
    Temperature = json_Dict['Temperature']

    # Push into DB Table
    sql = ("insert into Wind_Turbine_Data (Windspeed, EnergyEfficiency, Temperature) values (?,?,?)")
    data = [Windspeed, EnergyEfficiency, Temperature]
    curs.execute(sql, data)
    conn.commit()

#creating the client and connecting it
client = mqtt.Client()
client.on_message = on_message
print("connecting to broker")
client.connect("localhost", 1833, 60)  # connect to broker

#subsribing and getting the messages
client.on_connect = on_connect
client.on_message = on_message

#starting the loop
client.loop_start()

#endpoints
@app.route('/')
def hello_world():
    return render_template('home.html')

#endpoint 1
@app.route('/All')
def getAll():
    curs.execute("SELECT * FROM Wind_Turbine_Data")
    rv = curs.fetchall()
    data = []
    content = {}
    for result in rv:
        content = {'DayNumber': result[0], 'Windspeed': result[1], 'EnergyEfficiency': result[2], 'Temperature': result[3]}
        data.append(content)
    return jsonify(data)

#endpoint 2
@app.route('/Days')
def getDays():
    curs.execute("SELECT * FROM Wind_Turbine_Data")
    rv = curs.fetchall()
    data = []
    content = {}
    for result in rv:
        content = {'DayNumber': result[0]}
        data.append(content)
    return jsonify(data)

#endpoint 3
@app.route('/Speed_and_Efficiency')
def speedEff():
    curs.execute("SELECT * FROM Wind_Turbine_Data")
    rv = curs.fetchall()
    data = []
    content = {}
    for result in rv:
        content = {'Windspeed': result[1], 'EnergyEfficiency': result[2]}
        data.append(content)
    return jsonify(data)

#endpoint 4
@app.route('/TemperatureOverDays')
def list():
    curs.execute("SELECT * FROM Wind_Turbine_Data")
    rv = curs.fetchall()
    data = []
    content = {}
    for result in rv:
        content = {'DayNumber': result[0], 'Temperature': result[3]}
        data.append(content)
    return jsonify(data)

#running flask
if __name__ == '__main__':
    app.run()


from django.shortcuts import render
from django.http import JsonResponse
from .exceptions import UserDataException
import logging
from .services import (main_service,
                       get_list_messages_service,
                       get_one_message_service)

logger = logging.getLogger(__name__)

def main(request):
    """Обработчик основной страницы"""
    data = main_service(request)
    return render(request, 'core/index_list.html',
                  {'login': data.login, 'domain': data.domain})

def get_messages(request):
    """Получаем список сообщений"""
    try:
        messages = get_list_messages_service(request)
    except UserDataException as e:
        logger.error(e)
    return JsonResponse(messages.data, safe=False)

def get_one_message(request, id):
    """Получаем одно сообщение"""
    try:
        messages = get_one_message_service(request, id)
    except UserDataException as e:
        logger.error(e)
        return render(request, 'core/mail.html', {'error': e})
    return render(request, 'core/mail.html',
                  {'message': messages.data._asdict(),
                   'login': messages.user_data.login,
                   'domain': messages.user_data.domain})


from django.urls import path
from engine.tasks.views import *

urlpatterns = [
    path('create', TaskCreateView.as_view(), name='create-task'),
    path('delete/<int:pk>', TaskDeleteView.as_view(), name='delete-task'),
    path('update/<int:pk>', TaskUpdateView.as_view(), name='update-task'),
    path('read/<int:pk>', TaskReadView.as_view(), name='read-task'),
    path('done/<int:pk>', DoneView.as_view(), name='done-task'),
]


import serial
import numpy as np
import matplotlib.pyplot as plt
import time as t
import paho.mqtt.client as paho

serdev = '/dev/ttyUSB0'

s = serial.Serial(serdev, 9600)

mqttc = paho.Client()

host = "localhost"

port = 1883

# MQTT Callbacks

def on_connect(self, mosq, obj, rc):

    print("Connected rc: " + str(rc))

def on_message(mosq, obj, msg):

    print("[Received] Topic: " + msg.topic + ", Message: " + str(msg.payload) + "\n");

def on_subscribe(mosq, obj, mid, granted_qos):

    print("Subscribed OK")

def on_unsubscribe(mosq, obj, mid, granted_qos):

    print("Unsubscribed OK")

# Set callbacks

mqttc.on_message = on_message

mqttc.on_connect = on_connect

mqttc.on_subscribe = on_subscribe

mqttc.on_unsubscribe = on_unsubscribe

# Connect and subscribe to MQTT

print("Connecting to " + host )

mqttc.connect(host, port=1883, keepalive=60)
mqttc.subscribe("X", 0)
mqttc.subscribe("Y", 0)
mqttc.subscribe("Z", 0)
mqttc.subscribe("tilt", 0)
mqttc.subscribe("plot", 0)

# Variables for accel value

num_time=np.arange(0,10)
velo = np.arange(0,10,dtype=float)
a=0 #the index

#xbee settings

s.write("+++".encode())

char = s.read(2)

print("Enter AT mode.")

print(char.decode())

s.write("ATMY 0x140\r\n".encode())

char = s.read(3)

print("Set MY 0x140.")

print(char.decode())

s.write("ATDL 0x240\r\n".encode())

char = s.read(3)

print("Set DL 0x240.")

print(char.decode())

s.write("ATID 0x1\r\n".encode())

char = s.read(3)

print("Set PAN ID 0x1.")

print(char.decode())

s.write("ATWR\r\n".encode())

char = s.read(3)

print("Write config.")

print(char.decode())

s.write("ATMY\r\n".encode())

char = s.read(4)

print("MY :")

print(char.decode())

s.write("ATDL\r\n".encode())

char = s.read(4)

print("DL : ")

print(char.decode())

s.write("ATCN\r\n".encode())

char = s.read(3)

print("Exit AT mode.")

print(char.decode())

print("start sending RPC")

s.write(bytes("\r", 'UTF-8'))

line=s.readline()

print(line)

line=s.readline()

print(line)

t.sleep(1)

for i in range(0, 10):
    s.write(bytes("/getData/run\n", 'UTF-8'))
    line=s.readline()
    print(line)
    line=s.readline()
    print(line)

    velo[i] = float(line)
    print("velo[i]="+str(velo[i]))
    t.sleep(1)

for i in range(0,10):
    print(velo[i])
for i in range(0,10):
    print("mqtt publish")
    mqttc.publish("velocity", velo[i])
    print("send velo"+str(velo[i]))
    t.sleep(0.1)


import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import requests
import io
import math
from scipy import stats
import numpy as np
import researchpy as rp
import statsmodels.api as sm
import pprint

# -----------------------------------------------------------------------------
# ---------------------------- EXTRACCIÓN DE DATOS ----------------------------
# -----------------------------------------------------------------------------
string_csv = requests.get("https://gitlab.com/drvicsana/ciencia-datos/-/raw/master/datasets/fifa/players_20.csv").content
string_stream = io.StringIO(string_csv.decode("utf-8"))
player_df = pd.read_csv(string_stream)

# -----------------------------------------------------------------------------
# ---------------------------- EXPLORACIÓN DE DATOS ---------------------------
# -----------------------------------------------------------------------------

# numero de filas y columnas del df
pprint.pprint(player_df.shape)

# descripcion de los datos
pprint.pprint(player_df.describe())

# encabezado de los datos
pprint.pprint(player_df.head(5))

# descripcion categorica
print(player_df.describe(include=["object"]))

# -----------------------------------------------------------------------------
# ---------------------------- FILTRADO DE DATOS ------------------------------
# -----------------------------------------------------------------------------

# filtrado para equipos españoles
spanish_teams = ["Real Madrid", "FC Barcelona", "Atlético Madrid", "Valencia CF", "Villarreal CF", "Real Valladolid CF",
                 "Athletic Club de Bilbao", "Cádiz CF", "Granada CF", "Real Betis", "RCD Espanyol", "Real Sociedad", "Deportivo Alavés", "Elche CF", "Sevilla FC",
                 "CA Osasuna", "SD Eibar", "RC Celta", "Levante UD", "SD Huesca" ]

# posiciones posibles de cada jugador
defender_positions = [ "SW", "RWB", "LWB", "RB", "LB", "CB" ]
midfield_positions = [ "DM", "RW", "LW", "LM", "RM", "CM", "AM" ]
forward_positions = [ "CF", "RF", "LF", "ST" ]
goalkeeper_positions = ["GK"]

# crear nuevo df con jugadores españoles
spanish_players_df = player_df[ player_df.club.isin(spanish_teams) ]

# descripcion de los datos
pprint.pprint(spanish_players_df.describe())

# encabezado de los datos
pprint.pprint(spanish_players_df.head(5))

# descripcion categorica
print(spanish_players_df.describe(include=["object"]))

# -----------------------------------------------------------------------------
# ----------------------- CREACIÓN DE NUEVOS DATOS (COLUMNAS) -----------------
# -----------------------------------------------------------------------------

########### Funciones para definir la posición principal de un jugador ###########

# mete las posiciones de un jugador en una lista
def from_string_position_to_list(positions_string):
    return positions_string.replace(" ", "").split(",")

# dice si pertenece a la lista prospective
def is_position(positions_string, prospective_positions):
  list_positions = from_string_position_to_list(positions_string)
  found = False
  for position in list_positions :
    if position in prospective_positions :
      found = True
  return found

# decide a que posición pertenece
def determine_main_position(positions_string):
  count_defender = 0
  count_midfield = 0
  count_forward = 0
  count_goalkeeper = 0
  list_positions = from_string_position_to_list(positions_string)
  for position in list_positions :
    if position in defender_positions :
      count_defender = count_defender + 1
    elif position in midfield_positions :
      count_midfield = count_midfield + 1
    elif position in forward_positions :
      count_forward = count_forward + 1
    elif position in goalkeeper_positions :
      count_goalkeeper = count_goalkeeper + 1

  return max( [ (count_defender, "DF"), (count_midfield, "MF"), (count_forward, "FW"), (count_goalkeeper, "GK") ] )[1]

# funciones lambda que dice si un jugador es o no de una posición (esto testea todas las posiciones en las que juega el jugador indicado).
is_defender = lambda s : is_position(s, defender_positions)
is_midfielder = lambda s : is_position(s, midfield_positions)
is_forward = lambda s : is_position(s, forward_positions)
is_goalkeeper = lambda s: is_position(s, goalkeeper_positions)

# crea la columna "main_position" en la cual muestra la posición del player
spanish_players_df = spanish_players_df.assign( main_position = spanish_players_df.apply( lambda row: determine_main_position(row.player_positions), axis=1 ) )

# * determina si en algun caso el jugador es defensa * (utiliza la lista con todas las posiciones en las que juega el jugador).
spanish_players_df = spanish_players_df.assign( is_defender = spanish_players_df.apply( lambda row : is_position(row.player_positions, defender_positions), axis=1 ))

# determina si el jugador tiene como posicion principal defensa
spanish_players_df = spanish_players_df.assign( is_defender = spanish_players_df.apply( lambda row : True if ("DF" == row.main_position) else False, axis=1 ))
# determina si el jugador tiene como posicion principal delantero
spanish_players_df = spanish_players_df.assign( is_forward = spanish_players_df.apply( lambda row : True if ("FW" == row.main_position) else False, axis=1 ))

########### Definir de que continente es cada jugador en una nueva columna ###########

# definir cada continente
country_to_continent = { "Argentina": "America", "Slovenia": "Europe", "Belgium":"Europe", "Germany": "Europe", "Croatia":"Europe",
                        "Spain": "Europe", "Uruguay":"America", "France":"Europe", "Brazil":"America", "Costa Rica":"America",
                        "Netherlands":"Europe", "Wales": "Europe", "Colombia":"America", "Chile":"America", "Serbia":"Europe",
                        "Portugal":"Europe", "Ghana":"Africa", "Montenegro":"Europe", "Mexico":"America", "Denmark":"Europe",
                        "Central African Rep.":"Africa", "Czech Republic":"Europe", "Algeria":"Africa", "Israel":"Asia",
                        "England":"Europe", "Slovakia":"Europe", "Dominican Republic":"America", "Cameroon":"Africa",
                        "China PR":"Asia", "Japan":"Asia", "Russia":"Asia", "Norway":"Europe", "Turkey":"Asia",
                        "FYR Macedonia":"Europe", "Senegal":"Africa", "Morocco":"Africa", "Italy":"Europe",
                        "Ukraine":"Europe", "Korea Republic":"Asia", "Nigeria":"Africa", "Sweden":"Europe", "Venezuela":"America",
                        "Bosnia Herzegovina":"Europe", "Ivory Coast":"Africa", "Romania":"Europe", "Equatorial Guinea":"Africa",
                        "Ecuador":"America", "Albania":"Europe", "Mauritania":"Africa", "Benin":"Africa", "Switzerland":"Europe",
                        "DR Congo":"Africa"}

spanish_players_df.apply( lambda row: row.age*2, axis=1 )

# crea la columna "continent" en la cual muestra el continente del player
spanish_players_df = spanish_players_df.assign( continent = spanish_players_df.apply(lambda row: country_to_continent[row.nationality], axis=1) )

# -----------------------------------------------------------------------------
# ---------------- ESTADISTICA DESCRIPTIVA (DATOS CATEGÓRICOS) ----------------
# -----------------------------------------------------------------------------

# frecuencia absoluta por posición principal
pprint.pprint(spanish_players_df.main_position.value_counts())

# frecuencia relativa por posicion principal
pprint.pprint(spanish_players_df.main_position.value_counts(normalize=True))

# gráfico de queso de jugadores por posición
count_data = spanish_players_df.main_position.value_counts()
plt.figure(figsize=(10, 10))
fig = plt.pie( count_data, labels = count_data.index, autopct="%.2f", textprops={"fontsize":14} )

# gráfico de queso de jugadores por continente
count_data = spanish_players_df.continent.value_counts()
plt.figure(figsize=(10,10))
fig = plt.pie( count_data, labels = count_data.index, autopct="%.2f", textprops={"fontsize":14} )

# gráfico de barras donde se muestran la frecuencia de cada posición
count_data = spanish_players_df.main_position.value_counts(normalize=True)
axes = sns.barplot( x=count_data.index, y=count_data )
axes.set_xlabel("Position")
axes.set_ylabel("Percentage")
axes.set_title("Player positions")

# gráfico de barras donde se muestran la frecuencia del continente
count_data = spanish_players_df.continent.value_counts(normalize=True)
axes = sns.barplot( x=count_data.index, y=count_data )
axes.set_xlabel("Continent")
axes.set_ylabel("Percentage")
axes.set_title("Player continent")

#### TABLA DE CONTINGENCIAS ####

# tabla de contingencia del pie que usan los jugadores por posición principal
pd.crosstab( spanish_players_df.preferred_foot, spanish_players_df.main_position )

# lo mismo pero normalizado
pd.crosstab( spanish_players_df.preferred_foot, spanish_players_df.main_position, normalize=True )

# frecuencias condicionada a la preferencia del pie
pd.crosstab( spanish_players_df.preferred_foot, spanish_players_df.main_position, normalize = "index" )

# ahora las frecuencias condicionadas a la posición que juegan
pd.crosstab( spanish_players_df.preferred_foot, spanish_players_df.main_position, normalize = "columns" )

# ahora las frecuencias condicionadas al continente al que pertenecen
pd.crosstab( spanish_players_df.preferred_foot, spanish_players_df.continent, normalize = "columns" )

# -----------------------------------------------------------------------------
# ---------------- ESTADISTICA DESCRIPTIVA (DATOS NUMÉRICOS) ------------------
# -----------------------------------------------------------------------------

#### HISTOGRAMAS ####

# histograma del overall del conjunto de datos, bins: numero de cubetas en el histograma
sns.histplot(data=spanish_players_df, x="overall", bins=30)

# con probabilidad (la suma de las alturas de las cubetas será 1)
sns.histplot(data=spanish_players_df, x="overall", bins=30, stat="probability")

# con densidad (el área del histograma será 1)
sns.histplot(data=spanish_players_df, x="overall", bins=30, stat="density")

# comparación de distintos histogramas
# - x: datos a analizar (variable aleatoria) ,
# - hue: crea un histograma para cada clase de esta columna (diferencia entre histogramas).
sns.histplot(data=spanish_players_df, x="overall", bins=30, hue="preferred_foot", stat="density", common_norm=False)

# histograma por edad (densidades)
sns.histplot(data=spanish_players_df, x="age", bins=30, stat="density")

# histograma por salarios (densidades)
sns.histplot(data=spanish_players_df, x="wage_eur", bins=40, stat="density")

# comparación de histogramas con variable aleatoria edad diferenciando por dependencia de pie dominante
sns.histplot(data=spanish_players_df, x="age", bins=30, hue="preferred_foot", stat="density", common_norm=False)

#### BOX WHISKERS ####

# Caja Box Whiskers con info de los jugadores
sns.boxplot( x=spanish_players_df.overall )

# Dos cajas Box Whiskers en el mismo plot
sns.boxplot( x=spanish_players_df.preferred_foot, y=spanish_players_df.overall )

# -----------------------------------------------------------------------------
# ----------------------- INFERENCIA SOBRE PROPORCIONES -----------------------
# -----------------------------------------------------------------------------

# cogemos una muestra
small_sample = player_df.sample(n=100)

# calculamos la proporcion en la muestra (contamos los jugadores zurdos)
p = small_sample[ small_sample.preferred_foot == "Left" ].shape[0]/100
print(p)

#### INTERVALO DE CONFIANZA ####

# Genera 10 datos aleatorios procedentes de una normal con media 20 y desviación típica 2
random_data = stats.norm.rvs(loc=20, scale=2, size= 10)
print(random_data)
# Obtiene el valor de la función de densidad para el punto 21.3 en una normal con media 20 y desviación típica 2
print( stats.norm.pdf( 21.3, loc=20, scale=2 ) )
# Obtiene la probabilidad de observar valores menores o iguales a cero en una distribución normal con media 0 y desviación típica 1
print( stats.norm.cdf( 0 ,loc=0, scale=1 ) )
# Obtiene el percentil 97.5 (mayor o igual que el 97.5% de las posibles observaciones) para una normal con media 0 y desviación típica 1
print( stats.norm.ppf( 0.975, loc = 0, scale = 1 ) )
# Obtiene los extremos del intervalo centrado en la media que contiene el 95% de las observaciones en una normal con media 0 y desviación típica 1
print( stats.norm.interval( 0.95, loc=0, scale=1 ) )
# construimos el intervalo de confianza
lower_bound = 0
upper_bound = 0
Zalpha = stats.norm.ppf( 0.975, loc=0, scale=1 )
deviation = Zalpha * math.sqrt( p*(1-p)/100 )
interval = ( p - deviation, p + deviation )
print(interval)

# parametro poblacional de los jugadores de FIFA zurdos
true_p = player_df[ player_df.preferred_foot == "Left" ].shape[0]/player_df.shape[0]
print(true_p)

# intervalo de confianza de 95% con 100 muestras
small_sample = player_df.sample(n=100)
p = small_sample[ small_sample.preferred_foot == "Left" ].shape[0]/100
Zalpha = stats.norm.ppf( 0.975, loc=0, scale=1 )
deviation = Zalpha * math.sqrt( p*(1-p)/100 )
interval = ( p - deviation, p + deviation )
print("Intervalo de 95% (100 muestras) -> ",interval)
print("Tamaño -> ",((p + deviation)-(p - deviation)))

# intervalo de confianza de 95% con 1000 muestras
sample = player_df.sample(n=1000)
p1 = sample[ sample.preferred_foot == "Left" ].shape[0]/1000
Zalpha = stats.norm.ppf( 0.975, loc=0, scale=1 )
deviation_1 = Zalpha * math.sqrt( p1*(1-p1)/100 )
interval = ( p1 - deviation_1, p1 + deviation_1 )
print("Intervalo de 95% (1000 muestras) -> ",interval)
print("Tamaño -> ",((p1 + deviation_1)-(p1 - deviation_1)))

# se hacen 100 muestras en las que se testea si el intervalo de confianza contiene o no el valor, se representa una linea roja
# donde, si cruza con el intervalo es que ha acertado, tiene un % de acierto del 95%.
number_success = 0
intervals = []
n_rep = 100
for i in range(n_rep):
  n = 100

  small_sample = player_df.sample(n=n)

  p = small_sample[ small_sample.preferred_foot == "Left" ].shape[0]/n

  lower_bound = 0
  upper_bound = 0
  Zalpha = stats.norm.ppf( 0.975, loc=0, scale=1 )
  deviation = Zalpha * math.sqrt( p*(1-p)/n )
  interval = ( p - deviation, p + deviation )

  intervals.append(interval)

  if true_p >= interval[0] and true_p <= interval[1]:
    number_success = number_success + 1

print(number_success/n_rep)

plt.figure(figsize=(30,10))
plt.errorbar( x=list(range(1, n_rep+1)), y=[ (x[1]+x[0])/2 for x in intervals ], yerr=[ (x[1]-x[0])/2 for x in intervals ], fmt='o' )
plt.hlines( xmin=1, xmax=100, y=true_p, colors="red" )

#### PRUEBAS DE NORMALIDAD ####

sample = player_df.sample(n=100)

# prueba de normalidad de asimetria
#   H0:  El coeficiente de asimetría (CA) de la población sigue una asimetria propia de una población normal (CA = 0)
#   H1:  El coeficiente de asimetría (CA) de la población sigue una asimetría diferente de una población normal (CA != 0)
# si pvalue<=0.05 -> H1
# si pvalue>0.05 -> H0
print( stats.skewtest( sample.overall ) )

# prueba de normalidad de curtosis
#   H0:  El coeficiente de curtosis (CC) de la población sigue una curtosis propia de una población normal (CC=0)
#   H1:  El coeficiente de curtosis (CC) de la población tiene una curtosis diferente a la de una población normal (CC!=0)
# si pvalue<=0.05 -> H1
# si pvalue>0.05 -> H0
print( stats.kurtosistest( sample.overall ) )

# prueba de Kolmogorov-Smirnov
#   H0:  La función de distribución acumulada se comporta como la función de distribución acumulada de una distribución normal.
#   H1: La función de distribución acumulada difiere de la función de distribución acumulada de una distribución normal.
# si pvalue<=0.05 -> H1
# si pvalue>0.05 -> H0
stats.kstest( (sample.overall-sample.overall.mean())/sample.overall.std(), 'norm'  )

# prueba de Shapiro-Wilk
#   H0:  La variable aleatoria estudiada procede de una población normal
#   H1: La variable aleatoria estudiada no procede de una población normal
# si pvalue<=0.05 -> H1
# si pvalue>0.05 -> H0
stats.shapiro( sample.overall )

#### PAPEL PROBABILISTICO ####

# gráfico probabilistico (papel probabilistico) de overall con respecto a una normal
plot = sm.ProbPlot( sample.overall, dist="norm" )
plot.probplot(line="r")

#### PRUEBAS DE NORMALIDAD Y MUESTRAS GRANDES ####

# vamos a ver que ocurre con las pruebas en las que no pudimos
# descartar normalidad cuando la muestra es proporcionada es grande
new_sample1 = player_df.sample(n=1000)
new_sample2 = player_df.sample(n=2000)
new_sample3 = player_df.sample(n=4000)

print( stats.shapiro( new_sample1.overall ) )
print( stats.shapiro( new_sample2.overall ) )
print( stats.shapiro( new_sample3.overall ) )

print(stats.kstest( (new_sample1.overall-new_sample1.overall.mean())/new_sample1.overall.std(), 'norm'  ))
print(stats.kstest( (new_sample2.overall-new_sample2.overall.mean())/new_sample2.overall.std(), 'norm'  ))
print(stats.kstest( (new_sample3.overall-new_sample3.overall.mean())/new_sample3.overall.std(), 'norm'  ))

# analisis de toda la población
# (kde -> kernel density estimate: estimiacion de la funcion de densidad no parametrica)
axes = sns.kdeplot( (player_df.overall - player_df.overall.mean() )/(player_df.overall.std()), common_norm=False  )
# representación de una normal para comparar con el kde
sns.lineplot( y=[ stats.norm.pdf(x) for x in np.arange(-3, 3, 0.01) ], x= [x for x in np.arange(-3, 3, 0.01)] )
# histograma
sns.histplot(player_df.overall, bins=25)
# gráfico de probabilidad normal
plot = sm.ProbPlot( player_df.overall, dist="norm" )
plot.probplot(line="r")

# -----------------------------------------------------------------------------
# ------------ INFERENCIA SOBRE LA MEDIA DE POBLACIONES NORMALES --------------
# -----------------------------------------------------------------------------

#### INTERVALO DE CONFIANZA CON T DE STUDENT ####

# genera 10 datos aleatorios procedentes de una t de Student con 4 grados de libertad
# rvs -> random variates of given type
random_data = stats.t.rvs( 4, size= 10)
print(random_data)

# obtiene el valor de la función de densidad para el punto 1 en una t de Student con 4 grados de libertad
print( stats.t.pdf( 1, 4 ) )

# obtiene la probabilidad de observar valores menores o iguales a cero en una distribución t de student con 4 grados de libertad
print( stats.t.cdf( 0, 4 ) )

# obtiene el percentil 97.5 (mayor o igual que el 97.5% de las posibles observaciones) para una t de student con 4 grados de libertad
print( stats.t.ppf( 0.975, 4 ) )

# obtiene los extremos del intervalo centrado en 0 que contiene el 95% de las observaciones en una t de student con 4 grados de libertad
print( stats.t.interval( 0.95, 4 ) )

# intervalo de confianza del 95%
lower_bound = 0
upper_bound = 0
x = sample.overall.mean()
n = sample.shape[0]
t = stats.t.ppf(0.975, n-1 )
s = sample.overall.std()
lower_bound = x - t * s/math.sqrt(n)
upper_bound = x + t * s/math.sqrt(n)
print(lower_bound, upper_bound)
print("media muestral -> ",x)

# analisis de normalidad para el peso en kg de los jugadores
sample = player_df.sample(n=100)
print(" alpha -> 0.05 \n")
print( stats.skewtest( sample.weight_kg ) , "\n -> H0:  El coeficiente de asimetría (CA) de la población sigue una asimetria propia de una población normal (CA = 0)\n" )
print( stats.kurtosistest( sample.weight_kg ) , "\n -> H0:  El coeficiente de curtosis (CC) de la población sigue una curtosis propia de una población normal (CC=0)\n")
print( stats.kstest( (sample.weight_kg-sample.weight_kg.mean())/sample.weight_kg.std(), 'norm'  ) , "\n -> H0:  La función de distribución acumulada se comporta como la función de distribución acumulada de una distribución normal.\n")
print( stats.shapiro( sample.weight_kg ) , "\n -> H0:  La variable aleatoria estudiada procede de una población normal\n")
plot = sm.ProbPlot( sample.weight_kg, dist="norm" )
plot.probplot(line="r")
print("Despues de hacer varios tipos de pruebas de normalidad y todas diciendo que la distribución sigue una normal, concluimos que sigue una normal.\n")
# intervalo de confianza para la media de la variable weight_kg
lower_bound = 0
upper_bound = 0
x = sample.weight_kg.mean()
n = sample.shape[0]
t = stats.t.ppf(0.975, n-1 )
s = sample.weight_kg.std()
lower_bound = x - t * s/math.sqrt(n)
upper_bound = x + t * s/math.sqrt(n)
print("(lower_bound: ",lower_bound,", upper_bound: ", upper_bound,")")
print("media muestral -> ",x)

# -----------------------------------------------------------------------------
# ----------- INFERENCIA SOBRE LA VARIANZA EN POBLACIONES NORMALES ------------
# -----------------------------------------------------------------------------

#### INTERVALO DE CONFIANZA DE CHI-CUADRADO ####

# genera 10 datos aleatorios procedentes de una Chi cuadrado con 4 grados de libertad
random_data = stats.chi2.rvs( 4, size= 10)
print(random_data)

# obtiene el valor de la función de densidad para el punto 2 en una chi cuadrado con 4 grados de libertad
print( stats.chi2.pdf( 2, 4 ) )

# obtiene la probabilidad de observar valores menores o iguales a 1 en una distribución chi cuadrado con 4 grados de libertad
print( stats.chi2.cdf( 1, 4 ) )

# obtiene el percentil 97.5 (mayor o igual que el 97.5% de las posibles observaciones) para una chi cuadrado 4 grados de libertad
print( stats.chi2.ppf( 0.975, 4 ) )

# obtiene los extremos del intervalo que contiene el 95% de las observaciones en una chi cuadrado con 4 grados de libertad
print( stats.chi2.interval( 0.95, 4 ) )

# intervalo de confianza 95% para la varianza poblacional de overall
lower_bound = 0
upper_bound = 0
n = sample.shape[0]
s2 = sample.overall.std()
chi1 = stats.chi2.ppf(1-0.05/2, n - 1 )
chi2 = stats.chi2.ppf(0.05/2, n - 1 )
lower_bound = (n-1)*s2/chi1
upper_bound = (n-1)*s2/chi2
print(lower_bound, upper_bound)
print("varianza muestral -> ",s2)

# intervaol de confianza 99% para la varianza poblacional de weight_kg
lower_bound = 0
upper_bound = 0
n = sample.shape[0]
s2 = sample.weight_kg.std()
chi1 = stats.chi2.ppf(1-0.01/2, n - 1 )
chi2 = stats.chi2.ppf(0.01/2, n - 1 )
lower_bound = (n-1)*s2/chi1
upper_bound = (n-1)*s2/chi2
print(lower_bound, upper_bound)
print("varianza muestral -> ",s2)

# -----------------------------------------------------------------------------
# ---------------------- EXPLORACION DE DATOS (TAREA) -------------------------
# -----------------------------------------------------------------------------

# analizar normalidad y sacar los intervalos de confianza de media y varianza de 3 muestras del conjunto

sample = spanish_players_df.sample(n=100)

# histogramas
plt.figure(figsize=(20,5))
plt.subplot(1,3,1)
sns.histplot(data=spanish_players_df, x="potential", bins=30)
plt.title('Potential')
plt.subplot(1,3,2)
sns.histplot(data=spanish_players_df, x="international_reputation", bins=30)
plt.title('International Reputation')
plt.subplot(1,3,3)
sns.histplot(data=spanish_players_df, x="power_stamina", bins=30)
plt.title('Power Stamina')

# Gráficos probabilisticas
plt.figure()
sm.ProbPlot( sample.potential, dist="norm" ).probplot(line="r")
plt.title('Potential')
sm.ProbPlot( sample.international_reputation, dist="norm" ).probplot(line="r")
plt.title('International Reputation')
sm.ProbPlot( sample.power_stamina, dist="norm" ).probplot(line="r")
plt.title('Power Stamina')

# Pruebas de normalidad
print(" alpha -> 0.05 \n")

print("       Potential:\n")
print( stats.skewtest( sample.potential ) , "\n    -> H0:  El coeficiente de asimetría (CA) de la población sigue una asimetria propia de una población normal (CA = 0)" )
print( stats.kurtosistest( sample.potential ) , "\n    -> H0:  El coeficiente de curtosis (CC) de la población sigue una curtosis propia de una población normal (CC=0)")
print( stats.kstest( (sample.potential-sample.potential.mean())/sample.potential.std(), 'norm'  ) , "\n    -> H0:  La función de distribución acumulada se comporta como la función de distribución acumulada de una distribución normal.")
print( stats.shapiro( sample.potential ) , "\n     -> H0:  La variable aleatoria estudiada procede de una población normal")
print("**Despues de hacer varios tipos de pruebas de normalidad y todas diciendo que la distribución sigue una normal, concluimos que sigue una normal.**\n")

print("\n\n       International Reputation:\n")
print( stats.skewtest( sample.international_reputation ) , "\n    -> H1:  El coeficiente de asimetría (CA) de la población sigue una asimetría diferente de una población normal (CA != 0)" )
print( stats.kurtosistest( sample.international_reputation ) , "\n    -> H1:  El coeficiente de curtosis (CC) de la población tiene una curtosis diferente a la de una población normal (CC!=0)")
print( stats.kstest( (sample.international_reputation-sample.international_reputation.mean())/sample.international_reputation.std(), 'norm'  ) , "\n    -> H1:  La función de distribución acumulada difiere de la función de distribución acumulada de una distribución normal.")
print( stats.shapiro( sample.international_reputation ) , "\n     -> H1:  La variable aleatoria estudiada no procede de una población normal")
print("**Despues de hacer varios tipos de pruebas de normalidad y todas diciendo que la distribución NO sigue una normal, concluimos que NO sigue una normal.**\n")

print("\n\n       Power Stamina:\n")
print( stats.skewtest( sample.power_stamina ) , "\n    -> H1:  El coeficiente de asimetría (CA) de la población sigue una asimetría diferente de una población normal (CA != 0)" )
print( stats.kurtosistest( sample.power_stamina ) , "\n    -> H0:  El coeficiente de curtosis (CC) de la población sigue una curtosis propia de una población normal (CC=0)")
print( stats.kstest( (sample.power_stamina-sample.power_stamina.mean())/sample.power_stamina.std(), 'norm'  ) , "\n    -> H0:  La función de distribución acumulada se comporta como la función de distribución acumulada de una distribución normal.")
print( stats.shapiro( sample.power_stamina ) , "\n     -> H1:  La variable aleatoria estudiada no procede de una población normal")
print("**Despues de hacer varios tipos de pruebas de normalidad, aunque el de asimetría y la de shapiro dicen que la distribución no sigue una normal, observando el gráfico probabilistico y el histograma podemos concluir que sigue una normal.**\n")

# Intervalos de confianza

# Media
print("\n       Media:")
print("Potential")
lower_bound = 0
upper_bound = 0
x = sample.potential.mean()
n = sample.shape[0]
t = stats.t.ppf(0.975, n-1 )
s = sample.potential.std()
lower_bound = x - t * s/math.sqrt(n)
upper_bound = x + t * s/math.sqrt(n)
print("   Hay un 95% de probabilidad de que las muestras tengan una media de entre: (",lower_bound,", ", upper_bound,").")

print("Power Stamina")
lower_bound_1 = 0
upper_bound_1 = 0
x_1 = sample.power_stamina.mean()
n_1 = sample.shape[0]
t_1 = stats.t.ppf(0.975, n_1-1 )
s_1 = sample.power_stamina.std()
lower_bound_1 = x_1 - t_1 * s_1/math.sqrt(n_1)
upper_bound_1 = x_1 + t_1 * s_1/math.sqrt(n_1)
print("   Hay un 95% de probabilidad de que las muestras tengan una media de entre: (",lower_bound_1,", ", upper_bound_1,").")

# Varianza
print("\n       Varianza:")
print("Potential")
lower_bound_2 = 0
upper_bound_2 = 0
n_2 = sample.shape[0]
s2_2 = sample.potential.std()
chi1_2 = stats.chi2.ppf(1-0.05/2, n_2 - 1 )
chi2_2 = stats.chi2.ppf(0.05/2, n_2 - 1 )
lower_bound_2 = (n_2-1)*s2_2/chi1_2
upper_bound_2 = (n_2-1)*s2_2/chi2_2
print("   Hay un 95% de probabilidad de que las muestras tengan una varianza de entre: (",lower_bound_2,", ", upper_bound_2,").")

print("Power Stamina")
lower_bound_3 = 0
upper_bound_3 = 0
n_3 = sample.shape[0]
s2_3 = sample.power_stamina.std()
chi1_3 = stats.chi2.ppf(1-0.05/2, n_3 - 1 )
chi2_3 = stats.chi2.ppf(0.05/2, n_3 - 1 )
lower_bound_3 = (n_3-1)*s2_3/chi1_3
upper_bound_3 = (n_3-1)*s2_3/chi2_3
print("   Hay un 95% de probabilidad de que las muestras tengan una varianza de entre: (",lower_bound_3,", ", upper_bound_3,").")


from __future__ import print_function

import sys
import re
from csv import reader
from operator import add
from pyspark import SparkContext

def replace_blank(row):
    for index in range(0, 24):
        if row[index] == '':
            row[index] = 'UNDEFINED'
    return row

def dump_csv(row):
    line = ""
    for i in range(0, 24):
        if row[i] == '' or "," not in row[i]:
            line += row[i]
        else:
            line += '\"' + row[i] + '\"'
        if i != 23:
            line += ','
    return line

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: spark_replace_blank_columns <file>", file=sys.stderr)
        exit(-1)
    sc = SparkContext()
    lines = sc.textFile(sys.argv[1], 1)
    lines = lines.mapPartitions(lambda x : reader(x)).map(replace_blank).map(dump_csv)
    lines.saveAsTextFile("replace_blank_columns.out")
    sc.stop()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import configparser
'''
#生成一个配置文件
config = configparser.ConfigParser()
config["DEFAULT"] = {'ServerAliveInterval': '45',
                      'Compression': 'yes',
                     'CompressionLevel': '9'}

config['bitbucket.org'] = {}
config['bitbucket.org']['User'] = 'hg'
config['topsecret.server.com'] = {}
topsecret = config['topsecret.server.com']
topsecret['Host Port'] = '50022'     # mutates the parser
topsecret['ForwardX11'] = 'no'  # same here
config['DEFAULT']['ForwardX11'] = 'yes'
with open('example.cfg', 'w') as configfile:
   config.write(configfile)
'''

#读取
config = configparser.ConfigParser()
#config.sections()
config.read('example.ini')

# ########## 读 ##########
#secs = config.sections()
#print(secs)
#options = config.options('group2')
#print(options)

# 如果DEFAULT默认区段的配置文件和sections字段的配置起冲突的话,sections优先级高
#item_list = config.items('group2')
#print(item_list)

#val = config.get('group1','key')
#val = config.getint('group1','key')

# ########## 改写 ##########
sec = config.remove_section('group1')
config.write(open('i.cfg', "w"))

sec = config.has_section('wupeiqi')
sec = config.add_section('wupeiqi')
config.write(open('i.cfg', "w"))

#config.set('group2','k1',11111)
#config.write(open('i.cfg', "w"))

#config.remove_option('group2','age')
#config.write(open('i.cfg', "w"))


from __future__ import annotations

from logging import getLogger
from typing import Any, Dict, List, Mapping, Optional, Sequence, Set, Union

from buildarr.config import RemoteMapEntry
from buildarr.types import NonEmptyStr
from pydantic import Field, validator
from typing_extensions import Annotated, Self

from ...api import api_delete, api_get, api_post, api_put
from ...secrets import SonarrSecrets
from ..types import SonarrConfigBase

logger = getLogger(__name__)

class QualityGroup(SonarrConfigBase):
    """
    Quality group.

    Allows groups of quality definitions to be given the same prorioty in qualtity profiles.
    """

    name: NonEmptyStr
    members: Set[NonEmptyStr] = Field(..., min_items=1)

    def encode(self, group_id: int, quality_definitions: Mapping[str, Any]) -> Dict[str, Any]:
        return {
            "id": group_id,
            "name": self.name,
            "allowed": True,
            "items": [
                _encode_quality_str(quality_definitions, member, True) for member in self.members
            ],
        }

class QualityProfile(SonarrConfigBase):
    """
    The main things to consider when creating a quality profile are
    what quality settings to enable, and how to prioritise each.

    ```yaml
    ...
      quality_profiles:
        SDTV:
          upgrades_allowed: true
          upgrade_until: "Bluray-1080p"
          qualities:
          - "Bluray-480p"
          - "DVD"
          - name: "WEB 480p"
            members:
              - "WEBDL-480p"
              - "WEBRip-480p"
          - "SDTV"
    ```

    In Buildarr, the quality listed first (at the top) is given the highest priority, with
    subsequent qualities given lower priority. Qualities not explicitly defined are
    disabled (not downloaded).

    Sonarr supports grouping multiple qualities together to give them the same priority.
    In Buildarr, these are expressed by giving a `name` to the group, and listing the qualities
    under the `members` attribute.

    For more insight into reasonable values for quality profiles,
    refer to these guides from [WikiArr](https://wiki.servarr.com/sonarr/settings#quality-profiles)
    and TRaSH-Guides ([WEB-DL](https://trash-guides.info/Sonarr/Sonarr-Release-Profile-RegEx/),
    [anime](https://trash-guides.info/Sonarr/Sonarr-Release-Profile-RegEx-Anime/)).
    """

    upgrades_allowed: bool = False
    """
    Enable automatic upgrading if a higher quality version of the media file becomes available.

    If disabled, media files will not be upgraded after they have been downloaded.
    """

    qualities: Annotated[List[Union[NonEmptyStr, QualityGroup]], Field(min_items=1)]
    """
    The qualities to enable downloading episodes for. The order determines the priority
    (highest priority first, lowest priority last).

    Individual qualities can be specified using the name (e.g. `Bluray-480p`).

    Qualities can also be grouped together in a structure to give them the same
    priority level. A new version of the episode will not be downloaded if it is
    at least one of the qualities listed in the group, until a higher quality
    version is found.

    ```yaml
    ...
      qualities:
        - name: "WEB 480p"
          members:
            - "WEBDL-480p"
            - "WEBRip-480p"
    ```

    At least one quality must be specified.
    """

    upgrade_until: Optional[NonEmptyStr] = None
    """
    The maximum quality level to upgrade an episode to.
    For a quality group, specify the group name.

    Once this quality is reached Sonarr will no longer download episodes.

    This attribute is required if `upgrades_allowed` is set to `True`.
    """

    @validator("qualities")
    def validate_qualities(
        cls,
        value: List[Union[str, QualityGroup]],
    ) -> List[Union[str, QualityGroup]]:
        quality_name_map: Dict[str, Union[str, QualityGroup]] = {}
        for quality in value:
            for name in quality.members if isinstance(quality, QualityGroup) else [quality]:
                if name in quality_name_map:
                    error_message = f"duplicate entries of quality value '{name}' exist ("
                    other = quality_name_map[name]
                    if isinstance(quality, str) and isinstance(quality, str):
                        error_message += "both are non-grouped quality values"
                    else:
                        error_message += (
                            f"one as part of quality group '{quality.name}', "
                            if isinstance(quality, QualityGroup)
                            else "one as a non-grouped quality value, "
                        )
                        error_message += (
                            f"another as part of quality group '{other.name}'"
                            if isinstance(other, QualityGroup)
                            else "another as a non-grouped quality value"
                        )
                    error_message += ")"
                    raise ValueError(error_message)
                quality_name_map[name] = quality
        return value

    @validator("upgrade_until")
    def validate_upgrade_until(
        cls,
        value: Optional[str],
        values: Dict[str, Any],
    ) -> Optional[str]:
        try:
            upgrades_allowed: bool = values["upgrades_allowed"]
            qualities: Sequence[Union[str, QualityGroup]] = values["qualities"]
        except KeyError:
            return value
        # If `upgrades_allowed` is `False`, set `upgrade_until` to `None`
        # to make sure Buildarr ignores whatever it is currently set to
        # on the remote instance.
        if not upgrades_allowed:
            return None
        # Subsequent checks now assume that `upgrades_allowed` is `True`,
        # this parameter is required and defined to a valid value.
        if not value:
            raise ValueError("required if 'upgrades_allowed' is True")
        for quality in qualities:
            quality_name = quality.name if isinstance(quality, QualityGroup) else quality
            if value == quality_name:
                break
        else:
            raise ValueError("must be set to a value enabled in 'qualities'")
        return value

    @classmethod
    def _get_remote_map(
        cls,
        quality_definitions: Mapping[str, Mapping[str, Any]] = {},
        group_ids: Mapping[str, int] = {},
    ) -> List[RemoteMapEntry]:
        return [
            ("upgrades_allowed", "upgradeAllowed", {}),
            (
                "upgrade_until",
                "cutoff",
                {
                    "root_decoder": lambda vs: cls._upgrade_until_decoder(
                        items=vs["items"],
                        cutoff=vs["cutoff"],
                    ),
                    "root_encoder": lambda vs: cls._upgrade_until_encoder(
                        quality_definitions=quality_definitions,
                        group_ids=group_ids,
                        qualities=vs.qualities,
                        upgrade_until=vs.upgrade_until,
                    ),
                },
            ),
            (
                "qualities",
                "items",
                {
                    "decoder": lambda v: _decode_qualities(v),
                    "encoder": lambda v: _encode_qualities(quality_definitions, group_ids, v),
                },
            ),
        ]

    @classmethod
    def _upgrade_until_decoder(
        cls,
        items: Sequence[Mapping[str, Any]],
        cutoff: int,
    ) -> str:
        for quality_item in items:
            quality: Mapping[str, Any] = (
                quality_item  # Quality group
                if "id" in quality_item
                else quality_item["quality"]  # Quality definition (singular)
            )
            if quality["id"] == cutoff:
                return quality["name"]
        raise RuntimeError(
            "Inconsistent Sonarr instance state: "
            f"'cutoff' quality ID {cutoff} not found in 'items': {items}",
        )

    @classmethod
    def _upgrade_until_encoder(
        cls,
        quality_definitions: Mapping[str, Mapping[str, Any]],
        group_ids: Mapping[str, int],
        qualities: Sequence[Union[str, QualityGroup]],
        upgrade_until: Optional[str],
    ) -> int:
        if not upgrade_until:
            quality = qualities[0]
            return (
                group_ids[quality.name]
                if isinstance(quality, QualityGroup)
                else quality_definitions[quality]["id"]
            )
        return (
            group_ids[upgrade_until]
            if upgrade_until in group_ids
            else quality_definitions[upgrade_until]["id"]
        )

    @classmethod
    def _from_remote(cls, remote_attrs: Mapping[str, Any]) -> Self:
        return cls(**cls.get_local_attrs(cls._get_remote_map(), remote_attrs))

    def _create_remote(
        self,
        tree: str,
        secrets: SonarrSecrets,
        profile_name: str,
        quality_definitions: Mapping[str, Mapping[str, Any]],
    ) -> None:
        group_ids: Dict[str, int] = {
            quality_group.name: (1000 + i)
            for i, quality_group in enumerate(
                [q for q in self.qualities if isinstance(q, QualityGroup)],
                1,
            )
        }
        api_post(
            secrets,
            "/api/v3/qualityprofile",
            {
                "name": profile_name,
                **self.get_create_remote_attrs(
                    tree,
                    self._get_remote_map(quality_definitions, group_ids),
                ),
            },
        )

    def _update_remote(
        self,
        tree: str,
        secrets: SonarrSecrets,
        remote: Self,
        profile_id: int,
        profile_name: str,
        quality_definitions: Mapping[str, Mapping[str, Any]],
    ) -> bool:
        group_ids: Dict[str, int] = {
            quality_group.name: (1000 + i)
            for i, quality_group in enumerate(
                [q for q in self.qualities if isinstance(q, QualityGroup)],
                1,
            )
        }
        changed, remote_attrs = self.get_update_remote_attrs(
            tree,
            remote,
            self._get_remote_map(quality_definitions, group_ids),
            check_unmanaged=True,
            set_unchanged=True,
        )
        if changed:
            api_put(
                secrets,
                f"/api/v3/qualityprofile/{profile_id}",
                {"id": profile_id, "name": profile_name, **remote_attrs},
            )
            return True
        return False

    def _delete_remote(self, secrets: SonarrSecrets, profile_id: int) -> None:
        api_delete(secrets, f"/api/v3/qualityprofile/{profile_id}")

class SonarrQualityProfilesSettingsConfig(SonarrConfigBase):
    """
    Configuration parameters for controlling how Buildarr handles quality profiles.
    """

    delete_unmanaged: bool = False
    """
    Automatically delete quality profiles not defined in Buildarr.

    Out of the box Sonarr provides some pre-defined quality profiles.
    Take care when enabling this option, as those will also be deleted.
    """

    definitions: Dict[str, QualityProfile] = {}
    """
    Define quality profiles to configure on Sonarr here.

    If there are no quality profiles defined and `delete_unmanaged` is `False`,
    Buildarr will not modify existing quality profiles, but if `delete_unmanaged` is `True`,
    **Buildarr will delete all existing profiles. Be careful when using `delete_unmanaged`.**
    """

    @classmethod
    def from_remote(cls, secrets: SonarrSecrets) -> Self:
        return cls(
            definitions={
                profile["name"]: QualityProfile._from_remote(profile)
                for profile in api_get(secrets, "/api/v3/qualityprofile")
            },
        )

    def update_remote(
        self,
        tree: str,
        secrets: SonarrSecrets,
        remote: Self,
        check_unmanaged: bool = False,
    ) -> bool:
        changed = False
        profile_ids: Dict[str, int] = {
            profile_json["name"]: profile_json["id"]
            for profile_json in api_get(secrets, "/api/v3/qualityprofile")
        }
        quality_definitions: Dict[str, Dict[str, Any]] = {
            quality_json["title"]: quality_json["quality"]
            for quality_json in sorted(
                api_get(secrets, "/api/v3/qualitydefinition"),
                key=lambda q: q["weight"],
                reverse=True,
            )
        }
        for profile_name, profile in self.definitions.items():
            profile_tree = f"{tree}.definitions[{profile_name!r}]"
            if profile_name not in remote.definitions:
                profile._create_remote(
                    tree=profile_tree,
                    secrets=secrets,
                    profile_name=profile_name,
                    quality_definitions=quality_definitions,
                )
                changed = True
            elif profile._update_remote(
                tree=profile_tree,
                secrets=secrets,
                remote=remote.definitions[profile_name],
                profile_id=profile_ids[profile_name],
                profile_name=profile_name,
                quality_definitions=quality_definitions,
            ):
                changed = True
        return changed

    def delete_remote(self, tree: str, secrets: SonarrSecrets, remote: Self) -> bool:
        changed = False
        profile_ids: Dict[str, int] = {
            profile_json["name"]: profile_json["id"]
            for profile_json in api_get(secrets, "/api/v3/qualityprofile")
        }
        for profile_name, profile in remote.definitions.items():
            if profile_name not in self.definitions:
                profile_tree = f"{tree}.definitions[{profile_name!r}]"
                if self.delete_unmanaged:
                    logger.info("%s: (...) -> (deleted)", profile_tree)
                    profile._delete_remote(
                        secrets=secrets,
                        profile_id=profile_ids[profile_name],
                    )
                    changed = True
                else:
                    logger.debug("%s: (...) (unmanaged)", profile_tree)
        return changed

def _decode_qualities(value: Sequence[Mapping[str, Any]]) -> List[Union[str, QualityGroup]]:
    return [
        (
            QualityGroup(
                name=quality["name"],
                members=[member["quality"]["name"] for member in quality["items"]],
            )
            if quality["items"]
            else quality["quality"]["name"]
        )
        for quality in reversed(value)
        if quality["allowed"]
    ]

def _encode_qualities(
    quality_definitions: Mapping[str, Mapping[str, Any]],
    group_ids: Mapping[str, int],
    qualities: List[Union[str, QualityGroup]],
) -> List[Dict[str, Any]]:
    qualities_json: List[Dict[str, Any]] = []
    enabled_qualities: Set[str] = set()

    for quality in qualities:
        if isinstance(quality, QualityGroup):
            qualities_json.append(quality.encode(group_ids[quality.name], quality_definitions))
            for member in quality.members:
                enabled_qualities.add(member)
        else:
            qualities_json.append(_encode_quality_str(quality_definitions, quality, True))
            enabled_qualities.add(quality)

    for quality_name in quality_definitions.keys():
        if quality_name not in enabled_qualities:
            qualities_json.append(_encode_quality_str(quality_definitions, quality_name, False))

    return list(reversed(qualities_json))

def _encode_quality_str(
    quality_definitions: Mapping[str, Mapping[str, Any]],
    quality_name: str,
    allowed: bool,
) -> Dict[str, Any]:
    return {"quality": quality_definitions[quality_name], "items": [], "allowed": allowed}


import re
from django import forms

from mongoengine import *

from mumblr.entrytypes import EntryType, Comment, markup

class HtmlComment(Comment):
    """An HTML-based entry, which will be converted from the markup language
    specified in the settings.
    """
    rendered_content = StringField(required=True)

class TextEntry(EntryType):
    """An HTML-based entry, which will be converted from the markup language
    specified in the settings.
    """
    content = StringField(required=True)
    rendered_content = StringField(required=True)

    type = 'Text'

    def save(self):
        """Convert any markup to HTML before saving.
        """
        self.rendered_content = markup(self.content)
        super(TextEntry, self).save()

    class AdminForm(EntryType.AdminForm):
        content = forms.CharField(widget=forms.Textarea)

class LinkEntry(EntryType):
    """A link-based entry - the title is a link to the specified url and the
    content is the optional description.
    """
    link_url = StringField(required=True)
    description = StringField()

    type = 'Link'

    def rendered_content(self):
        if self.description:
            return markup(self.description, no_follow=False)
        return '
Link: %s
' % (self.link_url,
                                                     self.link_url)

    class AdminForm(EntryType.AdminForm):
        link_url = forms.URLField()
        description = forms.CharField(widget=forms.Textarea, required=False)

class ImageEntry(EntryType):
    """An image-based entry - displays the image at the given url along with
    the optional description.
    """
    image_url = StringField(required=True)
    description = StringField()

    type = 'Image'

    def rendered_content(self):
        url = self.image_url
        html = '' % url
        if self.description:
            html += markup(self.description)
        return html

    class AdminForm(EntryType.AdminForm):
        image_url = forms.URLField()
        description = forms.CharField(widget=forms.Textarea, required=False)

class VideoEntry(EntryType):
    """An video-based entry - will try to embed the video if it is of
    and known type e.g. YouTube
    """
    video_url = StringField(required=True)
    description = StringField()

    type = 'Video'

    embed_codes = {
        'vimeo': (
            '<object width="600" height="338"><param name="allowfullscr'
            'een" value="true" /><param name="allowscriptaccess" value='
            '"always" /><param name="movie" value="http://vimeo.com/moo'
            'galoop.swf?clip_id={{!ID}}&server=vimeo.com&show_t'
            'itle=0&show_byline=0&show_portrait=0&color=59a'
            '5d1&fullscreen=1" /><embed src="http://vimeo.com/mooga'
            'loop.swf?clip_id={{!ID}}&server=vimeo.com&show_tit'
            'le=0&show_byline=0&show_portrait=0&color=59a5d'
            '1&fullscreen=1" type="application/x-shockwave-flash" a'
            'llowfullscreen="true" allowscriptaccess="always" width="60'
            '0" height="338"></embed></object>'
        ),
        'youtube': (
            '<object width="600" height="338">'
            '<param name="movie" value="{{!ID}}"></param>'
            '<param name="allowFullScreen" value="true"></param>'
            '<param name="allowscriptaccess" value="always"></param>'
            '<embed src="http://www.youtube.com/v/{{!ID}}&fs=1&rel=&'
            'hd=10&showinfo=0&iv_load_policy=3" '
            'type="application/x-shockwave-flash" '
            'allowscriptaccess="always" allowfullscreen="true" '
            'width="600" height="338"></embed></object>'
        )
    }

    embed_patterns = (
        ('youtube', r'youtube\.com\/watch\?v=([A-Za-z0-9._%-]+)[&\w;=\+_\-]*'),
        ('vimeo', r'vimeo\.com\/(\d+)'),
    )

    def rendered_content(self):
        video_url = self.video_url
        for source, pattern in VideoEntry.embed_patterns:
            id = re.findall(pattern, video_url)
            if id:
                embed = VideoEntry.embed_codes[source]
                html = embed.replace('{{!ID}}', id[0])
                break
        else:
            html = 'Video: %s' % video_url

        if self.description:
            html += markup(self.description)
        return html

    class AdminForm(EntryType.AdminForm):
        video_url = forms.URLField()
        description = forms.CharField(widget=forms.Textarea, required=False)

EntryType.register(TextEntry)
EntryType.register(LinkEntry)
EntryType.register(ImageEntry)
EntryType.register(VideoEntry)


""" Imports from django, bouquet and category models. """
from django.contrib.admin import ModelAdmin
from django.contrib import admin
from .models import Bouquet, Category

@admin.register(Bouquet)
class BouquetAdmin(ModelAdmin):
    """
    Bouquet model view on django admin.
    """

    list_display = (
        'name',
        'sku',
        'category',
        'price',
    )

    ordering = ('category',)

@admin.register(Category)
class CategoryAdmin(ModelAdmin):
    """
    Category model view on django admin.
    """

    list_display = (
        'name',
    )


from lstore.table import Table, Record
from lstore.index import Index
from datetime import datetime
from lstore.config import *

class Query:
    """
    # Creates a Query object that can perform different queries on the specified table
    Queries that fail must return False
    Queries that succeed should return the result or True
    Any query that crashes (due to exceptions) should return False
    """
    def __init__(self, table):
        self.table = table
        self.index = Index(table)

    """
    # internal Method
    # Read a record with specified key
    # Returns True upon succesful deletion
    # Return False if record doesn't exist or is locked due to 2PL
    """

    def delete(self, key):
        rid = self.table.key_RID[key]
        del self.table.key_RID[key]
        del self.table.page_directory[rid]
        return True

    """
    # Insert a record with specified columns
    # Return True upon succesful insertion
    # Returns False if insert fails for whatever reason
    """
    def insert(self, *columns):
        key = columns[self.table.key_column]
        if key in self.table.key_RID.keys():
            return False;

        self.table.new_record.acquire()
        schema_encoding = '0' * self.table.num_columns
        indirection = MAX_INT
        rid = self.table.num_records
        time = datetime.now().strftime("%Y%m%d%H%M%S")
        base_id = rid
        meta_data = [rid, int(time), schema_encoding, indirection, base_id]
        columns = list(columns)
        meta_data.extend(columns)

        # self.table.lock_manager[key].acquire_wlock()
        self.table.base_write(meta_data)
        # self.table.lock_manager[key].release_wlock()
        self.table.new_record.release()
        return True

    """
    # Read matching record with specified search key
    # :param search_key: the value you want to search based on
    # :param search_key_index: the column index you want to search based on
    # :param projected_columns_index: what columns to return. array of 1 or 0 values.
    # Returns a list of Record objects upon success
    # Returns False if record locked by TPL
    # Assume that select will never be called on a key that doesn't exist
    """
    def find_column_bufferID(self, column):
        buffer_rid = []
        num_base = self.table.num_records - self.table.num_updates
        num_page_range = num_base % RECORD_PER_PAGERANGE + 1
        num_base_page = num_base % RECORD_PER_PAGE + 1
        for page_range in range(num_page_range):
            for base_page in range(num_base_page):
                buffer_id = (self.table.table_path, "base", str(column), str(page_range), str(base_page))
                buffer_rid.append(buffer_id)

        return buffer_rid

    def select(self, search_key, search_key_index, projected_columns_index):
        records = []
        column = []
        rids = []

        if search_key_index == self.table.key_column:
            if search_key in self.table.key_RID.keys():
                rids.append(self.table.key_RID[search_key])
        elif self.table.index.has_index(search_key_index):
            rids = self.table.index.locate(search_key_index, search_key)
        else:
            rids = self.table.find_value_rid(search_key_index, search_key)

        if len(rids) == 0:
                return []

        for rid in rids:
            result = self.table.find_record(rid)
            # print(result)
            column = result[DEFAULT_PAGE:DEFAULT_PAGE + self.table.num_columns + 1]

            # if record has update record
            if result[INDIRECTION] != MAX_INT:
            # use indirection of base record to find tail record
                rid_tail = result[INDIRECTION]

                result_tail = self.table.find_record(rid_tail)
                updated_column = result_tail[DEFAULT_PAGE:DEFAULT_PAGE + self.table.num_columns + 1]
                encoding = result[SCHEMA_ENCODING]
                encoding = self.find_changed_col(encoding)
                for i, value in enumerate(encoding):
                    if value == 1:
                        column[i] = updated_column[i]

            # take columns that is requested
            for i in range(self.table.num_columns):
                if projected_columns_index[i] == 0:
                    column[i] = None
            record = Record(rid, search_key, column)
            records.append(record)

        return records

    # helper function to find which columns have updated
    def find_changed_col(self, encoding):
        count = self.table.num_columns
        result = [0 for _ in range(count)]
        if encoding == 0:
            return result
        while encoding != 0:
            if encoding % 10 == 1:
                result[count - 1] = 1
            encoding = encoding // 10
            count -= 1
        return result

    """
    # Read matching record with specified search key
    # :param search_key: the value you want to search based on
    # :param search_key_index: the column index you want to search based on
    # :param projected_columns_index: what columns to return. array of 1 or 0 values.
    # :param relative_version: the relative version of the record you need to retreive.
    # Returns a list of Record objects upon success
    # Returns False if record locked by TPL
    # Assume that select will never be called on a key that doesn't exist
    """
    def select_version(self, search_key, search_key_index, projected_columns_index, relative_version):
        pass

    """
    # Update a record with specified key and columns
    # Returns True if update is succesful
    # Returns False if no records exist with given key or if the target record cannot be accessed due to 2PL locking
    """
    def update(self, primary_key, *columns):

        columns = list(columns)

        if primary_key not in self.table.key_RID.keys():
            return False
        if columns[self.table.key_column] in self.table.key_RID.keys():
            return False
        if columns[self.table.key_column] != None:
            return False

        self.table.update_record.acquire()
        tail_rid = self.table.num_records
        time = datetime.now().strftime("%Y%m%d%H%M%S")
        rid = self.table.key_RID[primary_key]
        base_id = rid
        result = self.table.find_record(rid)
        indirection = result[INDIRECTION]
        new_encoding = ''
        location_base = self.table.page_directory[rid]

        # first time update
        if indirection == MAX_INT:
            tail_indirect = rid
            for i, value in enumerate(columns):
                if value == None:
                    new_encoding += '0'
                    columns[i] = MAX_INT
                else:
                    new_encoding += '1'
        else:
            latest_tail = self.table.find_record(indirection)
            tail_indirect = latest_tail[RID]
            encoding = latest_tail[SCHEMA_ENCODING]
            encoding = self.find_changed_col(encoding)
            for i, value in enumerate(encoding):
                if columns[i] != None:
                    new_encoding += '1'
                else:
                    columns[i] = latest_tail[i + DEFAULT_PAGE]
                    if latest_tail[i + DEFAULT_PAGE] != MAX_INT:
                        new_encoding += '1'
                    else:
                        new_encoding += '0'

        # update base record
        self.table.update_value(INDIRECTION, location_base, tail_rid)
        self.table.update_value(SCHEMA_ENCODING, location_base, new_encoding)

        meta_data = [tail_rid, int(time), new_encoding, tail_indirect, base_id]
        meta_data.extend(columns)
        self.table.tail_write(meta_data)

        self.table.update_record.release()

        # for merge
        # location = self.table.page_directory[tail_rid]
        # self.table.merge_trigger(location)
        return True

    """
    :param start_range: int         # Start of the key range to aggregate
    :param end_range: int           # End of the key range to aggregate
    :param aggregate_columns: int  # Index of desired column to aggregate
    # this function is only called on the primary key.
    # Returns the summation of the given range upon success
    # Returns False if no record exists in the given range
    """
    def sum(self, start_range, end_range, aggregate_column_index):
        total_sum = 0
        column_index = aggregate_column_index + DEFAULT_PAGE
        for key in range(start_range, end_range + 1):
            if key in self.table.key_RID.keys():
                rid = self.table.key_RID[key]
                record = self.table.find_record(rid)
                # never been updated
                if record[INDIRECTION] == MAX_INT:
                    total_sum += record[column_index]
                else:
                # have been updated
                    tail_rid = record[INDIRECTION]
                    tail_record = self.table.find_record(tail_rid)
                    updated_column = tail_record[DEFAULT_PAGE:DEFAULT_PAGE + self.table.num_columns + 1]
                    encoding = tail_record[SCHEMA_ENCODING]
                    encoding = self.find_changed_col(encoding)
                    # print(updated_column)
                    # print(tail_record)
                    if encoding[aggregate_column_index] == 1:
                        total_sum += updated_column[aggregate_column_index]
                    else:
                        total_sum += record[column_index]

        return total_sum

    """
    :param start_range: int         # Start of the key range to aggregate
    :param end_range: int           # End of the key range to aggregate
    :param aggregate_columns: int  # Index of desired column to aggregate
    :param relative_version: the relative version of the record you need to retreive.
    # this function is only called on the primary key.
    # Returns the summation of the given range upon success
    # Returns False if no record exists in the given range
    """
    def sum_version(self, start_range, end_range, aggregate_column_index, relative_version):
        pass

    """
    incremenets one column of the record
    this implementation should work if your select and update queries already work
    :param key: the primary of key of the record to increment
    :param column: the column to increment
    # Returns True is increment is successful
    # Returns False if no record matches key or if target record is locked by 2PL.
    """
    def increment(self, key, column):
        r = self.select(key, self.table.key_column, [1] * self.table.num_columns)[0]
        if r is not False:
            updated_columns = [None] * self.table.num_columns
            updated_columns[column] = r[column] + 1
            u = self.update(key, *updated_columns)
            return u
        return False