import torch
import json
from torch.utils.data import Dataset
import numpy as np
from transformers import AutoTokenizer, RobertaTokenizerFast

from utils.utils import match_labels

import spacy
nlp = spacy.load("en_core_web_sm")

############################################################
#                                                          #
#                      DATASET CLASS                       #
#                                                          #
############################################################
class LegalNERTokenDataset(Dataset):

    def __init__(self, dataset_path, model_path, labels_list=None, split="train", use_roberta=False):
        self.data = json.load(open(dataset_path))
        self.split = split
        self.use_roberta = use_roberta
        if self.use_roberta:     ## Load the right tokenizer
            self.tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base")
        else:
            self.tokenizer = AutoTokenizer.from_pretrained(model_path)
        self.labels_list = sorted(labels_list + ["O"])[::-1]

        if self.labels_list is not None:
            self.labels_to_idx = dict(
                zip(sorted(self.labels_list)[::-1], range(len(self.labels_list)))
            )

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        item = self.data[idx]
        text = item["data"]["text"]

        ## Get the annotations
        annotations = [
            {
                "start": v["value"]["start"],
                "end": v["value"]["end"],
                "labels": v["value"]["labels"][0],
            }
            for v in item["annotations"][0]["result"]
        ]

        ## Tokenize the text
        if not self.use_roberta:
            inputs = self.tokenizer(
                text,
                return_tensors="pt",
                truncation=True,
                verbose=False
                )
        else:
            inputs = self.tokenizer(
                text,
                return_tensors="pt",
                truncation=True,
                verbose=False,
                padding='max_length'
            )

        ## Match the labels
        aligned_labels = match_labels(inputs, annotations)
        aligned_labels = [self.labels_to_idx[l] for l in aligned_labels]
        inputs["input_ids"] = inputs["input_ids"].squeeze(0).long()
        inputs["attention_mask"] = inputs["attention_mask"].squeeze(0).long()
        if not self.use_roberta:
            inputs["token_type_ids"] = inputs["token_type_ids"].squeeze(0).long()

        ## Get the labels
        if self.labels_list:
            labels = torch.tensor(aligned_labels).squeeze(-1).long()

            if labels.shape[0] < inputs["attention_mask"].shape[0]:
                pad_x = torch.zeros((inputs["input_ids"].shape[0],))
                pad_x[: labels.size(0)] = labels
                inputs["labels"] = aligned_labels
            else:
                inputs["labels"] = labels[: inputs["attention_mask"].shape[0]]

        return inputs


import argparse
from ast import arg
import re
import uuid
import inspect
from dataclasses import dataclass

from bin.contentctl_project.contentctl_core.application.factory.object_factory import ObjectFactory, ObjectFactoryInputDto
from bin.contentctl_project.contentctl_core.application.adapter.adapter import Adapter

@dataclass(frozen=True)
class ContentChangerInputDto:
    adapter : Adapter
    factory_input_dto : ObjectFactoryInputDto
    converter_func_name : str
    filter_key: str
    filter_value: str
    variables: list

class ContentChanger:

    def execute(self, input_dto: ContentChangerInputDto) -> None:
        objects = list()
        factory = ObjectFactory(objects)
        factory.execute(input_dto.factory_input_dto)

        filtered_objects = objects
        if input_dto.filter_key and input_dto.filter_value:
            filtered_objects = self.apply_key_value_filter(objects, input_dto.filter_key, input_dto.filter_value)

        converter_func = getattr(self, input_dto.converter_func_name)
        converter_func(filtered_objects, input_dto.variables)

        input_dto.adapter.writeObjectsInPlace(filtered_objects)

    @staticmethod
    def enumerate_content_changer_functions(exclude_functions: list[str] = ["enumerate_content_changer_functions", "execute", "example_converter_func"]) -> list[str]:
        members = inspect.getmembers(ContentChanger, predicate=inspect.isfunction)
        function_names = [function_object[0] for function_object in members if function_object[0] not in exclude_functions]
        return function_names

    def apply_key_value_filter(self, objects: list, key: str, value: str) -> None:
        new_list = list()
        for obj in objects:
            if str(obj[key]) == value:
                new_list.append(obj)

        return new_list

    def all(self, objects : list) -> None:
        for func_name in ContentChanger.enumerate_content_changer_functions():
            if func_name not in ["all", "change_test_file_format"]:
                print(f"calling {func_name}")
                func_object = getattr(self, func_name)
                func_object(objects)

    # Define Converter Functions here
    # def example_converter_func(self, objects : list) -> None:
    #     for obj in objects:
    #         obj['author'] = obj['author'].upper()

    # def add_unknown_context(self, objects : list) -> None:
    #     for obj in objects:
    #         if not 'context' in obj['tags']:
    #             obj['tags']['context'] = ['Unknown']

    # def add_default_message(self, objects : list) -> None:
    #     for obj in objects:
    #         if not 'message' in obj['tags']:
    #             obj['tags']['message'] = 'tbd'

    # def add_default_observable(self, objects : list) -> None:
    #     for obj in objects:
    #         if not 'observable' in obj['tags'] or ('observable' in obj['tags'] and len(obj['tags']['observable']) == 0):
    #             observables = []
    #             regexp_user = re.compile(r'user')
    #             if regexp_user.search(obj['search']):
    #                 observables.append({'name': 'user', 'type': 'User', 'role': ['Victim']})
    #             regexp_user = re.compile(r'dest')
    #             if regexp_user.search(obj['search']):
    #                 observables.append({'name': 'dest', 'type': 'Hostname', 'role': ['Victim']})
    #             if len(observables) == 0:
    #                 observables.append({'name': 'dest', 'type': 'Other', 'role': ['Other']})
    #             obj['tags']['observable'] = observables

    # def add_default_cis(self, objects : list) -> None:
    #     for obj in objects:
    #         if not 'cis20' in obj['tags']:
    #             obj['tags']['cis20'] = ['CIS 3', 'CIS 5', 'CIS 16']

    # def add_default_nist(self, objects : list) -> None:
    #     for obj in objects:
    #         if not 'nist' in obj['tags']:
    #             obj['tags']['nist'] = ['DE.CM']

    # def fix_broken_uuids(self, objects : list) -> None:
    #     for obj in objects:
    #         try:
    #             uuid.UUID(str(obj['id']))
    #         except:
    #             obj['id'] = str(uuid.uuid4())

    # def fix_wrong_kill_chain_phases(self, objects : list) -> None:
    #     valid_kill_chain_phases = [
    #         'Reconnaissance', 'Weaponization', 'Delivery',
    #         'Exploitation', 'Installation', 'Command And Control',
    #         'Actions on Objectives']
    #     for obj in objects:
    #         if 'kill_chain_phases' in obj['tags']:
    #             for value in obj['tags']['kill_chain_phases']:
    #                 if value not in valid_kill_chain_phases:
    #                     obj['tags']['kill_chain_phases'] = ['Exploitation']
    #                     break

    # def add_default_kill_chain_phases(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'kill_chain_phases' not in obj['tags']:
    #             obj['tags']['kill_chain_phases'] = ['Exploitation']
    #         if obj['tags']['kill_chain_phases'] == ['Privilege Escalation']:
    #             obj['tags']['kill_chain_phases'] = ['Exploitation']

    # def fix_wrong_calculated_risk_score(self, objects : list) -> None:
    #     for obj in objects:
    #         #Risk score must be an integer, so we round it to the nearest integer
    #         calculated_risk_score = round((obj['tags']['impact'] * obj['tags']['confidence'])/100)
    #         if calculated_risk_score != round(obj['tags']['risk_score']):
    #             obj['tags']['risk_score'] = calculated_risk_score

    # def add_asset_type_to_endpoint_detections(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'asset_type' not in obj['tags']:
    #             if '/endpoint/' in obj['file_path']:
    #                 obj['tags']['asset_type'] = 'Endpoint'

    # def fix_observables(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'observable' in obj['tags']:
    #             for observable in obj['tags']['observable']:
    #                 if observable['type'] == 'Parent Process':
    #                     observable['type'] = 'Process'
    #                 if observable['type'] == 'user':
    #                     observable['type'] = 'User'
    #                 if observable['type'] == 'process name':
    #                     observable['type'] = 'Process'

    # def fix_context(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'context' in obj['tags']:
    #             new_context = []
    #             for context in obj['tags']['context']:
    #                 if context == 'Stage:Exploitation':
    #                     context = 'Stage:Execution'
    #                 new_context.append(context)

    #             obj['tags']['context'] = list(dict.fromkeys(new_context))

    # def add_default_values_deprecated(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'context' not in obj['tags']:
    #             obj['tags']['context'] = ['Unknown']
    #         if 'message' not in obj['tags']:
    #             obj['tags']['message'] = 'tbd'
    #         if 'observable' not in obj['tags']:
    #             obj['tags']['observable'] = [{'name': 'field', 'type': 'Unknown', 'role': ['Unknown']}]

    # def fix_story(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'type' not in obj:
    #             print(obj['name'])
    #             if isinstance(obj['tags']['analytic_story'], list):
    #                 obj['tags']['analytic_story'] = obj['tags']['analytic_story'][0]

    # def remove_SAAWS(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'Splunk Security Analytics for AWS' in obj['tags']['product']:
    #             obj['tags']['product'].remove('Splunk Security Analytics for AWS')

    # def remove_testing_passed(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'automated_detection_testing' in obj['tags']:
    #             obj['tags'].pop('automated_detection_testing')

    # def change_test_file_format(self, objects : list) -> None:
    #     for obj in objects:
    #         obj['name'] = obj['name'] + ' Unit Test'

    # def fix_kill_chain(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'kill_chain_phases' in obj['tags']:
    #             if obj['tags']['kill_chain_phases'] == 'Exploitation':
    #                 obj['tags']['kill_chain_phases'] = ['Exploitation']

    # def add_default_confidence_impact_risk_score(self, objects : list) -> None:
    #     for obj in objects:
    #         updated = True
    #         if not 'confidence' in obj['tags']:
    #             updated = True
    #             obj['tags']['confidence'] = 50
    #         if not 'impact' in obj['tags']:
    #             updated = True
    #             obj['tags']['impact'] = 50
    #         if not 'risk_score' in obj['tags'] or updated == True:
    #             #Recalculate the risk score if we have added/updated
    #             #the confidence or impact fields OR the risk_score
    #             #was missing in the first place
    #             self.fix_wrong_calculated_risk_score([obj])

    # def fix_cc(self, objects : list) -> None:
    #     for obj in objects:
    #         if 'Command & Control' in obj['tags']['analytic_story']:
    #             obj['tags']['analytic_story'].remove('Command & Control')
    #             obj['tags']['analytic_story'].append('Command And Control')

    def update_description(self, objects : list, input_vars: list) -> None:
        for obj in objects:
            obj["description"] = input_vars[0]


# -*- coding: utf-8 -*-
"""

@author: Rubén
"""
import logging
import telegram

from telegram.error import NetworkError, Unauthorized
from time import sleep
from jikanpy import APIException, Jikan

import funciones as f
import constantes as c
import jikanpy

update_id = None
comando_ocupado = False
url = "https://www.etnassoft.com/api/v1/get/"

def main():
    global update_id
    bot = telegram.Bot(c.TELEGRAM_API)
    try:
        update_id = bot.get_updates()[0].update_id
    except IndexError:
        update_id = None

    logging.basicConfig(
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    while True:
        try:
            echo(bot)
        except NetworkError:
            sleep(1)
        except Unauthorized:
            update_id += 1

def echo(bot):
    global update_id
    global comando_ocupado
    conexion = f.sql_conexion()
    for update in bot.get_updates(offset=update_id, timeout=10):
        update_id = update.update_id + 1
        chat_id = str(update.message.chat.id)
        nick = str(update.message.chat.username)
        nombre = str(update.message.chat.first_name)
        apellidos = str(update.message.chat.last_name)
        fecha_actual = str(f.obtener_fechaactual())

        if(f.buscar_usuario(conexion, chat_id) == 0):
            f.insertar_usuario(conexion, chat_id, nick, nombre, apellidos)
            f.insertar_acceso(conexion, chat_id, fecha_actual)
        else:
            if(f.buscar_acceso(conexion, chat_id, fecha_actual) == 0):
                f.insertar_acceso(conexion, chat_id, fecha_actual)
            else:
                f.actualizar_acceso(conexion, chat_id, fecha_actual)

        if update.message:
            opcion = update.message.text
            if(comando_ocupado):
                if(comando_ocupado == 2):
                    f.enviar_mensaje(update, c.PROCESANDO_MENSAJE)
                    jikan = Jikan()
                    animes = jikan.search('anime', str(opcion))['results']
                    if(animes):
                        for i in range(10):
                            anime = animes[i]
                            f.operaciones_base_de_datos(
                                conexion, anime, chat_id)
                            f.enviar_imagen(
                                str(anime['image_url']), bot, update)
                            f.enviar_mensaje(update, "Id: {}\nTítulo: {}\nEpisodios: {}\nFecha de inicio: {}\nSinópsis: {}".format(
                                anime['mal_id'], anime['title'], anime['episodes'], anime['start_date'], anime['synopsis']))
                    else:
                        f.enviar_mensaje(update, c.INFORMACION_NO_ENCONTRADA)

                elif(comando_ocupado == 3):
                    f.enviar_mensaje(update, c.PROCESANDO_MENSAJE)
                    jikan = Jikan()
                    if(opcion.isdigit()):
                        try:
                            animes = jikan.genre(
                                type='anime', genre_id=int(opcion))['anime']
                            if(animes):
                                for i in range(5, 15, 1):
                                    anime = animes[i]
                                    f.operaciones_base_de_datos(
                                        conexion, anime, chat_id)
                                    f.enviar_imagen(
                                        str(anime['image_url']), bot, update)
                                    f.enviar_mensaje(update, "Id: {}\nTítulo: {}\nEpisodios: {}\nSinópsis: {}".format(
                                        anime['mal_id'], anime['title'], anime['episodes'], anime['synopsis']))
                            else:
                                f.enviar_mensaje(
                                    update, c.INFORMACION_NO_ENCONTRADA)
                        except jikanpy.exceptions.APIException:
                            f.enviar_mensaje(update, c.ID_NO_VALIDO)
                    else:
                        f.enviar_mensaje(update, c.ID_NO_VALIDO)
                elif(comando_ocupado == 4):
                    f.enviar_mensaje(update, c.PROCESANDO_MENSAJE)
                    jikan = Jikan()
                    if(opcion.isdigit()):
                        try:
                            anime = jikan.anime(int(opcion))
                            if(anime):
                                f.operaciones_base_de_datos(
                                    conexion, anime, chat_id)
                                f.enviar_imagen(
                                    str(anime['image_url']), bot, update)
                                f.enviar_mensaje(update, c.INFO_ANIME.format(anime['mal_id'],
                                                                             anime['title'],
                                                                             anime['title_japanese'],
                                                                             anime['synopsis'],
                                                                             anime['episodes'],
                                                                             anime['duration'],
                                                                             anime['rating'],
                                                                             anime['score'],
                                                                             anime['opening_themes'],
                                                                             anime['ending_themes'],
                                                                             anime['trailer_url']))
                            else:
                                f.enviar_mensaje(
                                    update, c.INFORMACION_NO_ENCONTRADA)
                        except jikanpy.Exception.APIException:
                            f.enviar_mensaje(update, c.ID_NO_VALIDO)
                    else:
                        f.enviar_mensaje(update, c.ID_NO_VALIDO)
                comando_ocupado = False
            else:
                if(opcion == c.START):
                    f.enviar_mensaje(update, c.BIENVENIDA)
                elif(opcion == c.RECOMENDAR_ANIMES):
                    comando_ocupado = 1
                    f.enviar_mensaje(update, c.PROCESANDO_MENSAJE)
                    jikan = Jikan()
                    animes = jikan.top(type='anime', page=1)['top']
                    if(animes):
                        for i in range(10):
                            anime = animes[i]
                            f.operaciones_base_de_datos(
                                conexion, anime, chat_id)
                            f.enviar_imagen(
                                str(anime['image_url']), bot, update)
                            f.enviar_mensaje(update, "Id: {}\nTitulo: {}\nEpisodios: {}\nFecha de inicio: {}".format(
                                anime['mal_id'], anime['title'], anime['episodes'], anime['start_date']))
                    else:
                        f.enviar_mensaje(update, c.INFORMACION_NO_ENCONTRADA)
                    comando_ocupado = False
                elif(opcion == c.BUSCAR_ANIME):
                    comando_ocupado = 2
                    f.enviar_mensaje(update, c.INGRESAR_TITULO_ANIME)
                elif(opcion == c.BUSCAR_ANIME_POR_GENERO):
                    comando_ocupado = 3
                    f.enviar_mensaje(update, c.LISTA_GENEROS)
                elif(opcion == c.MOSTRAR_ANIME):
                    comando_ocupado = 4
                    f.enviar_mensaje(update, c.INGRESAR_ID_ANIME)
                else:
                    f.enviar_mensaje(update, c.MENSAJE_NO_ENTENDIDO)

if __name__ == '__main__':
    main()


import argparse

import hermes_client as hermes

# Define command line arguments.
_parser = argparse.ArgumentParser("Renames a group of metrics.")
_parser.add_argument(
    "-g", "--group",
    help="A metrics group identifier",
    dest="group",
    type=str
    )
_parser.add_argument(
    "-n", "--new-name",
    help="New metrics group identifier",
    dest="new_name",
    type=str
    )

def _main(args):
    """Main entry point.

    """
    hermes.metrics.rename(args.group, args.new_name)

# Main entry point.
if __name__ == '__main__':
    _main(_parser.parse_args())


# CreateHTMLFromCSV.py

# Creates a Google Maps HTML document containing
# markers contained in the specified CSV file

# Fall 2018
# J.Fay for ENV859

import sys, os
import pandas as pd
import folium
from folium.plugins import MarkerCluster

#Set the working directory
os.chdir(os.path.dirname(sys.path[-1])+'\\1_CSVtoHTMLwithPython')

#Filename to save the output to:
outputHTML = 'Violations_folium.html'

#Read the data into a Pandas dataFrame
df = pd.read_csv('Pennsylvania Oil and Gas Violations.csv')

#Remove rows missing lat/long data
df.dropna(subset=['latitude','longitude'],axis='rows',how='any',inplace=True)

#Get the mean latitude and longitude, to set as the center of the map
meanLat = df['latitude'].mean()
meanLng = df['longitude'].mean()

#Create the folium map object
wellMap = folium.Map(location=(meanLat,meanLng),zoom_start=8)

#Create a marker cluster object which enables a cleaner display of markers
marker_cluster = MarkerCluster()
marker_cluster.add_to(wellMap)

#Add as Clustered Markers
for row in df.iterrows():
    #Print dots to show progress
    print ('.',end='')

    #Get the data from the row; it's the 2nd object (1st is the row's index)
    data = row[1]

    #Get the row's data that we want
    lat = data['latitude']
    lon = data['longitude']
    name = data['Violation Code']

    #Create a marker
    m = folium.Marker((lat,lon),popup=str(name))

    #Add it to the marker cluster object
    m.add_to(marker_cluster)

# Save the map
wellMap.save(outputHTML)


import os
import pandas as pd
import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer

nltk.download('vader_lexicon')

# Initialize the sentiment intensity analyzer
sid = SentimentIntensityAnalyzer()

# Define function for sentiment analysis
def analyze_sentiment(text):
    text = str(text)  # Convert text to string in case it's not
    analysis = sid.polarity_scores(text)
    return analysis['compound']  # return the compound score

# Define your directory
directory = 'C:/Users/Marcos/Desktop/Uni/PI/DataMining/2014'

# Iterate over each file in the directory
for filename in os.listdir(directory):
    # Only process .xlsx files
    if filename.endswith('.xlsx'):
        # Load the data
        df = pd.read_excel(os.path.join(directory, filename), engine='openpyxl')

        # Apply sentiment analysis to the 'Review' column
        df['Sentiment'] = df['Review'].apply(analyze_sentiment)

        # Calculate averages
        sentiment_average = df['Sentiment'].mean()
        rating_average = df['Rating'].mean()

        # Normalize sentiment average to a 0-1 scale and then scale up to a 0-10 scale
        sentiment_average = (sentiment_average + 1) * 5

        # Round the averages to one decimal place
        sentiment_average = round(sentiment_average, 1)
        rating_average = round(rating_average, 1)

        # Create strings that represent the averages as a fraction of 10
        sentiment_average = f'{sentiment_average}/10'
        rating_average = f'{rating_average}/10'

        # Create a new DataFrame with the averages
        averages_df = pd.DataFrame({'Sentiment': [sentiment_average], 'Rating': [rating_average]}, index=['Average'])

        # Append averages to the bottom of the original DataFrame
        df_with_averages = df.append(averages_df, ignore_index=False)

        # Create a new filename for the output
        new_filename = f'{os.path.splitext(filename)[0]}_sentiment.xlsx'

        # Save the results without including the DataFrame's index
        df_with_averages.to_excel(os.path.join(directory, new_filename), index=False)


import torch
import pytest

from onmt.modules.sparse_activations import (
    Sparsemax,
    Tsallis15,
    SparsemaxTopK,
    Tsallis15TopK,
)

from onmt.modules.root_finding import (
    sparsemax_bisect,
    tsallis_bisect,
)

funcs = [
    Sparsemax(dim=1),
    Tsallis15(dim=1),
    SparsemaxTopK(dim=1),
    Tsallis15TopK(dim=1),
    sparsemax_bisect,
    tsallis_bisect,
]

@pytest.mark.parametrize('func', funcs)
@pytest.mark.parametrize('dtype', (torch.float32, torch.float64))
def test_mask(func, dtype):
    torch.manual_seed(42)
    x = torch.randn(2, 6, dtype=dtype)
    x[:, 3:] = -float('inf')
    x0 = x[:, :3]

    y = func(x)
    y0 = func(x0)

    y[:, :3] -= y0

    assert torch.allclose(y, torch.zeros_like(y))

@pytest.mark.parametrize('alpha', (1.25, 1.5, 1.75, 2.25))
def test_mask_alphas(alpha):
    torch.manual_seed(42)
    x = torch.randn(2, 6)
    x[:, 3:] = -float('inf')
    x0 = x[:, :3]

    y = tsallis_bisect(x, alpha)
    y0 = tsallis_bisect(x0, alpha)

    y[:, :3] -= y0

    assert torch.allclose(y, torch.zeros_like(y))


""" The cluster_plot series of functions that plots all of the clustered stars and
    information. """
import scipy.stats as spy
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.colors as pltc
from mpl_toolkits import mplot3d

def iqr_calc_angle(selected_cluster, distcen, column):
    """ Calculated interquartile range of a cluster group and is scaled to distance
     from angle. """
    return spy.iqr(selected_cluster[column])*3.14159/180*distcen

def color_setup(clustered_data, membership_plot, unique_group, palette):
    """ If plotting a membership probability color map is wanted, the input dataframe must be
    separated into a different dataframe for each cluster.  Doing this with a dictionary
    is fast and efficient.  A second dictionary is made to set the color map for each cluster
    number.  If plotting membership probability is not needed, we instead set a unique color
    to each cluster and set non-clustered field stars to black. """
    import pandas as pd

    if membership_plot == 1 or membership_plot == 2:
        clustered_dict = {elem: pd.DataFrame for elem in unique_group}
        for key in clustered_dict.keys():
            clustered_dict[key] = clustered_data[:][clustered_data.clusternum == key]
        cmapdict = {-1:"Greys", 0:"Blues", 1:"Oranges", 2:"Greens", 3:"Reds",\
         4:"Purples", 5:"copper", 6:"cool", 7:"bone", 8:"autumn", 9:"GnBu",\
          10:"PuBu", 11:"Reds", 12:"Oranges", 13:"PuBu", 14:"YlGn", 15:"Greys"}
        return clustered_dict, cmapdict
    return [palette[x] if x >= 0 else (0.0, 0.0, 0.0) for x in clustered_data["clusternum"]]

def distance_hist(selected_cluster, distcen, distiqr, step, palette):
    """ A subplot illustrating the cluster distance histogram.  Additionally, the median
    cluster distance and distance IQR are plotted. IQR is preferred for potentially
    non-normally distributed data."""
    plt.subplot(223)
    plt.xlabel("Distance (pc)", fontsize=15)
    plt.ylabel("Number", fontsize=15)
    plt.xlim(distcen-250, distcen+250)
    bins = np.linspace(distcen-300, distcen+300, 30)
    plt.hist(selected_cluster["distance"], bins, alpha=0.5, color=palette[step])
    plt.gcf().text(0.09, 0.43, "Distance = "+str(int(distcen))+" pc", fontsize=12)
    plt.gcf().text(0.32, 0.43, "Distance IQR = "+str(int(distiqr))+" pc", fontsize=12)

def ra_dec_plot_setup(clustered_data, distcen, selected_cluster):
    """ A subplot illustrating the cluster scaled RA and DEC is setup.  The median cluster
    RA and DEC and their IQR in pc space are plotted.  """
    plt.subplot(222)
    plt.ylabel("DEC", fontsize=15)
    plt.xlabel("RA Scaled", fontsize=15)
    ramin = min(clustered_data["ratransform"])
    ramax = max(clustered_data["ratransform"])
    decmin = min(clustered_data["dec"])
    decmax = max(clustered_data["dec"])
    raiqr = iqr_calc_angle(selected_cluster, distcen, "ratransform")
    ramedian = np.median(selected_cluster["rawrapped"])

    # The texts are placed in data space, which pushes them outside the plot and automatically
    # rescales the plot positions to make space for the text.
    plt.text(ramin, decmax+(decmax-decmin)*0.16, "RA = "+str(round(ramedian, 3)), fontsize=12)
    plt.text(ramin+(ramax-ramin)*0.6, decmax+(decmax-decmin)*0.16, "RA IQR =\
     "+str(round(raiqr, 2))+" pc", fontsize=12)
    decmedian = np.median(selected_cluster["dec"])
    deciqr = iqr_calc_angle(selected_cluster, distcen, "dec")
    plt.text(ramin, decmax+(decmax-decmin)*0.08, "DEC = "+str(round(decmedian, 3)), fontsize=12)
    plt.text(ramin+(ramax-ramin)*0.6, decmax+(decmax-decmin)*0.08, "DEC IQR =\
     "+str(round(deciqr, 2))+" pc", fontsize=12)

def pm_plot_setup(selected_cluster, pmraiqr, pmdeciqr):
    """ A subplot illustrating the cluster proper motions is setup.  Additionally, the
    median cluster RA PM and DEC PM (in mas/yr) and their IQR (in km/s) space are plotted. """
    plt.subplot(221)
    plt.ylabel("DEC PM (mas/yr)", fontsize=15)
    plt.xlabel("RA PM (mas/yr)", fontsize=15)
    pmramedian = np.median(selected_cluster["pmra"])
    pmdecmedian = np.median(selected_cluster["pmdec"])
    plt.gcf().text(0.08, 0.978, "RA PM = "+str(round(pmramedian, 2))+" mas/yr", fontsize=12)
    plt.gcf().text(0.08, 0.948, "DEC PM = "+str(round(pmdecmedian, 2))+" mas/yr", fontsize=12)
    plt.gcf().text(0.28, 0.978, "RA PM IQR = "+str(round(pmraiqr, 2))+" km/s", fontsize=12)
    plt.gcf().text(0.28, 0.948, "DEC PM IQR = "+str(round(pmdeciqr, 2))+" km/s", fontsize=12)
    # We do not want to plot the entire PM space, and zoom to around +/- 10 mas/yr on both
    # axis centered on the cluster median.
    plt.ylim(pmdecmedian-10, pmdecmedian+10)
    plt.xlim(pmramedian-10, pmramedian+10)

def cmd_plot_setup(clustered_data, selected_cluster):
    """ A subplot illustrating the cluster CMD is setup."""
    plt.subplot(224)
    plt.xlabel("BP-RP", fontsize=15)
    plt.ylabel("M$_{\mathrm{G}}$", fontsize=15)
    ymin = min(clustered_data["M_G"])
    ymax = max(clustered_data["M_G"])
    plt.ylim(ymax, ymin-1)
    plt.xlim(-0.6, 3.5)
    return len(selected_cluster), ymin

def plot_map(unique_group, clustered_dict, cmapdict, normalize, step, xvar_col, yvar_col):
    """ For when a membership probability color map is desired (i.e., membership_plot == 1
     or 2).  This function plots each cluster separately with its own separate color map,
     while it plots non-clustered data as simply black."""
    for group in unique_group:
        if group == -1:
            plt.scatter(clustered_dict[group][xvar_col], clustered_dict[group][yvar_col], s=0.002,\
             alpha=1, color="black")
        elif group == step:
            plt.scatter(clustered_dict[group][xvar_col], clustered_dict[group][yvar_col], s=6,\
             alpha=1, c=clustered_dict[group]["clusterprob"], cmap=cmapdict[group], norm=normalize)
            cbar = plt.colorbar()
            cbar.set_label('Membership Probability')
        else:
            plt.scatter(clustered_dict[group][xvar_col], clustered_dict[group][yvar_col], s=0.02,\
             alpha=1, c=clustered_dict[group]["clusterprob"], cmap=cmapdict[group], norm=normalize)

def plot_map3D(real_clusters, clustered_dict, cmapdict, normalize, step, selected_cluster, distcen):
    """ For when a membership probability color map is desired (i.e., membership_plot == 1 or 2).
    This function plots each cluster separately with its own separate color map, while
    it plots non-clustered data as simply black."""
    ax=plt.axes(projection='3d')
    ax.set_xlabel("RA (pc)", fontsize=12)
    ax.set_ylabel("DEC (pc)", fontsize=12)
    ax.set_zlabel("Distance (pc)", fontsize=12)
    selected_cluster["ra3D"]=selected_cluster["ratransform"]*3.14159/180*distcen
    selected_cluster["dec3D"]=selected_cluster["dectransform"]*3.14159/180*distcen
    dimension = (max(selected_cluster["ra3D"]) - min(selected_cluster["ra3D"]) +\
     max(selected_cluster["dec3D"]) - min(selected_cluster["dec3D"]))/4
    ax.set_xlim(min(selected_cluster["ra3D"]),max(selected_cluster["ra3D"]))
    ax.set_ylim(min(selected_cluster["dec3D"]),max(selected_cluster["dec3D"]))
    ax.set_zlim(distcen-dimension,distcen+dimension)
    for group in real_clusters:
        if group == step:
            ax.scatter3D(clustered_dict[group]["ratransform"]*3.14159/180*clustered_dict[group]["distance"],\
             clustered_dict[group]["dectransform"]*3.14159/180*clustered_dict[group]["distance"],\
             clustered_dict[group]["distance"], s=6, alpha=1, c=clustered_dict[group]["clusterprob"],\
             cmap=cmapdict[group], norm=normalize)
        elif group != -1:
            ax.scatter3D(clustered_dict[group]["ratransform"]*3.14159/180*clustered_dict[group]["distance"],\
             clustered_dict[group]["dectransform"]*3.14159/180*clustered_dict[group]["distance"],\
             clustered_dict[group]["distance"], s=3, alpha=1, c=clustered_dict[group]["clusterprob"],\
             cmap=cmapdict[group], norm=normalize)

def cluster_plot(clustered_data, membership_plot):
    """ This main function sets up the colors, figures, and plots the data based on the
    requested membership or not color scheme. """

# We count the number of found clusters, set up the color palette, and set tick label
# font size to 12 for clarity.
    unique_group = clustered_data.clusternum.unique()
    palette = sns.color_palette('bright', max(unique_group)+1)
    dec_median = np.median(clustered_data["dec"])
    clustered_data["dectransform"] = clustered_data["dec"] - dec_median
    if membership_plot == 1 or membership_plot == 2:
        clustered_dict, cmapdict = color_setup(clustered_data, membership_plot,\
         unique_group, palette)
    else:
        colors = color_setup(clustered_data, membership_plot, unique_group, palette)
    normalize = pltc.Normalize(vmin=0, vmax=1)
    plt.rcParams['xtick.labelsize'] = 12
    plt.rcParams['ytick.labelsize'] = 12
    real_clusters = []

# This loop iterates through all clustered groups found with hdbscan and plots the
# requested clusters that pass the checks.
    for step in range(max(unique_group)+1):

        selected_cluster = clustered_data.loc[clustered_data["clusternum"] == step].copy()
# The median stellar distance is found (in parsecs), which is used to transform proper
# motions and RA and DEC into spatial scales.  Because these distributions are not
# always Gaussian, the interquartile ranges are used to analyze parameter variations
# throughout a cluster.
        distcen = np.median(selected_cluster["distance"])
        distiqr = spy.iqr(selected_cluster["distance"])
        pmraiqr = iqr_calc_angle(selected_cluster, distcen, "pmra")*0.271795
        pmdeciqr = iqr_calc_angle(selected_cluster, distcen, "pmdec")*0.271795
        full_count = len(selected_cluster)
        faint_count = len(selected_cluster.loc[selected_cluster["M_G"] > 10])
        if (pmraiqr < 3 and pmdeciqr < 3 and distiqr < 300 and faint_count/full_count < 0.8):
            real_clusters.append(step)

# Hdbscan will typically create a cluster from the overdensity in the field disk stars,
# producing apparent clusters but with large PM variation.  Therefore, we require that
# the transformed PM variation be less than 3 km/s in each axis to be displayed as a
# valid cluster.  If a legitimate cluster is identified but retains a lot of
# noise/non-members, a 3 km/s cut may remove it from display.  In these cases, you may
# increase the cut to 4 or 5 km/s, etc., to first analyze its output, but in the end I
# recommend adjusting the hdbscan min_samples or the parameter scalings.
    for step in real_clusters:

        selected_cluster = clustered_data.loc[clustered_data["clusternum"] == step].copy()
# The median stellar distance is found (in parsecs), which is used to transform proper
# motions and RA and DEC into spatial scales.  Because these distributions are not
# always Gaussian, the interquartile ranges are used to analyze parameter variations
# throughout a cluster.
        distcen = np.median(selected_cluster["distance"])
        distiqr = spy.iqr(selected_cluster["distance"])
        pmraiqr = iqr_calc_angle(selected_cluster, distcen, "pmra")*0.271795
        pmdeciqr = iqr_calc_angle(selected_cluster, distcen, "pmdec")*0.271795
# To set up the plots, we set a separate Figure for each cluster.  For when membership
# probability is not shown, we set all clustered data to a large size and nonclustered
# data to small data size.
        plt.figure(step, figsize=(14, 10))
        if membership_plot != 1 or membership_plot != 2:
            sizes = [4 if x == step else 0.003 for x in clustered_data["clusternum"]]

        distance_hist(selected_cluster, distcen, distiqr, step, palette)
        ra_dec_plot_setup(clustered_data, distcen, selected_cluster)
        if membership_plot == 1 or membership_plot == 2:
            plot_map(unique_group, clustered_dict, cmapdict, normalize, step,\
             "ratransform", "dec")
        else:
            plt.scatter(clustered_data["ratransform"], clustered_data["dec"],\
             s=sizes, alpha=1, color=colors)
        pm_plot_setup(selected_cluster, pmraiqr, pmdeciqr)
        if membership_plot == 1 or membership_plot == 2:
            plot_map(unique_group, clustered_dict, cmapdict, normalize, step, "pmra", "pmdec")
        else:
            plt.scatter(clustered_data["pmra"], clustered_data["pmdec"], s=sizes,\
             alpha=1, color=colors)
        number, ymin = cmd_plot_setup(clustered_data, selected_cluster)
        if membership_plot == 1 or membership_plot == 2:
            xlabelpos = 1.57
            plot_map(unique_group, clustered_dict, cmapdict, normalize, step, "bp_rp", "M_G")
        else:
            xlabelpos = 1.95
            plt.scatter(clustered_data["bp_rp"], clustered_data["M_G"], s=sizes,\
             alpha=1, color=colors)
        plt.text(xlabelpos, ymin+0.5, "Cluster Number "+str(step), fontsize=12)
        plt.text(xlabelpos, ymin+1.68, "Cluster Count = "+str(number), fontsize=12)
        plt.tight_layout()
        if membership_plot == 2:
            plt.figure(step+1+max(unique_group)+1, figsize=(10, 8))
            plot_map3D(real_clusters, clustered_dict, cmapdict, normalize, step,\
             selected_cluster, distcen)

    plt.show()