import os
import sys

import numpy as np
import random
import argparse
from random import sample
import time

import utils
import config

import tensorflow as tf
from keras.utils import multi_gpu_model
from keras.models import Sequential
from keras.layers import Dense, Dropout, Permute, Conv1D
from keras.layers import Embedding
from keras.layers import LSTM
from keras.layers import CuDNNLSTM
from keras.layers import Activation
from keras import optimizers
from keras.models import load_model
from keras import callbacks
from keras import metrics

if __name__ == "__main__":
    args = utils.parse_arguments()

    # print(args)
    # print(args.reload)
    # print('type(args.reload): {}'.format(type(args.reload)))
    # print(args.no_train)
    # print('type(args.no_train): {}'.format(type(args.no_train)))

    if args.reload is not None:
        config.RELOAD_MODEL = args.reload

    if args.no_train is not None:
        config.TRAIN_MODEL = args.no_train

    if args.samples is not None:
        config.NUM_SAMPLES = args.samples

    if args.epochs is not None:
        config.EPOCHS = args.epochs

    if args.iterations is not None:
        config.ITERATIONS = args.iterations

    config.K_FOLDS = args.k_folds

    if args.batch_size is not None:
        config.MAX_BATCH_SIZE = args.batch_size

    if args.dataset_path is not None:
        config.DATASET_PATH = args.dataset_path

    if args.data_width is not None:
        config.DATA_WIDTH = args.data_width

    if args.max_data_width is not None:
        config.MAX_DATA_WIDTH = args.max_data_width

    if args.min_training_data_width is not None:
        config.MIN_TRAINING_DATA_WIDTH = args.min_training_data_width

    if args.max_training_data_width is not None:
        config.MAX_TRAINING_DATA_WIDTH = args.max_training_data_width

    if (((config.MAX_TRAINING_DATA_WIDTH is None) or (config.MIN_TRAINING_DATA_WIDTH is None)) and
            ((config.MAX_TRAINING_DATA_WIDTH is not None) or (config.MIN_TRAINING_DATA_WIDTH is not None))):

        raise("--max-training-data-width or --min-training-data-width is set but the other is not.  Provide a value for both.")

    print('config.MIN_TRAINING_DATA_WIDTH: {}'.format(
        config.MIN_TRAINING_DATA_WIDTH))
    print('config.MAX_TRAINING_DATA_WIDTH: {}'.format(
        config.MAX_TRAINING_DATA_WIDTH))

    if config.MIN_TRAINING_DATA_WIDTH is None:
        config.MIN_TRAINING_DATA_WIDTH = config.DATA_WIDTH
        config.MAX_TRAINING_DATA_WIDTH = config.DATA_WIDTH
    else:
        config.DATA_WIDTH = config.MAX_TRAINING_DATA_WIDTH

    config.RANDOMIZE_INPUT_DATA = args.randomize_input_data
    if config.RANDOMIZE_INPUT_DATA is True:
        config.CONTIGUOUS_INPUT_DATA = False
    else:
        config.CONTIGUOUS_INPUT_DATA = args.continguous_input_data

    print('Starting with the following configuration:')
    print('------------------------------------------')
    print('    Behavioral Parameters:')
    print('\tTrain the Model?     : {}'.format(config.TRAIN_MODEL))
    print('\tReload the Model?    : {}'.format(config.RELOAD_MODEL))

    print('    Input Data Parameters:')
    print('\tDATASET_PATH         : {}'.format(config.DATASET_PATH))
    print('\tDATA_WIDTH           : {}'.format(config.DATA_WIDTH))
    print('\tMAX_DATA_WIDTH       : {}'.format(config.MAX_DATA_WIDTH))
    print('\tTARGET_DFA (Truth)   : {}'.format(config.TARGET_DFA))
    print('\tNUM_SAMPLES (per DFA): {}'.format(config.NUM_SAMPLES))
    print('\tCONTIGUOUS_INPUT_DATA: {}'.format(config.CONTIGUOUS_INPUT_DATA))
    print('\tRANDOMIZE_INPUT_DATA : {}'.format(config.RANDOMIZE_INPUT_DATA))

    print('    Trainig Parameters:')
    print('\tEPOCHS               : {}'.format(config.EPOCHS))
    print('\tITERATIONS           : {}'.format(config.ITERATIONS))
    print('\tK_FOLDS              : {}'.format(config.K_FOLDS))
    print('\tMAX_BATCH_SIZE       : {}'.format(config.MAX_BATCH_SIZE))

    print('    Model Data:')
    print('\tMODEL_DATA_PATH      : {}'.format(config.MODEL_DATA_PATH))

    print('    Logging Data:')
    print('\tLOG_DATA_PATH        : {}'.format(config.LOG_DATA_PATH))
    print('\tLOG_DATA_PATH        : {}'.format(config.LOG_DATA_PATH))
    print('------------------------------------------')
    print()

    if not os.path.exists(config.MODEL_DATA_PATH):
        os.makedirs(config.MODEL_DATA_PATH)

    if not os.path.exists(config.LOG_DATA_PATH):
        os.makedirs(config.LOG_DATA_PATH)

    if not os.path.exists(config.TENSORBOARD_LOG_DATA):
        os.makedirs(config.TENSORBOARD_LOG_DATA)

    ##########################################################
    # Load Data
    ##########################################################
    print("Loading Training and Testing data:")
    print("\tLoading data ... ", end='')
    sys.stdout.flush()
    start_time = time.time()

    input_data = None
    output_data = None

    for dw in range(config.MIN_TRAINING_DATA_WIDTH, config.MAX_TRAINING_DATA_WIDTH+1):
        step_input_data, step_output_data = utils.load_data(
            data_width=dw,
            target_dfa=config.TARGET_DFA,
            base_path=config.DATASET_PATH,
            qty_per_dfa=config.NUM_SAMPLES,
            load_contiguous=config.CONTIGUOUS_INPUT_DATA,
            load_random=config.RANDOMIZE_INPUT_DATA,
            max_data_size=config.MAX_DATA_WIDTH,
        )

        if input_data is None:
            input_data = step_input_data
            output_data = step_output_data
        else:
            input_data = np.append(input_data, step_input_data, axis=0)
            output_data = np.append(output_data, step_output_data, axis=0)

    print("COMPLETED! {:.4f}s".format(time.time() - start_time))
    print("Training and Testing data loaded")

    if (len(input_data) // 2) < config.NUM_SAMPLES:
        print('WARNING: The number of samples loaded from the input files ({}) is less than the numbe requested ({})'.format(
            (len(input_data) // 2), config.NUM_SAMPLES))

    if (len(input_data) < 1):
        raise('No input data was loaded')

    # Hack to allow for optimization when loading different sample sizes from different files.
    # Not sure what the upper limit is for the video card(s).  Guessing it's 200,000 or less.
    config.MAX_BATCH_SIZE = min(50000, len(input_data))

    ##########################################################
    # Prepare Model
    ##########################################################

    if config.RELOAD_MODEL is True:
        ##########################################################
        # Load Model
        ##########################################################
        model = load_model(config.MODEL_DATA_PATH + '/model.h5')

    else:
        ##########################################################
        # Build Model
        ##########################################################

        with tf.device('/cpu:0'):
            model = Sequential()

            if config._BACKEND == 'tensorflow':
                print('INFORMATIONAL: Creating model using CuDNNLSTM')
                model.add(
                    CuDNNLSTM(1,
                              input_shape=(config.MAX_DATA_WIDTH, 1),
                              unit_forget_bias=True
                              )
                )
            else:
                print('INFORMATIONAL: Creating model using LSTM')
                model.add(
                    LSTM(1,
                         input_shape=(config.MAX_DATA_WIDTH, 1),
                         activation='tanh',
                         recurrent_activation='sigmoid',
                         unit_forget_bias=True,
                         recurrent_dropout=0.99,
                         )
                )

            model.add(Activation('sigmoid'))
            # model.add(Activation('tanh'))
            model.compile(loss='binary_crossentropy',
                               optimizer='rmsprop',
                               metrics=[
                                   metrics.binary_accuracy,
                                   metrics.binary_crossentropy,
                                   metrics.mae,
                               ])

        parallel_model = multi_gpu_model(model, gpus=2)
        parallel_model.compile(loss='binary_crossentropy',
                               optimizer='rmsprop',
                               metrics=[
                                   metrics.binary_accuracy,
                                   metrics.binary_crossentropy,
                                   metrics.mae,
                               ])

    ############################################################
    # k-fold train/test
    ############################################################
    training_start_time = time.time()
    timestamp = str(training_start_time)
    running_score = []
    plateaued = False
    try:
        print('Performing {} iteration(s) of training.'.format(config.ITERATIONS))
        for iteration in range(config.ITERATIONS):
            if plateaued is True:
                break

            print('Starting iteration #{}'.format(iteration))

            for d, v_in, v_out, t_in, t_out in utils.kfold(input_data, output_data, config.K_FOLDS):
                print('Performing k-fold #{:02}'.format(d))

                if config.TRAIN_MODEL is True:

                    if config.RANDOMIZE_INPUT_DATA is True:
                        tf_log_dir = os.path.join(
                            config.TENSORBOARD_LOG_DATA, 'rand')
                    elif config.CONTIGUOUS_INPUT_DATA is True:
                        tf_log_dir = os.path.join(
                            config.TENSORBOARD_LOG_DATA, 'cont')
                    else:
                        tf_log_dir = os.path.join(
                            config.TENSORBOARD_LOG_DATA, 'step')

                    tf_log_dir = os.path.join(
                        tf_log_dir, str(config.NUM_SAMPLES))
                    tf_log_dir = os.path.join(
                        tf_log_dir, str(config.DATA_WIDTH))
                    tf_log_dir = os.path.join(tf_log_dir, str(config.EPOCHS))
                    tf_log_dir = os.path.join(
                        tf_log_dir, str(config.ITERATIONS))
                    tf_log_dir = os.path.join(tf_log_dir, str(iteration))
                    tf_log_dir = os.path.join(tf_log_dir, timestamp)
                    tf_log_dir = os.path.join(tf_log_dir, 'd{:02}'.format(d))
                    os.makedirs(tf_log_dir, exist_ok=True)

                    print("Logging TensorBoard data to: {}".format(tf_log_dir))

                    cb_tb = callbacks.TensorBoard(log_dir=tf_log_dir,
                                                  histogram_freq=0,
                                                  write_graph=True,
                                                  write_grads=False,
                                                  write_images=False)

                    if config.RANDOMIZE_INPUT_DATA is True:
                        csv_log_dir = os.path.join(
                            config.CSV_LOG_DATA, 'rand')
                    elif config.CONTIGUOUS_INPUT_DATA is True:
                        csv_log_dir = os.path.join(
                            config.CSV_LOG_DATA, 'cont')
                    else:
                        csv_log_dir = os.path.join(
                            config.CSV_LOG_DATA, 'step')

                    csv_log_dir = os.path.join(
                        csv_log_dir, str(config.NUM_SAMPLES))
                    csv_log_dir = os.path.join(
                        csv_log_dir, str(config.DATA_WIDTH))
                    csv_log_dir = os.path.join(csv_log_dir, str(config.EPOCHS))
                    csv_log_dir = os.path.join(
                        csv_log_dir, str(config.ITERATIONS))
                    csv_log_dir = os.path.join(csv_log_dir, str(iteration))
                    csv_log_dir = os.path.join(csv_log_dir, timestamp)
                    csv_log_dir = os.path.join(csv_log_dir, 'd{:02}'.format(d))
                    os.makedirs(csv_log_dir, exist_ok=True)

                    print("Logging CSV data to: {}".format(csv_log_dir))

                    cb_csv = callbacks.CSVLogger(
                        os.path.join(csv_log_dir, 'training.csv'))

                    # print(
                    #     "Setting Reduce Learning Rate on Plateau to monitor the 'mean_absolute_error'")
                    # cb_lrop = callbacks.ReduceLROnPlateau(
                    #     monitor='mean_absolute_error',
                    #     factor=0.1,
                    #     patience=5,
                    #     verbose=0,
                    #     mode='min',
                    #     epsilon=0.001,
                    #     cooldown=10,
                    #     min_lr=0
                    # )

                    print(
                        "Setting Early Stopping to monitor the 'mean_absolute_error' for a change of 0.0001 and delta of 1000.")
                    cb_es = callbacks.EarlyStopping(
                        monitor='mean_absolute_error', min_delta=0.0001, patience=5, mode='min')

                    # cb_plateaued = LambdaCallback(
                    #     on_train_end=lambda mean_absolute_error: print(mean_absolute_error))

                    parallel_model.fit(
                        t_in,
                        t_out,
                        batch_size=config.MAX_BATCH_SIZE,
                        epochs=config.EPOCHS,
                        callbacks=[cb_tb, cb_csv, cb_es]
                    )
                pass

                print("Evaluating Performance...")

                score = parallel_model.evaluate(
                    v_in,
                    v_out,
                    batch_size=config.MAX_BATCH_SIZE
                )
                print(score)
                running_score.append(score)

            score = parallel_model.evaluate(
                input_data,
                output_data,
                batch_size=config.MAX_BATCH_SIZE
            )
    except KeyboardInterrupt as e:
        pass
    finally:
        if config.TRAIN_MODEL is True:
            print('Training duration: {:.4f} seconds'.format(
                time.time() - training_start_time))
            print("Saving Model")

            # model_file_path = os.path.join(
            #     config.MODEL_DATA_PATH, 'model.h5')
            # model_json_file_path = os.path.join(
            #     config.MODEL_DATA_PATH, 'model.json')

            # model_json = parallel_model.to_json()
            # with open(model_json_file_path, "w") as json_file:
            #     json_file.write(model_json)
            # # serialize weights to HDF5
            # model.save_weights(model_file_path)

            # model_file_path = os.path.join(config.MODEL_DATA_PATH,
            #                                '{}_model.h5'.format(timestamp))
            # model_json_file_path = os.path.join(config.MODEL_DATA_PATH,
            #                                '{}_model.json'.format(timestamp))
            # model_json = parallel_model.to_json()
            # with open(model_json_file_path, "w") as json_file:
            #     json_file.write(model_json)
            # # serialize weights to HDF5
            # model.save_weights(model_file_path)

            model.save(config.MODEL_DATA_PATH + '/model.h5')
            model.save(os.path.join(config.MODEL_DATA_PATH,
                                    '{}_model.h5'.format(timestamp)))

            print("Saved model to disk")

    t_loss = 0.0
    t_binary_accuracy = 0.0
    t_binary_crossentropy = 0.0
    t_mean_absolute_error = 0.0

    for loss, binary_accuracy, binary_crossentropy, mean_absolute_error in running_score:
        t_loss += loss
        t_binary_accuracy += binary_accuracy
        t_binary_crossentropy += binary_crossentropy
        t_mean_absolute_error += mean_absolute_error

    t_loss = t_loss / len(running_score)
    t_binary_accuracy = t_binary_accuracy / len(running_score)
    t_binary_crossentropy = t_binary_crossentropy / len(running_score)
    t_mean_absolute_error = t_mean_absolute_error / len(running_score)
    print('Average Scores:')
    print('\tAvg. Total Loss: {}'.format(t_loss))
    print('\tAvg. Total Binary Accuracy: {}'.format(t_binary_accuracy))
    print('\tAvg. Total Binary Cross-Entropy: {}'.format(t_binary_crossentropy))
    print('\tAvg. Total Mean Absolute Error: {}'.format(t_mean_absolute_error))
    print('Final Pass:')
    print('\tLoss: {}'.format(score[0]))
    print('\tAccuracy: {}'.format(score[1]))


"""
IK LEG rig setup
main module
"""

import sys

from . import project

from ..rigLib.rig import IK_Leg

import maya.cmds as cmds

sys.path.append('C:/Users/user/Documents/maya/2017/scripts/KOMODO/code/python')

sceneScale = project.sceneScale
mainProjectPath = project.mainProjectPath

modelFilePath = None

builderSceneFilePath = '%s/%s/builder/%s_builder.mb'

def builder(characterName):
    """
    main function to build IK leg rig
    :param characterName: characterName:str,
    :return:
    """
    # new Scene
    cmds.file(new=1, force=1)

    # import builder Scene
    # builderFile = builderSceneFilePath % (mainProjectPath, characterName, characterName)
    builderFile = 'C:/Users/user/Documents/maya/2017/scripts/KOMODO/code/assets/IK_LEG/builder'
    cmds.file(builderFile, i=1)

    # make controlSetup
    makeControlSetup()

def makeControlSetup():

    topJoint = 'L_Skin_hip'
    pvLocator = 'pvLocator'
    revLocator = ['CBank_LOC', 'EBank_LOC', 'Heel_LOC', 'Pivot_LOC']
    rollLocator = 'rollLocator'

    IK_Leg.build(topJoint=topJoint,
                 pvLocator=pvLocator,
                 revLocator=revLocator,
                 prefix='L_',
                 rigScale=sceneScale*5,
                 rollCtrlLOC=rollLocator)


from django.shortcuts import render, redirect

from my_music_app.album.models import Album
from my_music_app.user_profile.forms import ProfileCreateForm, ProfileDeleteForm
from my_music_app.user_profile.models import Profile

def profile_details(request):
    profile = Profile.objects.first()
    albums_count = Album.objects.all().count()

    context = {
        'profile': profile,
        'albums_count': albums_count
    }

    return render(request, 'user_profile/profile-details.html', context)

def delete_profile(request):
    profile = Profile.objects.first()

    if request.method == 'GET':
        form = ProfileDeleteForm(instance=profile)
    else:
        form = ProfileDeleteForm(request.POST, instance=profile)

        if form.is_valid():
            form.save()
            return redirect('index')

    context = {
        'form': form
    }

    return render(request, 'user_profile/profile-delete.html', context)


import spacy

nlp = spacy.load("fr_core_news_sm")

text = "Le constructeur Citröen présente la e-Méhari Courrèges au public."

# Traite le texte
doc = nlp(text)

# Itère sur les entités
for ent in doc.ents:
    # Affiche le texte de l'entité et son label
    print(ent.text, ent.label_)

# Obtiens la portion pour "e-Méhari Courrèges"
e_mehari_courreges = doc[5:7]

# Affiche la portion de texte
print("Entité manquante :", e_mehari_courreges.text)


import os
import tempfile
import datetime
import shutil
from lxml import etree
from lxml.etree import QName

from .metadata import Metadata
from .myzipfile import ZipFile, is_zipfile, ZIP_DEFLATED, ZIP_STORED
from .exceptions import BadFormat, WriteEpubException
from .utils import person_sort_name, xstr

class Epub3:

    ns = {
        'n': 'urn:oasis:names:tc:opendocument:xmlns:container',
        'opf': 'http://www.idpf.org/2007/opf',
        'dc': 'http://purl.org/dc/elements/1.1/'
    }

    def __init__(self, file):

        self.file = file
        self.version = ''
        self.tree = None
        self.content_root = ''
        self.opf_file = ''
        self.cover_href = ''
        self.cover_image_data = None

        self.load()

    def load(self):
        if is_zipfile(self.file):
            z = ZipFile(self.file)
            container = z.read('META-INF/container.xml')
            tree = etree.fromstring(container)
            self.opf_file = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=self.ns)[0]
            opf_data = z.read(self.opf_file)
            z.close()
            self.content_root = os.path.split(self.opf_file)[0]
            if self.content_root:
                self.content_root += '/'
            self.tree = etree.fromstring(opf_data)
            v = self.tree.xpath('/opf:package/@version', namespaces=self.ns)[0]
            self.version = v[:1]
            if self.version != '3':
                raise BadFormat('wrong epub3 version')
        else:
            raise BadFormat('wrong zip format')

    def save(self):

        temp_dir = tempfile.mkdtemp(prefix='em')
        dest_file = os.path.join(temp_dir, os.path.split(self.file)[1])
        src_zip = ZipFile(self.file, mode='r')
        dest_zip = ZipFile(dest_file, mode='w')
        try:
            for f in src_zip.infolist():
                if f.filename == self.opf_file:
                    dest_zip.writestr(self.opf_file, etree.tostring(self.tree, encoding='utf-8',
                                      method='xml', xml_declaration=True, pretty_print=True))

                elif f.filename == self.cover_href:
                    if self.cover_image_data is not None:
                        dest_zip.writestr(self.content_root + self.cover_href, self.cover_image_data)
                elif f.filename == 'mimetype':
                    buf = src_zip.read(f)
                    dest_zip.writestr(f.filename, buf, ZIP_STORED)
                else:
                    buf = src_zip.read(f)
                    dest_zip.writestr(f.filename, buf, ZIP_DEFLATED)
            src_zip.close()
            dest_zip.close()
            shutil.copyfile(dest_file, self.file)
            shutil.rmtree(temp_dir)
        except Exception:
            shutil.rmtree(temp_dir)
            raise WriteEpubException

    def get_metadata(self, short_author_name=False):

        metadata = Metadata()

        metadata.identifier = {}
        metadata.identifier['value'] = ''
        metadata.identifier['attrib'] = {}
        identifier = self.get('opf:metadata/dc:identifier')
        if identifier is not None:
            metadata.identifier['value'] = identifier.text
            for attr in identifier.attrib:
                metadata.identifier['attrib'][attr] = identifier.attrib[attr]

        title_list = self.getall('opf:metadata/dc:title')
        for e in title_list:
            value = self.get_element_refines(self.get_element_id(e), property='title-type')
            if value == 'main' or not value:
                metadata.title = e.text
        creator_list = self.getall('opf:metadata/dc:creator')
        for e in creator_list:
            value = self.get_element_refines(self.get_element_id(e), property='role')
            if value == 'aut' or not value:
                metadata.author.append(e.text)
                metadata.author_sort.append(person_sort_name(e.text))
            elif value == 'trl':
                metadata.translator.append(e.text)
        metadata.series = xstr(self.get('opf:metadata/opf:meta[@name="calibre:series"]/@content'))
        metadata.series_index = xstr(self.get('opf:metadata/opf:meta[@name="calibre:series_index"]/@content'))
        node_list = self.getall('opf:metadata/dc:subject')
        metadata.tag = []
        for n in node_list:
            metadata.tag.append(n.text)
        metadata.description = xstr(self.get('opf:metadata/dc:description/text()'))
        metadata.lang = xstr(self.get('opf:metadata/dc:language/text()'))
        metadata.date = xstr(self.get('opf:metadata/dc:date/text()'))
        metadata.publisher = xstr(self.get('opf:metadata/dc:publisher/text()'))
        metadata.cover_image_data = self.get_cover_image()
        metadata.format = 'epub'
        metadata.file = self.file

        return metadata

    def get_element_refines(self, id, property):

        value = self.get('opf:metadata/opf:meta[@refines="#{}" and @property="{}"]/text()'.format(id, property))
        if value is not None:
            return xstr(value)
        return ''

    def get_element_id(self, elem):

        return elem.attrib['id'] if 'id' in elem.attrib else ''

    def get_cover_image(self):

        cover_data = None
        cover_href = self.get_cover_href(self.get_cover_id())
        if cover_href is not None:
            z = ZipFile(self.file)
            cover_data = z.read(self.content_root + cover_href)
            z.close()

        return cover_data

    def get_cover_id(self):

        cover_id = self.get('opf:metadata/opf:meta[@name="cover"]/@content')
        if cover_id is None:
            cover_id = self.get('opf:manifest/opf:item[@properties="cover-image"]/@id')
        return cover_id

    def get_cover_href(self, cover_id):

        return self.get('opf:manifest/opf:item[@id="{}"]/@href'.format(cover_id))

    def set_metadata(self, metadata):

        meta = self.Element('opf:metadata')

        if metadata.identifier['value']:
            node = self.SubElement(meta, 'dc:identifier')
            node.text = metadata.identifier['value']
            for attr in metadata.identifier['attrib']:
                node.attrib[attr] = metadata.identifier['attrib'][attr]

        if metadata.title:
            node = self.SubElement(meta, 'dc:title')
            node.text = metadata.title
            node.attrib['id'] = 'title'
            self.add_refines(meta, id='title', property='title-type', text='main')

        idx = 1
        for author in metadata.author:
            self.add_creator(meta, name=author, id='author{}'.format(idx), role='aut')
            idx += 1

        idx = 1
        for translator in metadata.translator:
            self.add_creator(meta, name=translator, id='translator{}'.format(idx), role='trl')
            idx += 1

        for tag in metadata.tag:
            self.SubElement(meta, 'dc:subject').text = tag

        if metadata.description:
            self.SubElement(meta, 'dc:description').text = metadata.description

        if metadata.publisher:
            self.SubElement(meta, 'dc:publisher').text = metadata.publisher

        if metadata.date:
            self.SubElement(meta, 'dc:date').text = metadata.date

        if metadata.lang:
            self.SubElement(meta, 'dc:language').text = metadata.lang

        if metadata.series:
            node = self.SubElement(meta, 'opf:meta')
            node.attrib['name'] = 'calibre:series'
            node.attrib['content'] = metadata.series

        if metadata.series_index:
            node = self.SubElement(meta, 'opf:meta')
            node.attrib['name'] = 'calibre:series_index'
            node.attrib['content'] = str(metadata.series_index)

        node = self.SubElement(meta, 'opf:meta')
        node.attrib['property'] = 'dcterms:modified'
        node.text = datetime.datetime.now().isoformat(timespec='seconds') + 'Z'

        if metadata.cover_image_data is not None:
            cover_id = self.get_cover_id()
            if cover_id is None:
                cover_id = 'coverimage'
            self.cover_href = self.get_cover_href(cover_id)

            node = self.SubElement(meta, 'opf:meta')
            node.attrib['name'] = 'cover'
            node.attrib['content'] = cover_id

            if self.cover_href is None:
                self.cover_href = 'cover.jpg'
            else:
                cover_name, _ = os.path.splitext(self.cover_href)
                self.cover_href = cover_name + '.jpg'

            node = self.get('opf:manifest/opf:item[@id="{}"]'.format(cover_id))
            if node is None:
                node = self.Element('opf:item')
                node.attrib['id'] = cover_id
                manifest_node = self.get('opf:manifest')
                manifest_node.append(node)
            node.attrib['href'] = self.cover_href
            node.attrib['properties'] = 'cover-image'
            node.attrib['media-type'] = 'image/jpeg'
            self.cover_image_data = metadata.cover_image_data

            meta_node = self.get('opf:metadata')
            meta_node.getparent().replace(meta_node, meta)
            self.save()

    def add_creator(self, parent, name, id, role):

        node = self.SubElement(parent, 'dc:creator')
        node.text = name
        node.attrib['id'] = id
        self.add_refines(parent, id=id, property='file-as', text=person_sort_name(name, first_delimiter=', '))
        self.add_refines(parent, id, 'role', role)

    def add_refines(self, parent, id, property, text):

        node = self.SubElement(parent, 'opf:meta')
        node.attrib['refines'] = '#{}'.format(id)
        node.attrib['property'] = property
        node.text = text

    def Element(self, name):

        ns, tag = name.split(':')
        return etree.Element(QName(self.ns[ns], tag), nsmap=self.ns)

    def SubElement(self, parent, name):

        ns, tag = name.split(':')
        return etree.SubElement(parent, QName(self.ns[ns], tag))

    def get(self, name):

        result_list = self.tree.xpath(name, namespaces=self.ns)
        for r in result_list:
            return r

    def getall(self, name):

        return self.tree.xpath(name, namespaces=self.ns)


import os
import datetime
import pickle
import requests
import bs4 as bs
import pandas as pd
import pandas_datareader as web
from quantipy.basics.data_struct import StocksData, OptionsData, FuturesData

class BaseImport(object):
    mapping = {}
    datatypes = {}

    def __init__(self):
        self.dataframe = pd.DataFrame()

    def _enhance_data(self):
        pass

    def _data_cleansing(self):
        pass

    def _remap(self, mapping):
        self.dataframe.rename(columns=mapping, inplace=True)

    def import_data(self, **kwargs):
        if "datasource" not in kwargs:
            if self.dataframe.empty:
                return
        else:
            self.dataframe = kwargs["datasource"]

        if "mapping" in kwargs:
            self._remap(kwargs["mapping"])
        else:
            self._remap(self.mapping)

        self._data_cleansing()
        self._enhance_data()

        return self.dataframe

class BaseCSVImport(BaseImport):
    def __init__(self):
        super().__init__()
        self.datatypes = {}
        self.mapping = {}

    def import_data(self, filename, delimiter=",", **kwargs):
        if "datatypes" in kwargs:
            self.datatypes = kwargs["datatypes"]

        if "mapping" in kwargs:
            self.mapping = kwargs["mapping"]

        self.dataframe = pd.read_csv(filename, delimiter=delimiter, dtype=self.datatypes)
        super().import_data(mapping=self.mapping)
        return self.dataframe

class OptionVueImport(BaseCSVImport):
    """" Import data from a CSV file that was exported in OptionVue

         Todo: Futures import does not work correctly. Also the OptionVue export does not supply an expiration date
               for futures contracts as of 17.FEB.2017.
    """

    def __init__(self):
        super().__init__()
        self.datatypes = {
            "Time": "str",
            "Date": "str",
            "Exp.Date": "str"
        }
        self.mapping = {"Symbol": "symbol",
                           "Item Type": "type",
                           "Last": "last",
                           "Change": "change",
                           "High": "high",
                           "Low": "low",
                           "Volume": "volume",
                           "Theta": "theta",
                           "Time": "time",
                           "Asked": "ask",
                           "Bid": "bid",
                           "Date": "date",
                           "Delta": "delta",
                           "Description": "description",
                           "Exp.Date": "exp_date",
                           "Gamma": "gamma",
                           "Mid IV": "iv",
                           "Market Price": "market_price",
                           "Open": "open",
                           "Open Interest": "open_interest",
                           "Vega": "vega",
                           "Call/Put": "p_c",
                           "Previous Close": "prev_close",
                           "Rho": "rho",
                           "Strike Price": "strike",
                           "SV": "atm_sv"}

    def _data_cleansing(self):
        #drop columns that are not needed
        self.dataframe.drop(["Original Price", "Prob.Finish.ITM", "Projected Volty", "Time Premium", "Ask IV",
                             "At Price", "Existing Posn.", "Bid IV", "Pcnt To Double", "Percent O/U", "Th.Price",
                             "Trade"], axis=1, inplace=True)

        #fill NaN values for columns and set datatypes
        self.dataframe["volume"].fillna(0, inplace=True)
        self.dataframe["volume"].astype(int)
        self.dataframe["open_interest"].fillna(0, inplace=True)
        self.dataframe["open_interest"].astype(int)
        self.dataframe["exp_date"].fillna("", inplace=True)

        # Filter out all rows that don't have a market price, these couldn't be traded
        self.dataframe = self.dataframe[self.dataframe["market_price"] > 0.0]

        # If date columns are filled, prefix them with the century and convert to datetime object
        self.dataframe.loc[self.dataframe["date"] != "", "date"] = "20" + self.dataframe["date"]
        self.dataframe.loc[self.dataframe["exp_date"] != "", "exp_date"] = "20" + self.dataframe["exp_date"]
        self.dataframe["date"] = pd.to_datetime(self.dataframe["date"])
        self.dataframe["exp_date"] = pd.to_datetime(self.dataframe["exp_date"])

    def _enhance_dataframe(self):
        # Extract Symbol
        self.dataframe["symbol"] = self.dataframe["symbol"].str.split(' ').str[0]
        # Add close data to current data
        self.dataframe["close"] = self.dataframe["market_price"]
        # Add percent change
        self.dataframe["change_pct"] = (self.dataframe["close"]
                                        - self.dataframe["prev_close"]) / self.dataframe["prev_close"]
        self.dataframe["dte"] = (self.dataframe["exp_date"] - self.dataframe["date"]).dt.days

    def import_data(self, filename, delimiter=",", **kwargs):
        """ Import CSV files exported from OptionVue. These files contain stocks, futures and options data
            in the same file. The export structure is dynamic, depending on the list selected in OptionVue
        """
        super().import_data(filename, delimiter, **kwargs)

        #self.dataframe.to_csv("./Data/OptionVue/conv.csv", index=False, float_format='%.12g')

        stock_data = self.dataframe[self.dataframe["type"] == "S"]
        option_data = self.dataframe[self.dataframe["type"] == "G"].append(self.dataframe[self.dataframe["type"] == "O"])
        future_data = self.dataframe[self.dataframe["type"] == "F"]

        return stock_data, future_data, option_data

class YahooStockImport(BaseImport):
    _symbol_pickle_filename = "./Data/Yahoo/sp500tickers.pkl"
    _stock_dir = "./Data/Yahoo/Stocks"

    def __init__(self):
        super().__init__()
        # This is the mapping from Yahoo's web-data to the fieldnames used internally in QuantiPy
        self.mapping = {
            "Date": "date",
            "Open": "open",
            "High": "high",
            "Low": "low",
            "Volume": "volume",
            "Close": "close",
            "Adj Close": "adj_close",
            "Description": "description",
            "Symbol": "symbol"
        }

    def _enhance_data(self):
        self.dataframe["factor"] = self.dataframe["close"] / self.dataframe["adj_close"]
        self.dataframe["close"] = self.dataframe["adj_close"]
        self.dataframe["open"] = self.dataframe["open"] / self.dataframe["factor"]
        self.dataframe["high"] = self.dataframe["high"] / self.dataframe["factor"]
        self.dataframe["low"] = self.dataframe["low"] / self.dataframe["factor"]

        # Last data = close
        self.dataframe["last"] = self.dataframe["close"]
        # Previous close from previously processed data
        self.dataframe["prev_close"] = self.dataframe["close"].shift(1)
        # Open Interest, yahoo doesn't have this info so set to 0
        self.dataframe["open_interest"] = 0
        # Market price is set to close, because no bid/ask available
        self.dataframe["market_price"] = self.dataframe["close"]
        # Bid/ask is set to close
        self.dataframe["bid"] = self.dataframe["close"]
        self.dataframe["ask"] = self.dataframe["close"]
        # Time is always EOD
        self.dataframe["time"] = datetime.time(22, 0)
        # Create change and change percent from previous
        self.dataframe["change"] = self.dataframe["close"] - self.dataframe["prev_close"]
        self.dataframe["change_pct"] = self.dataframe["change"] / self.dataframe["prev_close"]

    def _web_load_symbols(self):
        resp = requests.get('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
        soup = bs.BeautifulSoup(resp.text, "lxml")
        table = soup.find('table', {'class': 'wikitable sortable'})
        tickers = []
        for row in table.findAll('tr')[1:]:
            ticker, description = row.findAll('td')[0].text, row.findAll('td')[1].text
            tickers.append((ticker, description))

        with open(self._symbol_pickle_filename, "wb") as f:
            pickle.dump(tickers, f)
        return tickers

    def _load_symbols(self):
        if os.path.exists(self._symbol_pickle_filename):
            with open(self._symbol_pickle_filename, "rb") as f:
                tickers = pickle.load(f)
        else:
            tickers = self._web_load_symbols()
        return tickers

    def import_data(self, stock_symbol=[], from_date="2001-01-01", to_date="2099-01-01",
                    reload_sp500_symbols=False, reload_data=False, **kwargs):
        if not os.path.exists(self._stock_dir):
            os.makedirs(self._stock_dir)

        symbols = []
        if stock_symbol:
            symbols = stock_symbol
        else:
            if reload_sp500_symbols:
                symbols = self._web_load_symbols()
            else:
                symbols = self._load_symbols()

        if not symbols:
            return

        start_date = datetime.datetime.strptime(from_date, "%Y-%m-%d")
        end_date = datetime.datetime.strptime(to_date, "%Y-%m-%d")

        del self.dataframe
        self.dataframe = pd.DataFrame()

        for symbol in symbols:
            if type(symbol) == str:
                symbol_to_load, description = symbol, symbol
            else:
                symbol_to_load, description = symbol[0], symbol[1]

            df = pd.DataFrame()
            if reload_data or not os.path.exists(self._stock_dir + "/{}.csv".format(symbol_to_load)):
                try:
                    df = web.DataReader(symbol_to_load, 'yahoo', start_date, end_date)
                    df["Description"] = description
                    df["Symbol"] = symbol_to_load
                    df.to_csv(self._stock_dir + "/{}.csv".format(symbol_to_load))
                    df.reset_index(inplace=True)
                except:
                    continue
            else:
                with open(self._stock_dir + "/{}.csv".format(symbol_to_load)) as datafile:
                    df = pd.read_csv(self._stock_dir + "/{}.csv".format(symbol_to_load), parse_dates=True)
                    # Filter all dates that are not within the selected period. Filtering a vector should be faster than
                    # Looping through csv
                    df = df[(df["Date"] >= from_date) & (df["Date"] <= to_date)]

            self.dataframe = self.dataframe.append(df)

        super().import_data()
        return self.dataframe


import os
import sys
import pylab
import numpy
import torch
import time
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.gridspec as gridspec
import matplotlib.cm as cm
import copy
#import astropy.stats

from scipy.spatial.transform import Rotation as R

def _update_attached_visualization_bounds(subgridspec, visualization_bounds):
    """
    Attach visualization bounds to subgridspec or update existing bounds.
    """

    if(hasattr(subgridspec, "visualization_bounds")):
        new_bounds=[]

        for ind_b,cur_b in enumerate(subgridspec.visualization_bounds):
            new_b=(min([cur_b[0], visualization_bounds[ind_b][0]]), max([cur_b[1], visualization_bounds[ind_b][1]]))

            new_bounds.append(new_b)

        setattr(subgridspec, "visualization_bounds", new_bounds)

    else:
        setattr(subgridspec, "visualization_bounds", visualization_bounds)

def find_bins(trace, percentiles=[5.0,95.0], num_bins=50, use_outlier_binning=False):

    if(use_outlier_binning):
        perc=numpy.percentile(trace, percentiles)

        bins=list(numpy.linspace(perc[0], perc[1], num_bins-2))

        new_edges=[min(trace)-1e-5]+bins+[max(trace)+1e-5]

        new_edges=numpy.array(new_edges)

    else:

        new_edges=numpy.linspace(min(trace)-1e-5, max(trace)+1e-5, num_bins)

    return new_edges

def obtain_bins_and_visualization_regions(samples, model, percentiles=[5.0,95.0], relative_buffer=0.1, num_bins=50, s2_norm="standard", use_outlier_binning=False):

    """
    Uses samples and pdf defs to calculate binning and visualization regions.

    Euclidean:
    Obtain an uneven binning based on Bayesian Blocks and a region that makes sense for visualization based on 1-d percentiles.
    Adds a relative buffer to both sides.

    Other:
    Non-Euclidean manifolds have fixed bounds for binning and visualization, so this is easy.
    """

    cur_index=0

    visualization_bounds=[]
    density_eval_bounds=[]
    histogram_edges=[]

    for pdf_index, pdf_def in enumerate(model.pdf_defs_list):

        if(pdf_def[0]=="e"):
            dim=int(pdf_def[1:])

            for sub_index in range(cur_index, cur_index+dim):

                np_samples=samples[:,sub_index].cpu().numpy()
                # vis bounds
                boundaries=numpy.percentile(np_samples, percentiles)

                relative_extra=relative_buffer*(boundaries[1]-boundaries[0])

                visualization_bounds.append((boundaries[0]-relative_extra, boundaries[1]+relative_extra))

                # density bounds
                min_val = float(min(samples[:, sub_index]).cpu())
                max_val = float(max(samples[:, sub_index]).cpu())

                relative_extra=0.2*(max_val-min_val)

                density_eval_bounds.append((min_val-relative_extra, max_val+relative_extra))

                # histo bins
                #edges = astropy.stats.bayesian_blocks(np_samples, p0=0.1, gamma=0.9)

                ## find bins based on percentiles
                edges=find_bins(np_samples, percentiles=percentiles, num_bins=num_bins)
                histogram_edges.append(edges)

            cur_index+=dim

        elif(pdf_def[0]=="s"):

            if(int(pdf_def[1])==1):

                visualization_bounds.append((0,2*numpy.pi))
                density_eval_bounds.append((0,2*numpy.pi))
                histogram_edges.append(numpy.linspace(0, 2*numpy.pi, num_bins))
                cur_index+=1

            elif(int(pdf_def[1])==2):

                if(s2_norm=="standard"):

                    visualization_bounds.append((0,numpy.pi))
                    visualization_bounds.append((0,2*numpy.pi))

                    density_eval_bounds.append((0,numpy.pi))
                    density_eval_bounds.append((0,2*numpy.pi))

                    histogram_edges.append(numpy.linspace(0, numpy.pi, num_bins))
                    histogram_edges.append(numpy.linspace(0, 2*numpy.pi, num_bins))

                else:

                    visualization_bounds.append((-2.0,2.0))
                    visualization_bounds.append((-2.0,2.0))

                    density_eval_bounds.append((0,numpy.pi))
                    density_eval_bounds.append((0,2*numpy.pi))

                    histogram_edges.append(numpy.linspace(-2.0,2.0, num_bins))
                    histogram_edges.append(numpy.linspace(-2.0,2.0, num_bins))

                cur_index+=2

        elif(pdf_def[0]=="i"):

            lower=model.layer_list[pdf_index][-1].low_boundary
            higher=model.layer_list[pdf_index][-1].high_boundary

            visualization_bounds.append((lower,higher))

            density_eval_bounds.append((lower, higher))

            histogram_edges.append(numpy.linspace(lower,higher, num_bins))

        elif(pdf_def[0]=="a"):

            lower=0.0
            higher=1.0

            dim=dim=int(pdf_def[1:])

            for dindex in range(dim):
                visualization_bounds.append((lower,higher))

                density_eval_bounds.append((lower, higher))

                histogram_edges.append(numpy.linspace(lower,higher, num_bins))

    return visualization_bounds, density_eval_bounds, histogram_edges

def calculate_contours(pdf_vals, bin_volumes, probs=[0.68, 0.95]):
    totsum = 0.0
    flattend_pdf = pdf_vals.flatten()
    #flattend_volumes = bin_volumes.flatten()

    sorta = numpy.argsort(flattend_pdf)[::-1]

    contour_values = []

    cur_prob_index = 0

    for ind, pdf_eval in enumerate(flattend_pdf[sorta]):
        totsum += pdf_eval*bin_volumes#flattend_volumes[sorta][ind]

        if (totsum > probs[cur_prob_index]):
            contour_values.append(pdf_eval)
            cur_prob_index += 1

        if (cur_prob_index >= len(probs)):
            break

    return contour_values

def get_bounds_from_contour(cres, boundary=0.1):

    cont_min_x = 9999999.9
    cont_max_x = -9999999.9

    cont_min_y = 999999999.9
    cont_max_y = -9999999999.9

    for i in cres.allsegs[0]:

        for j in i:

            if (j[0] < cont_min_x):
                cont_min_x = j[0]
            if (j[0] > cont_max_x):
                cont_max_x = j[0]

            if (j[1] < cont_min_y):
                cont_min_y = j[1]
            if (j[1] > cont_max_y):
                cont_max_y = j[1]

    return cont_min_x, cont_max_x, cont_min_y, cont_max_y

def get_density_eval_minmax_values(samples):
    """
    Obtain a simple estimate of pdf extent based on min/max values of samples.
    Used to determine a region for density evaluation (not plotting).
    """
    mins_maxs = []

    for ind in range(samples.shape[1]):
        min_val = float(min(samples[:, ind]).cpu())
        max_val = float(max(samples[:, ind]).cpu())

        mins_maxs.append((min_val, max_val))

    return mins_maxs

def get_pdf_on_grid(mins_maxs, npts, model, conditional_input=None, s2_norm="standard", s2_rotate_to_true_value=False, true_values=None):

    side_vals = []

    bin_volumes = 1.0#numpy.ones([npts]*len(mins_maxs))
    glob_ind = 0
    #has_high_dim_spheres = False
    cinput = None

    sin_zen_mask=[]

    used_npts=npts
    if(npts<2):
        used_npts=2

    for pdf_index,pdf in enumerate(model.pdf_defs_list):
        this_sub_dim = int(pdf[1])
        if (pdf == "s2" and s2_norm=="lambert"):
            #has_high_dim_spheres = True
            side_vals.append(numpy.linspace(-2, 2, used_npts))
            bin_volumes *= (side_vals[-1][1] - side_vals[-1][0])

            side_vals.append(numpy.linspace(-2, 2, used_npts))
            bin_volumes *= (side_vals[-1][1] - side_vals[-1][0])

            sin_zen_mask.extend([0,0])

        elif(pdf=="s2" and s2_norm=="standard"):

          sin_zen_mask.extend([0,0])

          zen_vals=numpy.linspace(mins_maxs[glob_ind][0]+1e-4, mins_maxs[glob_ind][1]-1e-4, used_npts)
          side_vals.append(zen_vals)
          bin_volumes*=(side_vals[-1][1] - side_vals[-1][0])

          side_vals.append(numpy.linspace(1e-4, 2*numpy.pi-1e-4, used_npts))
          bin_volumes *= (side_vals[-1][1] - side_vals[-1][0])

        elif(pdf=="s2"):
          raise Exception("s2_norm ", s2_norm, " unknown .")
        elif("i1" in pdf):

            side_vals.append(numpy.linspace(model.layer_list[pdf_index][-1].low_boundary+1e-5, model.layer_list[pdf_index][-1].high_boundary-1e-5, used_npts))

            bin_volumes *= (side_vals[-1][1] - side_vals[-1][0])
            sin_zen_mask.append(0)
        else:

            for ind, mm in enumerate(mins_maxs[glob_ind:glob_ind +
                                               this_sub_dim]):

                side_vals.append(numpy.linspace(mm[0], mm[1], used_npts))
                bin_volumes *= (side_vals[-1][1] - side_vals[-1][0])

                sin_zen_mask.append(0)

        glob_ind += this_sub_dim

    eval_positions = numpy.meshgrid(*side_vals)

    torch_positions = torch.from_numpy(
        numpy.resize(
            numpy.array(eval_positions).T,
            (used_npts**len(mins_maxs), len(mins_maxs))))

    eval_positions = torch_positions.clone()

    mask_inner = torch.ones(len(torch_positions)) == 1

    ## check s2 or simplex visualization
    for ind, pdf_def in enumerate(model.pdf_defs_list):

        if (pdf_def == "s2" and s2_norm=="lambert"):

            fix_point=None

            if(s2_rotate_to_true_value and true_values is not None):
              fix_point=true_values[model.target_dim_indices_intrinsic[ind][0]:model.target_dim_indices_intrinsic[ind][1]]

            mask_inner = mask_inner & (torch.sqrt(
                (eval_positions[:, model.target_dim_indices_intrinsic[ind][0]:model.
                                target_dim_indices_intrinsic[ind][1]]**2).sum(axis=1)) <
                                       2)
            ## transform s2 subdimensions from equal-area lambert dimension to real spherical dimensiosn the model can use

            eval_positions[:, model.target_dim_indices_intrinsic[ind][0]:model.
                           target_dim_indices_intrinsic[ind]
                           [1]] = cartesian_lambert_to_spherical(
                               eval_positions[:, model.
                                              target_dim_indices_intrinsic[ind][0]:model.
                                              target_dim_indices_intrinsic[ind][1]], fix_point=fix_point)

            # need some extra care, it seems sometimes nans can appear in the trafo step (only happened on GPU?)
            mask_inner=torch.isfinite(eval_positions[:,0]) & mask_inner
            mask_inner=torch.isfinite(eval_positions[:,1]) & mask_inner

        elif("c" in pdf_def):

            ## simplex .. mask everything outside allowed region
            mask_inner=mask_inner & (eval_positions[:, model.target_dim_indices_intrinsic[ind][0]:model.target_dim_indices_intrinsic[ind][1] ].sum(axis=1) < 1.0)

    if (conditional_input is not None):

        if(type(conditional_input)==list):

            cinput=[]
            for ci in conditional_input:
                cinput.append(ci.repeat(used_npts**len(mins_maxs), 1)[mask_inner])

            if(cinput[0].is_cuda):
                eval_positions=eval_positions.to(cinput[0])
        else:
            cinput = conditional_input.repeat(used_npts**len(mins_maxs), 1)[mask_inner]
            if(cinput.is_cuda):
                eval_positions=eval_positions.to(cinput)

    log_res, _, _ = model(eval_positions[mask_inner], conditional_input=cinput, force_intrinsic_coordinates=True)

    ## update s2+lambert visualizations by adding sin(theta) factors to get proper normalization
    for ind, pdf_def in enumerate(model.pdf_defs_list):
        if (pdf_def == "s2" and s2_norm=="lambert"):
            ## first coordinate is theta currently

            upd=torch.log(torch.sin(eval_positions[mask_inner][:,model.target_dim_indices_intrinsic[ind][0]:model.target_dim_indices_intrinsic[ind][0]+1])).sum(axis=-1)

            ## angle -> cartesian -> subtract
            log_res-=upd

    ## no conditional input and only s2 pdf .. mask bad regions
    flagged_coords=numpy.array([])
    if(conditional_input is None and model.pdf_defs_list[0]=="s2"):

      problematic_pars=model.layer_list[0][0].return_problematic_pars_between_hh_and_intrinsic(eval_positions[mask_inner], flag_pole_distance=0.02)

      if(problematic_pars.shape[0]>0):
        if(s2_norm=="lambert"):
          fix_point=None
          if(s2_rotate_to_true_value and true_values is not None):
              fix_point=true_values[model.target_dim_indices[ind][0]:model.target_dim_indices[ind][1]]
          problematic_pars=spherical_to_cartesian_lambert(problematic_pars, fix_point=fix_point)
      flagged_coords=problematic_pars.cpu().numpy()

    res = (-600.0)*torch.ones(len(torch_positions)).type_as(torch_positions).to(log_res)
    res[mask_inner] = log_res  #.exp()

    res = res.cpu().numpy()
    numpy_positions=eval_positions.cpu().numpy()
    if((numpy.isfinite(res)==False).sum()>0):
      print("Non-finite evaluation during PDF eval for plotting..")
      print((numpy.isfinite(res)==False).sum())
      print(numpy_positions[(numpy.isfinite(res)==False)])

      if(cinput is None):
        r,_,_=model(eval_positions[mask_inner][torch.isfinite(log_res)==False][:])
      else:
        r,_,_=model(eval_positions[mask_inner][torch.isfinite(log_res)==False][:], conditional_input=cinput[torch.isfinite(log_res)==False])
      print(r)
      raise Exception()

    #######################

    res.resize([used_npts] * len(mins_maxs))

    resized_torch_positions = torch_positions.cpu().numpy()
    resized_torch_positions.resize([used_npts] * len(mins_maxs) + [len(mins_maxs)])

    ## add in sin(theta) factors into density

    """

    for ind, sz in enumerate(sin_zen_mask):
      if(sz==1):
        slice_mask=(None,)*ind+(slice(None,None),)+(None,)*(len(sin_zen_mask)-1-ind)

        zen_vals=numpy.sin(numpy.linspace(mins_maxs[ind][0]+1e-4, mins_maxs[ind][1]-1e-4, npts))

        ## log val, adding zenith factors where needed
        res+=numpy.log(zen_vals[slice_mask])
    """

    return resized_torch_positions, res, bin_volumes, sin_zen_mask, flagged_coords

def rotate_coords_to(theta, phi, target, reverse=False):

  target_theta=target[0].cpu().numpy()
  target_phi=target[1].cpu().numpy()

  phi=phi.cpu().numpy()
  theta=theta.cpu().numpy()

  x=numpy.cos(target_phi)*numpy.sin(target_theta)
  y=numpy.sin(target_phi)*numpy.sin(target_theta)
  z=numpy.cos(target_theta)

  ###########

  axis=-numpy.cross(numpy.array([x,y,z]), numpy.array([0,0,1]))
  axis_len=numpy.sqrt((axis**2).sum())
  axis/=axis_len

  rot_angle=numpy.pi-target_theta
  if(reverse):
    rot_angle=-rot_angle

  axis*=rot_angle.item()

  rot_matrix = R.from_rotvec(axis)
  ###########

  x=numpy.cos(phi)*numpy.sin(theta)
  y=numpy.sin(phi)*numpy.sin(theta)
  z=numpy.cos(theta)

  vals=numpy.concatenate([x[:,None], y[:,None],z[:,None]], axis=1)

  res=torch.from_numpy(rot_matrix.apply(vals))

  ##########

  theta=numpy.arccos(res[:,2])
  non_finite_mask=numpy.isfinite(theta)==False
  larger=non_finite_mask & (res[:,2] > 0)
  smaller=non_finite_mask & (res[:,2] < 0)

  theta[smaller]=numpy.pi
  theta[larger]=0.0

  phi=numpy.arctan2(res[:,1],res[:,0])

  #phi_smaller_mask=phi<0
  #phi[phi_smaller_mask]=phi[phi_smaller_mask]+2*numpy.pi

  return theta.to(target), phi.to(target)

def cartesian_lambert_to_spherical(xl, fix_point=None):

    ## first go to spherical lambert

    r = torch.sqrt((xl**2).sum(axis=1))
    phi = torch.acos(xl[:, 0] / r)
    larger_mask = (xl[:, 1] >= 0)

    phi = larger_mask * phi + (larger_mask == 0) * (2 * numpy.pi - phi)
    theta = 2 * torch.acos(r / 2.0)

    if(fix_point is not None):

      theta, phi = rotate_coords_to(theta, phi, fix_point, reverse=True)

    ## now go to spherical real coordinates

    return torch.cat([theta[:, None], phi[:, None]], dim=1)

def spherical_to_cartesian_lambert(spherical, fix_point=None):

    #####################

    theta = spherical[:, 0]
    phi_lambert = spherical[:, 1]

    ######################
    if(fix_point is not None):
      theta, phi_lambert = rotate_coords_to(theta, phi_lambert, fix_point)

    r_lambert = 2 * torch.cos(theta / 2.0)

    x_l = r_lambert * torch.cos(phi_lambert)
    y_l = r_lambert * torch.sin(phi_lambert)

    return torch.cat([x_l[:, None], y_l[:, None]], dim=1)

def show_sample_contours(ax,
                         samples,
                         bins=50,
                         color="white",
                         contour_probs=[0.68, 0.95],
                         sin_zen_mask=[0,0]):

    ## if bins is a list, make sure samples are within bounds by artifical intervention

    new_samples = samples
    if (type(bins) == list):

        bounds_x_mask = (samples[:, 0] < bins[0][-1]) & (samples[:, 0] >
                                                         bins[0][0])
        bounds_y_mask = (samples[:, 1] < bins[1][-1]) & (samples[:, 1] >
                                                         bins[1][0])

        total_included_mask = bounds_x_mask & bounds_y_mask

        if (total_included_mask.sum() < 100):
            print("TOTAL INCLUDED SUM ", total_included_mask.sum())
            print(
                "too few SAMPLES IN PLOTTING RANGE ... fake entries to evaluate fake contour"
            )
            somex = numpy.random.uniform(bins[0][0],
                                         bins[0][-1],
                                         size=len(samples))
            somey = numpy.random.uniform(bins[1][0],
                                         bins[1][-1],
                                         size=len(samples))

            new_samples = numpy.zeros_like(samples)
            new_samples[:, 0] = somex
            new_samples[:, 1] = somey

    bin_fillings, xedges, yedges = numpy.histogram2d(new_samples[:, 0],
                                                     new_samples[:, 1],
                                                     bins=bins,
                                                     density=True)

    xvals = 0.5 * (xedges[1:] + xedges[:-1])
    yvals = 0.5 * (yedges[1:] + yedges[:-1])
    bw = (xedges[1] - xedges[0]) * (yedges[1] - yedges[0])

    bin_volumes=bw

    contour_values = calculate_contours(bin_fillings, bin_volumes, probs=contour_probs)

    ## reverse
    contour_values = contour_values[::-1]

    bounds=None

    try:
        ret = ax.contour(xvals,
                       yvals,
                       bin_fillings.T,
                       levels=contour_values,
                       colors=color)

        fmt_dict = dict()

        for ind, cprob in enumerate(contour_probs[::-1]):
          if(ind<len(contour_values)):
              fmt_dict[contour_values[ind]] = "%d" % (int(cprob * 100)) + r" %"

        ax.clabel(ret,
                fontsize=9,
                inline=1,
                fmt=fmt_dict,
                levels=contour_values,
                colors=color)

        bounds = get_bounds_from_contour(ret)

    except:
      print("Contour failed")
      return [[-1.0,1.0], [-1.0,1.0]]

    return [[bounds[0], bounds[1]], [bounds[2], bounds[3]]]

def plot_density_with_contours(ax,
                               log_evals,
                               evalpositions,
                               bin_volumes,
                               pts_per_dim,
                               color="black",
                               contour_probs=[0.68, 0.95]):

    xvals=evalpositions[:,0,0]
    yvals=evalpositions[0,:,1]

    #xr = (evalpositions[:, :, 0].min(), evalpositions[:, :, 0].max())
    #yr = (evalpositions[:, :, 1].min(), evalpositions[:, :, 1].max())

    xr = (xvals[0], xvals[-1])
    yr = (yvals[0], yvals[-1])

    contour_values = calculate_contours(numpy.exp(log_evals), bin_volumes,
                                        probs=contour_probs)

    ## reverse

    contour_values = contour_values[::-1]

    pcol_result = ax.pcolorfast(xr, yr, numpy.exp(log_evals).T)

    #contour_x = numpy.linspace(evalpositions[:, :, 0].min(),
    #                           evalpositions[:, :, 0].max(), pts_per_dim)
    #contour_y = numpy.linspace(evalpositions[:, :, 1].min(),
    #                           evalpositions[:, :, 1].max(), pts_per_dim)

    valid_contour=True

    if(len(contour_values)>=2):
      if(contour_values[0]==contour_values[1]):
        valid_contour=False

    if(len(contour_probs) != len(contour_values)):
      valid_contour=False

    if(valid_contour):
      contour_x=xvals
      contour_y=yvals

      bins = [contour_x, contour_y]

      res = ax.contour(contour_x,
                       contour_y,
                       numpy.exp(log_evals).T,
                       levels=contour_values,
                       colors=color)

      fmt_dict = dict()

      for ind, cprob in enumerate(contour_probs[::-1]):
          fmt_dict[contour_values[ind]] = "%d" % (int(cprob * 100)) + r" %"

      ax.clabel(res,
                fontsize=9,
                inline=1,
                fmt=fmt_dict,
                levels=contour_values,
                colors=color)

    pylab.colorbar(pcol_result, ax=ax)

def get_basic_gridlines():

  n_theta=5
  n_phi=10

  gridlines=[]

  for g in numpy.linspace(0.1,numpy.pi-0.1, n_theta):
    azis=torch.linspace(0,2*numpy.pi, 100)
    zens=torch.ones_like(azis)*g
    gl=torch.cat( [zens[:,None], azis[:,None]], dim=1)
    gridlines.append(gl)

  for a in numpy.linspace(0,2*numpy.pi-2*numpy.pi/n_phi, n_phi):
    zens=torch.linspace(0,numpy.pi,100)
    azis=torch.ones_like(zens)*a
    gl=torch.cat( [zens[:,None], azis[:,None]], dim=1)
    gridlines.append(gl)

  return gridlines

def plot_joint_pdf(pdf,
                   fig,
                   gridspec,
                   samples,
                   subgridspec=None,
                   conditional_input=None,
                   bounds=None,
                   multiplot=False,
                   total_pdf_eval_pts=10000,
                   true_values=None,
                   plot_only_contours=False,
                   contour_probs=[0.68, 0.95],
                   contour_color="white",
                   autoscale=True,
                   skip_plotting_density=False,
                   hide_labels=False,
                   s2_norm="standard",
                   colormap=cm.rainbow,
                   s2_rotate_to_true_value=True,
                   s2_show_gridlines=True,
                   skip_plotting_samples=False,
                   var_names=[],
                   relative_buffer=0.1,
                   vis_percentiles=[3.0,97.0],
                   show_relative_std=0,
                   use_outlier_binning=False,
                   **kwargs):

    plot_density = False
    dim = len(samples[0])
    if (pdf.total_target_dim == 1 and dim == pdf.total_target_dim):
        plot_density = True

    if (pdf.total_target_dim == 2 and dim == pdf.total_target_dim):
        plot_density = True

    if (conditional_input is not None):
        if(type(conditional_input)==list):
            if (conditional_input[0].shape[0] > 1):
                plot_density = False
        else:
            if (conditional_input.shape[0] > 1):
                plot_density = False

    if (skip_plotting_density):
        plot_density = False

    ##

    visualization_bounds, density_eval_bounds, histogram_edges=obtain_bins_and_visualization_regions(samples, pdf, percentiles=vis_percentiles, relative_buffer=relative_buffer, use_outlier_binning=use_outlier_binning, num_bins=50, s2_norm=s2_norm)

    if(bounds is not None):
        visualization_bounds=bounds

    ## true positions are typically labels
    plotted_true_values=None
    if(true_values is not None):
      plotted_true_values=copy.deepcopy(true_values)

    totalpts = total_pdf_eval_pts
    pts_per_dim = int(totalpts**(1.0 / float(dim)))

    samples = samples.cpu().clone()

    gridline_dict=None
    if(s2_show_gridlines and "s2" in pdf.pdf_defs_list):
      gridline_dict=dict()
      for ind, pdf_type in enumerate(pdf.pdf_defs_list):
        if(pdf_type=="s2"):
          gridline_dict[(pdf.target_dim_indices[ind][0], pdf.target_dim_indices[ind][1])]=get_basic_gridlines()

    ## transform samples to lambert space if necessary
    for ind, pdf_type in enumerate(pdf.pdf_defs_list):
        if (pdf_type == "s2" and s2_norm=="lambert"):
            ## transform to labmert space

            ## calculate fix point if rotation for visualtizion is desired
            fix_point=None

            if(s2_rotate_to_true_value and true_values is not None):

              fix_point=true_values[pdf.target_dim_indices[ind][0]:pdf.target_dim_indices[ind][1]]

            ## transform samples to lambert space
            samples[:,
                    pdf.target_dim_indices[ind][0]:pdf.target_dim_indices[ind]
                    [1]] = spherical_to_cartesian_lambert(
                        samples[:, pdf.target_dim_indices[ind][0]:pdf.
                                target_dim_indices[ind][1]], fix_point=fix_point)

            ## transform true value to lambert space
            if(plotted_true_values is not None):

                res=spherical_to_cartesian_lambert(true_values[pdf.target_dim_indices[ind][0]:pdf.
                target_dim_indices[ind][1]].unsqueeze(0), fix_point=fix_point)

                plotted_true_values[pdf.target_dim_indices[ind][0]:pdf.
                target_dim_indices[ind][1]]=res.squeeze(0)

            ## transform gridlines to lambert space
            if(s2_show_gridlines):
              tup=(pdf.target_dim_indices[ind][0],pdf.target_dim_indices[ind][1])

              new_list=[]

              for l in gridline_dict[tup]:
                new_list.append(spherical_to_cartesian_lambert(l, fix_point=fix_point))

              gridline_dict[tup]=new_list

    samples = samples.cpu().numpy()

    pdf_conditional_input = conditional_input

    if (pdf_conditional_input is not None):

        if(type(pdf_conditional_input)==list):
            pdf_conditional_input=[ci[0:1] for ci in pdf_conditional_input]
        else:
            pdf_conditional_input = pdf_conditional_input[0:1]

    evalpositions, log_evals, bin_volumes, sin_zen_mask, unreliable_spherical_regions= get_pdf_on_grid(
        density_eval_bounds,
        pts_per_dim,
        pdf,
        conditional_input=pdf_conditional_input,
        s2_norm=s2_norm,
        s2_rotate_to_true_value=s2_rotate_to_true_value,
        true_values=true_values)

    total_pdf_integral=numpy.exp(log_evals).sum()*bin_volumes

    if (dim == 1):

        if (subgridspec is None):

            subgridspec = gridspec.subgridspec(1, 1)
            ax = fig.add_subplot(subgridspec[0, 0])
            setattr(subgridspec, "axdict", {"ax": ax})

            ## find ax in existing gridspec
            """
            for ax in fig.get_axes():
                ax_geometry=ax.get_subplotspec().get_geometry()

                if(subgridspec!=ax.get_subplotspec().get_gridspec() ):
                    continue

                if(ax_geometry[0]==1 and ax_geometry[1]==1):
                    break
            """
        ax=subgridspec.axdict["ax"]

        ## attach/update "visualization_bounds" to subgridspec
        _update_attached_visualization_bounds(subgridspec, visualization_bounds)

        ax.hist(samples[:, 0], bins=histogram_edges[0], density=True)

        if (plot_density):

            ax.plot(evalpositions[:, 0], numpy.exp(log_evals), color="k")

        if (true_values is not None):
            ax.axvline(true_values[0].cpu().numpy(), color="red", lw=2.0)

        if (hide_labels):
            ax.set_yticklabels([])
            ax.set_xticklabels([])

        ax.set_xlim(subgridspec.visualization_bounds[0][0], subgridspec.visualization_bounds[0][1])

    elif (dim == 2 and multiplot == False):

        if (subgridspec is None):
            subgridspec = gridspec.subgridspec(1, 1)
            ax = fig.add_subplot(subgridspec[0, 0])
            setattr(subgridspec, "axdict", {"ax": ax})

        ax=subgridspec.axdict["ax"]

        _update_attached_visualization_bounds(subgridspec, visualization_bounds)

        if(plot_density):
            plot_density_with_contours(ax, log_evals, evalpositions,
                                     bin_volumes, pts_per_dim)

        ## plot a histogram density from samples

        if ( (plot_only_contours == False) and (plot_density == False) and (skip_plotting_samples==False)):

            ax.hist2d(samples[:, 0],
                      samples[:, 1],
                      bins=histogram_edges,
                      density=True)

        if (contour_probs != [] and skip_plotting_samples==False):

            _ = show_sample_contours(ax,
                                     samples,
                                     bins=histogram_edges,
                                     color=contour_color,
                                     contour_probs=contour_probs,
                                     sin_zen_mask=sin_zen_mask)

        ## mark poles
        if(len(unreliable_spherical_regions)>0):

          ax.plot(unreliable_spherical_regions[:,0], unreliable_spherical_regions[:,1], color="orange", marker="x", lw=0.0)

        ## plot true values
        if (plotted_true_values is not None):

            ax.plot([plotted_true_values[0].cpu().numpy()], [plotted_true_values[1].cpu().numpy()],
                    color="red",
                    marker="o",
                    ms=3.0)

        ## plot gridlines if desired
        if(s2_show_gridlines and gridline_dict is not None):

          for gl in gridline_dict[(0,2)]:
            np_gl=gl.cpu().numpy()

            ax.plot(np_gl.T[0], np_gl.T[1], color="gray", alpha=0.5)

        ax.set_xlim(subgridspec.visualization_bounds[0][0], subgridspec.visualization_bounds[0][1])
        ax.set_ylim(subgridspec.visualization_bounds[1][0], subgridspec.visualization_bounds[1][1])

        if (hide_labels):
            ax.set_yticklabels([])
            ax.set_xticklabels([])

    else:

        ###########
        if (subgridspec is None):

            subgridspec = gridspec.subgridspec(dim, dim)

            setattr(subgridspec, "axdict", {})

            for ind1 in range(dim):
                for ind2 in range(dim):
                    if(ind2>ind1):
                        continue

                    subgridspec.axdict[(ind1,ind2)]=fig.add_subplot(subgridspec[ind1, ind2])
                    ## make sure background looks similar to histogram empty bins

        ## attach/update "visualization_bounds" to subgridspec
        _update_attached_visualization_bounds(subgridspec, visualization_bounds)

        for ind1 in range(dim):
            for ind2 in range(dim):

                if (ind2 < ind1):

                    """
                    found_ax=False
                    for ax in fig.get_axes():

                        ax_geometry=ax.get_subplotspec().get_geometry()
                        print("checking ... ", ax_geometry)
                        num_rows=ax_geometry[0]
                        this_gridspec=ax.get_gridspec()

                        if(subgridspec!=ax.get_subplotspec().get_gridspec() ):
                            continue

                        if(ax_geometry[2]==(ind1*num_rows+ind2+1) and (num_rows==dim)):
                            found_ax=True

                            break

                    if(found_ax==False):
                        print("have not found in off diagonal ", ind1, ind2)
                        ax = fig.add_subplot(subgridspec[ind1, ind2])
                        print("added AX ", ax.get_subplotspec().get_geometry())
                    """

                    ax=subgridspec.axdict[(ind1,ind2)]

                    if (plot_only_contours == False):
                        print("plot histo")
                        ax.hist2d(samples[:, ind2],
                                  samples[:, ind1],
                                  bins=[histogram_edges[ind2], histogram_edges[ind1]],
                                  density=True,
                                  cmap=colormap,
                                  cmin=1e-20)

                    if (plotted_true_values is not None):
                        ax.plot([plotted_true_values[ind2].cpu().numpy()], [plotted_true_values[ind1].cpu().numpy()],
                                color="red",
                                marker="o",
                                ms=3.0)

                    new_samples = numpy.concatenate(
                        [samples[:, ind2:ind2 + 1], samples[:, ind1:ind1 + 1]],
                        axis=1)
                    print("IND 2", ind2, "ind 1 ", ind1)
                    if (contour_probs != []):
                        _ = show_sample_contours(
                            ax,
                            new_samples,
                            bins=[histogram_edges[ind2], histogram_edges[ind1]],
                            color=contour_color,
                            contour_probs=contour_probs)

                    ax.set_xlim(subgridspec.visualization_bounds[ind2][0], subgridspec.visualization_bounds[ind2][1])
                    ax.set_ylim(subgridspec.visualization_bounds[ind1][0], subgridspec.visualization_bounds[ind1][1])

                    ## always hide labels if left or bottom
                    if(ind2==0 and ind1<(dim-1)):
                        ax.set_xticklabels([])

                        fontsize = ax.get_window_extent().height/5.0
                        #for lab in ax.get_yticklabels():
                        #    lab.set_fontsize(fontsize)

                        if("labels" in kwargs):
                            assert(len(kwargs["labels"])==dim)

                            ax.set_ylabel(kwargs["labels"][ind1])

                    elif((ind1==(dim-1)) and (ind2!=0)):
                        ax.set_yticklabels([])

                        fontsize = ax.get_window_extent().width/5.0
                        for lab in ax.get_xticklabels():
                            #lab.set_fontsize(fontsize)
                            lab.set_rotation(45)

                        if("labels" in kwargs):
                            assert(len(kwargs["labels"])==dim)

                            ax.set_xlabel(kwargs["labels"][ind2])

                    elif(ind1<(dim-1) and ind2>0):
                        ax.set_yticklabels([])
                        ax.set_xticklabels([])

                    else:
                        ## lower left corner
                        fontsize = ax.get_window_extent().width/5.0
                        for lab in ax.get_xticklabels():
                            #lab.set_fontsize(fontsize)
                            lab.set_rotation(45)

                        fontsize = ax.get_window_extent().height/5.0
                        #for lab in ax.get_yticklabels():
                        #    lab.set_fontsize(fontsize)

                        if("labels" in kwargs):
                            assert(len(kwargs["labels"])==dim)

                            ax.set_xlabel(kwargs["labels"][ind2])
                            ax.set_ylabel(kwargs["labels"][ind1])

                    if (hide_labels):
                        ax.set_yticklabels([])
                        ax.set_xticklabels([])

                elif (ind2 == ind1):

                    ## looking for ax
                    """
                    found_ax=False
                    for ax in fig.get_axes():

                        ax_geometry=ax.get_subplotspec().get_geometry()
                        num_rows=ax_geometry[0]

                        if(subgridspec!=ax.get_subplotspec().get_gridspec() ):
                            continue

                        if(ax_geometry[2]==(ind1*num_rows+ind2+1) and (num_rows==dim)):
                            found_ax=True

                            break

                    if(found_ax==False):
                        print("have not found hte ax ....", ax, ind2, ind1)
                        ax = fig.add_subplot(subgridspec[ind1, ind2])

                        print("added AX ", ax.get_subplotspec().get_geometry())
                    """

                    ##############

                    ax=subgridspec.axdict[(ind1,ind2)]

                    color="black"
                    if("color_1d" in kwargs.keys()):
                        color=kwargs["color_1d"]
                    ax.hist(samples[:, ind1], bins=histogram_edges[ind1], histtype="step", density=True, color=color)

                    if(show_relative_std):
                        std=numpy.std(samples[:, ind1])
                        relative_wrt_axwidth=0.5*(visualization_bounds[ind2][1]-visualization_bounds[ind2][0])/std

                        ax.set_title("%.1f" % (relative_wrt_axwidth),fontsize=ax.get_window_extent().width/4.5)

                    if (plotted_true_values is not None):
                        ax.axvline(plotted_true_values[ind1].cpu().numpy(), color="red", lw=2.0)

                    ax.set_xlim(subgridspec.visualization_bounds[ind2][0], subgridspec.visualization_bounds[ind2][1])

                    # 1-d hists do not need y labels
                    ax.set_yticklabels([])

                    if (hide_labels or ind2 < (dim-1)):

                        ax.set_xticklabels([])
                    else:
                        fontsize = ax.get_window_extent().width/5.0
                        for lab in ax.get_xticklabels():
                            #lab.set_fontsize(fontsize)
                            lab.set_rotation(45)

                        if("labels" in kwargs):
                            assert(len(kwargs["labels"])==dim)

                            ax.set_xlabel(kwargs["labels"][ind2])

    return subgridspec, total_pdf_integral

def visualize_pdf(pdf,
                  fig,
                  gridspec=None,
                  subgridspec=None,
                  conditional_input=None,
                  nsamples=10000,
                  total_pdf_eval_pts=10000,
                  bounds=None,
                  true_values=None,
                  plot_only_contours=False,
                  contour_probs=[0.68, 0.95],
                  contour_color="white",
                  autoscale=True,
                  seed=None,
                  skip_plotting_density=False,
                  hide_labels=False,
                  s2_norm="standard",
                  colormap=cm.rainbow,
                  s2_rotate_to_true_value=True,
                  s2_show_gridlines=True,
                  skip_plotting_samples=False,
                  var_names=[],
                  num_iterative_steps=-1,
                  relative_vis_buffer=0.1,
                  vis_percentiles=[3.0, 97.0],
                  show_relative_std=0,
                  use_outlier_binning=False,
                  **kwargs
                  ):

    with torch.no_grad():
      sample_conditional_input = conditional_input
      if (conditional_input is not None):

        if(type(conditional_input)==list):

            sample_conditional_input=[]

            for inp_ind, ci in enumerate(conditional_input):

                this_ci=ci

                if (len(this_ci.shape) == 1):
                    this_ci=this_ci.unsqueeze(0)

                if (this_ci.shape[0] == 1):
                    this_ci = this_ci.repeat(nsamples, 1)

                sample_conditional_input.append(this_ci)

            ci_batch_size=sample_conditional_input[0].shape[0]

        else:

            if (len(conditional_input.shape) == 1):
                sample_conditional_input = sample_conditional_input.unsqueeze(0)

            if (sample_conditional_input.shape[0] == 1):
                sample_conditional_input = sample_conditional_input.repeat(nsamples, 1)

            ci_batch_size=sample_conditional_input.shape[0]

      if (gridspec is None):
          gridspec = fig.add_gridspec(1, 1)[0, 0]

      if(sample_conditional_input is not None):

        if(num_iterative_steps>0):

            assert(ci_batch_size%num_iterative_steps==0), "Number of total sample points must be divisible by iterative steps!"

            num_per_step=int(ci_batch_size/num_iterative_steps)
            samples=[]

            for cur_step in range(num_iterative_steps):

                if(type(sample_conditional_input)==list):
                    this_sample_input=[si[cur_step*num_per_step:cur_step*num_per_step+num_per_step] for si in sample_conditional_input]

                    cur_samples, _, _, _ = pdf.sample(
                      conditional_input=this_sample_input,
                      seed=seed+cur_step if seed is not None else None,
                      force_intrinsic_coordinates=True)
                else:
                    cur_samples, _, _, _ = pdf.sample(
                      conditional_input=sample_conditional_input[cur_step*num_per_step:cur_step*num_per_step+num_per_step],
                      seed=seed+cur_step if seed is not None else None,
                      force_intrinsic_coordinates=True)

                samples.append(cur_samples)

            samples=torch.cat(samples, dim=0)

        else:

            samples, _, _, _ = pdf.sample(
              conditional_input=sample_conditional_input,
              seed=seed,
              force_intrinsic_coordinates=True)
      else:

        # check if PDF involves only a "x" flow (which does not nothing.. enforce device then)
        used_dtype, used_device=pdf.obtain_current_dtype_n_device()

        if(used_dtype is None):
            if("dtype" in kwargs):
                used_dtype=kwargs["dtype"]

        if(used_device is None):
            if("device" in kwargs):
                used_device=kwargs["device"]

        if(num_iterative_steps>0):
            assert(nsamples%num_iterative_steps==0), "Number of total sample points must be divisible by iterative steps!"

            num_per_step=int(nsamples/num_iterative_steps)
            samples=[]

            for cur_step in range(num_iterative_steps):

                cur_samples, _, _, _ = pdf.sample(
                  samplesize=num_per_step,
                  seed=seed+cur_step if seed is not None else None, device=used_device, dtype=used_dtype, force_intrinsic_coordinates=True)

                samples.append(cur_samples)

            samples=torch.cat(samples, dim=0)

        else:

          samples, samples_base, evals, evals_base = pdf.sample(
              samplesize=nsamples,
              seed=seed,
              force_intrinsic_coordinates=True,
              device=used_device,
              dtype=used_dtype)

      higher_dim_spheres = False

      new_subgridspec, total_pdf_integral = plot_joint_pdf(
          pdf,
          fig,
          gridspec,
          samples,
          subgridspec=subgridspec,
          conditional_input=conditional_input,
          bounds=bounds,
          multiplot=False,
          total_pdf_eval_pts=total_pdf_eval_pts,
          true_values=true_values,
          plot_only_contours=plot_only_contours,
          contour_probs=contour_probs,
          contour_color=contour_color,
          autoscale=autoscale,
          skip_plotting_density=skip_plotting_density,
          hide_labels=hide_labels,
          s2_norm=s2_norm,
          colormap=colormap,
          s2_rotate_to_true_value=s2_rotate_to_true_value,
          s2_show_gridlines=s2_show_gridlines,
          skip_plotting_samples=skip_plotting_samples,
          var_names=var_names,
          relative_buffer=relative_vis_buffer,
          vis_percentiles=vis_percentiles,
          show_relative_std=show_relative_std,
          use_outlier_binning=use_outlier_binning,
          **kwargs)

    return samples, new_subgridspec, total_pdf_integral


from collections import deque
from heapq import heappop, nlargest, nsmallest, heapify

# unpacking a sequence into separate variables
# using _
data = ['wang', 21, (2001, 4, 1), 71, 180]
name, age, brithday, _, _ = data
print(name, age, brithday)

# unpacking elements from iterables of arbitrary length
# using *, *_
name, age, *other = data
print(name, other)
name, *_ = data
vals = [1, 2, 3, 4, 5, 6, 7]

def sum_(vals: list[int]):
    head, *tail = vals
    return head + sum_(tail) if tail else head

print(sum_(vals))

# keeping the last n items
def search(lines, pattern, history=5):
    previous_lines = deque(maxlen=history)
    for line in lines:
        if pattern in line:
            yield line, previous_lines
        previous_lines.append(line)

with open(r'/tmp/base.vim') as f:
    for line, previous_lines in search(f, 'highlight', 1):
        for pline in previous_lines:
            print(pline, end=' ')
        print(line, end=' ')
        print('-'*20)

# finding the largest or smallest n items
nums = [1, 8, 2, 23, 7, -4, 18, 23, 42, 37, 2]
print(nlargest(3, nums))  # Prints [42, 37, 23]
print(nsmallest(3, nums))  # Prints [-4, 1, 2]
portfolio = [
    {'name': 'IBM', 'shares': 100, 'price': 91.1},
    {'name': 'AAPL', 'shares': 50, 'price': 543.22},
    {'name': 'FB', 'shares': 200, 'price': 21.09},
    {'name': 'HPQ', 'shares': 35, 'price': 31.75},
    {'name': 'YHOO', 'shares': 45, 'price': 16.35},
    {'name': 'ACME', 'shares': 75, 'price': 115.65}
]
cheap = nsmallest(3, portfolio, key=lambda s: s['price'])
expensive = nlargest(3, portfolio, key=lambda s: s['price'])
print(cheap)
print(expensive)
heapify(nums)   # 堆排序， 优先队列
print(nums)
n = len(nums)
for i in range(n):
    print(heappop(nums))