import tweepy, os
from dotenv import load_dotenv

#read in and set env variables
load_dotenv()
APPKEY = os.environ.get('TWITTERAPPKEY')
APPSECRET = os.environ.get('TWITTERAPPSECRET')
ACCOUNTKEY = os.environ.get('BOTACCOUNTKEY')
ACCOUNTSECRET = os.environ.get('BOTACCOUNTSECRET')

f = open("../DownloadTest/info.txt", "r")
lyric, style = f.readline().strip(), f.readline().strip()

# auth setup for 1.1 endpoint
auth = tweepy.OAuthHandler(APPKEY, APPSECRET)
auth.set_access_token(ACCOUNTKEY, ACCOUNTSECRET)
helper=tweepy.API(auth)
img = helper.simple_upload('../DownloadTest/swin/step-024_SwinIR_large.png')
print(img)
art_id = img.media_id_string
vid = helper.media_upload('../DownloadTest/process.mp4', wait_for_async_finalize=True)
print(vid)
vid_id=vid.media_id_string

bot = tweepy.Client(consumer_key = APPKEY, consumer_secret = APPSECRET,
                    access_token = ACCOUNTKEY, access_token_secret = ACCOUNTSECRET)
art_tweet = bot.create_tweet(text=lyric, media_ids=[art_id])
print(art_tweet)
art_tweet_id = art_tweet.data.get('id')
vid_tweet = bot.create_tweet(media_ids=[vid_id], in_reply_to_tweet_id = art_tweet_id)
print(vid_tweet)


import numpy as np
from collections import Counter, defaultdict

class KNN:
    # KNN 클래스 생성, k의 기본값 3으로 설정
    def __init__(self, k=3):
        self.k = k

    # 두 점 사이의 유클리디안 거리를 계산해주는 함수
    def euclidean_distance(self, x1, x2):
        return np.sqrt(np.sum((x1 - x2) ** 2))

    # 주어진 테스트 데이터에 대한 레이블을 예측하여 반환하는 함수
    def predict(self, X_train, y_train, test_data):
        # 각 훈련 데이터 포인트와의 거리를 계산
        distance = [(self.euclidean_distance(test_data, x), y) for x, y in zip(X_train, y_train)]
        # 거리를 기준으로 정렬
        sorted_distance = sorted(distance, key=lambda x: x[0])
        # 가장 가까운 k개의 레이블 찾기
        k_nearest_label = [item[1] for item in sorted_distance[:self.k]]
        # 이 중에서 가장 많은 레이블을 예측값으로 반환
        most = Counter(k_nearest_label).most_common(1)
        return most[0][0]

    # 가중치를 사용하여 테스트 데이터에 대한 레이블을 예측하여 반환하는 함수
    def weighted_predict(self, X_train, y_train, test_data):
        # 테스트 포인트와 모든 훈련 데이터 포인트 사이의 거리 계산
        distance = [(self.euclidean_distance(test_data, x), y) for x, y in zip(X_train, y_train)]
        # 거리를 기준으로 정렬
        sorted_distance = sorted(distance, key=lambda x: x[0])
        # 가장 가까운 k개의 데이터 포인트 선택
        k_nearest = sorted_distance[:self.k]
        vote = defaultdict(float)
        for distance, label in k_nearest:
            # 거리가 0인 경우, 해당 레이블을 직접 반환
            if distance == 0:
                return label
            # 가중치를 계산 (거리의 역수)
            weight = 1 / distance
            vote[label] += weight

        # 가장 높은 가중치를 가진 레이블 반환
        return max(vote, key=vote.get)


import numpy as np
import ruamel.yaml
from sdk.slm_sdk import SLMsdk
from utils.prairie_interface import PrairieInterface
from utils.parse_markpoints import ParseMarkpoints
from utils.utils_funcs import load_mat_file, tangent
import sys
import os
from datetime import datetime
import time
from pathlib import Path
import experiments
import matlab.engine

class Blimp(SLMsdk, PrairieInterface, ParseMarkpoints):

    def __init__(self):
        '''
        interface between pycontrol, prarie view software and
        medowlark slm
        '''
        # alows stopping and starting in the pycontrol gui
        if not hasattr(self, '_im_init'):

            self.time_now = datetime.now().strftime('%Y-%m-%d-%H%M%S')
            self.assign_from_yaml()

            #init inherited classes
            self.sdk = SLMsdk()
            self.prairie = PrairieInterface()

            # connect to the SLM
            try:
                self.sdk.SLM_connect()
            except:
                print('close other SLM connections')
                time.sleep(5)
                raise

            print('starting matlab engine')
            self.eng = matlab.engine.start_matlab()
            print('matlab engine initialised')
            # the numbers of the SLM trials produced by pycontrol (error in task if not continous list of ints)
            self.SLM_tnums= []
            # the barcodes of the SLM trials
            self.SLM_barcodes = []
            # the times of the SLM trials
            self.SLM_times = []
            self._run_time_prev = 0

            #get the experiment function defined in the yaml
            try:
                self.experiment_class = getattr(experiments, self.yaml_dict['experiment'])
            except:
                raise Exception('Could not find experiment defined  in yaml')
            # inits the experiment class with and instance of the Blimp class (this is probably a horrible way of doing this)
            self.experiment = self.experiment_class(self)

            #set the uncaging laser power to 0 and open the uncaging shutter
            self.prairie.pl.SendScriptCommands('-SetLaserPower Uncaging 0')
            time.sleep(1)
            self.prairie.pl.SendScriptCommands('-OverrideHardShutter Uncaging open')
            print('\n\nDID YOU TYPE IN THE ANIMALS NAME INTO THE GUI????????????????')
            # dont init the class more than once
            self._im_init = True

    @property
    def yaml_dict(self):
        #load yaml
        _base_path = os.path.dirname(__file__)
        _yaml_path = os.path.join(_base_path, "blimp_settings.yaml")

        #the ruamel module preserves the comments and order of the yaml file
        self._yaml = ruamel.yaml.YAML()

        with open(_yaml_path, 'r') as stream:
            return self._yaml.load(stream)

    def assign_from_yaml(self):

        '''get attrs and paths from te blimp_settings.yaml file'''
        self.group_size = self.yaml_dict['group_size']
        self.duration = self.yaml_dict['duration']
        self.num_spirals = self.yaml_dict['num_spirals']
        self.spiral_revolutions = self.yaml_dict['spiral_revolutions']
        self.mWperCell = self.yaml_dict['mWperCell']
        self.inter_group_interval = self.yaml_dict['inter_group_interval']
        self.num_repeats = self.yaml_dict['num_repeats']
        #calculate spiral size as 0-1 ratio of fov size
        self.spiral_size = self.yaml_dict['spiral_size'] / (self.yaml_dict['FOVsize_UM_1x'] / self.yaml_dict['zoom'])
        self.naparm_path = self.yaml_dict['naparm_path']
        self._output_path = self.yaml_dict['output_path']

        self.output_folder = os.path.join(self._output_path, self.time_now)
        if not os.path.exists(self.output_folder):
            os.makedirs(self.output_folder)

        # save the current settings yaml into the todays folder
        with open(os.path.join(self.output_folder, '{}_blimp_settings.yaml'.format(self.time_now)), 'w+') as f:
            self._yaml.dump(self.yaml_dict, f)

    def mw2pv(self, x):
        '''returns the PV value required for specific PV value based on DigitalPowerMeasurments notebook fitting'''
        _popt_path = self.yaml_dict['popt_path']
        _popt = np.load(_popt_path)

        return(tangent(x, *_popt))

    def update(self, new_data, _run_time):

        ''' called every time there is a new print, state or event in pycontrol '''

        #all prints that occur from the board
        _pycontrol_print = [nd for nd in new_data if nd[0] == 'P']

        if _run_time < self._run_time_prev:
            self.__init__()

        self._run_time_prev = _run_time
        # break function if no new print statement is found
        if _pycontrol_print:
            _pycontrol_print = _pycontrol_print[0][2]
        else:
            return

        # an SLM trial is initiated
        if 'Trigger SLM trial' in _pycontrol_print:

            print('begininng SLM trial')
            self.slm_print = _pycontrol_print
            self.trial_runtime = round(_run_time,4) # dont need more than ms precision
            # search the SLM trial string for the number and barcode
            _space_split = _pycontrol_print.split(' ')
            self.trial_number = [_space_split[i+1] for i,word in enumerate(_space_split) if word == 'Number'][0]
            self.barcode = [_space_split[i+1] for i,word in enumerate(_space_split) if word == 'Barcode'][0]

            #append to the alignment lists
            self.SLM_tnums.append(self.trial_number)
            self.SLM_barcodes.append(self.barcode)
            self.SLM_times.append(self.trial_runtime)

            # begin SLM trial
            self.experiment.slm_trial()

        elif 'Trigger NOGO trial' in _pycontrol_print:
            print('beginning nogo trial')
            self.trial_runtime = round(_run_time,4) # dont need more than ms precision
            _space_split = _pycontrol_print.split(' ')

            self.trial_number = [_space_split[i+1] for i,word in enumerate(_space_split) if word == 'Number'][0]
            self.barcode = [_space_split[i+1] for i,word in enumerate(_space_split) if word == 'Barcode'][0]

            self.experiment.nogo_trial()

    def update_test(self):
        '''development function to test the update function called from pycontrol'''
        self.trial_runtime = 1
        self.trial_number = 1
        self.barcode = 1
        self.experiment.run_experiment()

    def write_output(self, time_stamp=None, trial_number=None, barcode=None, info=None):

        #the txtfile to write alignent information to
        self.txtfile = os.path.join(self.output_folder, 'blimpAlignment.txt')
        with open(self.txtfile, 'a') as f:
            f.write('Time stamp: {0}. Trial Number {1}. Barcode {2}. Info: {3} \n'.format(time_stamp, trial_number, barcode, info))

if __name__ == '__main__':
    Blimp()


from lxml import html
import requests
from time import sleep
import json
import argparse
from collections import OrderedDict
from time import sleep

class Scraper_Manager:

    def dow_scrape(self):
        url = "https://finance.yahoo.com/quote/%5EDJI/"
        response = requests.get(url, verify=False)
        parser = html.fromstring(response.text)
        summary_table = parser.xpath('// *[ @ id = "quote-header-info"] / div[3] / div / span')
        # print(summary_table[0].text)
        try:
            # print("break")
            return summary_table[0].text
        except:
            print("Failed to parse json response")
            return {"error": "Failed to parse json response"}

    def volume_scrape(self, ticker):
        url = "http://finance.yahoo.com/quote/%s?p=%s" % (ticker, ticker)
        response = requests.get(url, verify=False)
        # print("Parsing %s" % (url))
        # sleep(2)
        parser = html.fromstring(response.text)
        # print(response.text)
        summary_table = parser.xpath('//div[contains(@data-test,"summary-table")]//tr')
        summary_data = []
        try:
            for table_data in summary_table:
                raw_table_key = table_data.xpath('.//td[contains(@class,"C(black)")]//text()')
                raw_table_value = table_data.xpath('.//td[contains(@class,"Ta(end)")]//text()')
                if(raw_table_key[0] == 'Volume'):
                    table_value = ''.join(raw_table_value).strip()
                    summary_data.append(table_value)
                if (raw_table_key[0] == 'Avg. Volume'):
                    table_value = ''.join(raw_table_value).strip()
                    summary_data.append(table_value)
            return summary_data
        except:
            print("Failed to parse json response")
            return {"error": "Failed to parse json response"}

    def industry_scrape(self, ticker):
        url = "https://finance.yahoo.com/quote/%s/profile?ltr=1" % (ticker)
        response = requests.get(url, verify=False)
        # sleep(0)
        parser = html.fromstring(response.text)
        summary_table = parser.xpath('//div[contains(@class, "asset-profile-container")] / div / div / p[2] / span[4]')
        # print(summary_table[0].text)
        return summary_table[0].text


# -*- coding: utf-8 -*-
"""
Created on Tue Dec 19 14:38:20 2017

@author: gelina
"""

import csv
import datetime
import urllib.request
import urllib
import json
import math
import ast

def valid_parstation_parjour(FICHIER):
	"""
     Compte le nombre de validations total pour n'importe quel jour
     donné en une sation donnée

     Args:
         le fichier csv des données

     Returns:
         dictionaire des données

	 >>> d = valid_parstation_parjour('validations.csv'):
     >>> d['2017-05-10']['LES HALLES']
	 41413
     """
	with open(FICHIER, 'r') as f:
		r = csv.reader(f,delimiter=';')
		l = list(r) # l'itérable est converti en liste

		d = dict() # on costruit un dictionnaire qui mettra en relation un jour et toutes les données de ce jour
		for line in l[1:]:
		     if(line[0] in d):
		             d[line[0]]+= [[line[4],line[6],line[7]]] #on ajoute si la date est deja dans le dictionaire
		     else:
		             d[line[0]]= [[line[4],line[6],line[7]]] # on crée l'entrée sinon

		for date in d.keys():
		     s = dict() # on crée un nouveau dictionaire par jour qui assosiera les stations et leur nbre de visiteur
		     for line in d[date]:
		         if(line[0] in s):
		                 if (line[2] == 'Moins de 5'):
		                 	s[line[0]]+=4
		                 	# le STIF met la mention moins de 5 pour des raisons d'anonymat
		                 else:
		                 	s[line[0]]+=int(line[2]) #on ajoute si le dictionnaire contient deja la station pour ce jour
		         else:
		                 if (line[2] == 'Moins de 5'):
		                 	s[line[0]]=4
		                 	# le STIF met la mention moins de 5 pour des raisons d'anonymat
		                 else:
		                 	s[line[0]]=int(line[2]) # on definit la station dans le dictionnaire sinon
		     d[date] = s
		return d

def weekdaydetection(dico):
	"""
     sépare le jeu de donnée en 2 : les jours de la semaine et les autres

     Args:
         le dictionaire créé par la fonction valid_parstation_parjour

     Returns:
         liste de 2 dictionaires des données
         le 1er element (l[0]) de la liste est les jours de le semanine
         le 2eme (l[1]) les jours en weekend

	 >>> d = valid_parstation_parjour('validations.csv'):
     >>> l = weekdaydetection(d)
	 >>> '2017-01-01' in l[0]
	 False
	 >>> '2017-01-02' in l[1]
	 False
	 >>> '2017-01-03' in l[0]
	 True
	 >>> '2017-02-12' in l[1]
	 True

     """
	l = list()
	weekday = dict()
	weekend = dict()
	#On crée ici deux dictionaires et une liste
	# Les deux dictionaires contiennent soit les elements des jours qui tombent pendant une semeine ou ceux qui tombent un weekend

	for day in dico.keys():
		dayofweek = datetime.date(int(day[0:4]),int(day[5:7]),int(day[8:10])).isoweekday()
		# pour chaque date on va extraire de la string l'année, le mois et le jour
		# on construit avec cela un objet date et on appelle son attribut isoweekday ( pour lundi, 7 pour dimanche)

		if dayofweek < 6:
			weekday[day] = dico[day]
			#si c'est un jour de semaine, le ranger dans le dictionnaire semaine

		else :
			weekend[day] = dico[day]
			# si c'est un jour de WE , le ranger dans le dico weekend

	l.append(weekday)
	l.append(weekend)

	#on met les deux dico dans l et on retourne la liste
	return l

def moyennesurannee(dico):

	moyennesta = dict()
	for day in dico.keys():
		for station in dico[day].keys():
			if station in moyennesta:
				moyennesta[station] += dico[day][station]
			else:
				moyennesta[station] = dico[day][station]

	for station in moyennesta.keys():
		moyennesta[station] = math.ceil(moyennesta[station]/len(dico.keys()))

	return moyennesta

def split_hist_data(moyennesta, limite):
	l = list()
	moy1 = dict()
	moy2 = dict()
	l=[moy1]+[moy2]
	for data in moyennesta.keys():
		if moyennesta[data] < limite :
			moy1[data] = moyennesta[data]
		else:
			moy2[data] = moyennesta[data]
	return l

def build_stations_coordonates(filegeo):

	with open(filegeo, 'r') as f:
		r = csv.reader(f,delimiter=';')
		l = list(r) # l'itérable est converti en liste
		geos = dict()
		for station in l[1:]:
			lat = ast.literal_eval(station[1])['coordinates'][0]
			longi = ast.literal_eval(station[1])['coordinates'][1]
			geos[station[7]] = [lat, longi]
		return geos

def build_map_data(geostation,moyennesta):
	# construit un dictionaire associant un tableau [lon, lat] avec le nombre de visiteurs
	# cela est fait pour chaque station

	mapdata = list()
	for station in moyennesta.keys():
		data = moyennesta[station]
		if station in geostation.keys():

			geo = geostation[station]
			mapdata.append([data,geo])
	return mapdata


import speech_recognition
from gtts import gTTS
from pygame import mixer

def listen():
    """
    The listen function uses the speech_recognition class which is based on Google's
    dictation (speech recognition) API to convert audio strings into text.   If the
    function cannot process the message, it throws class-defined errors.

    :return: The search query from the user
    """

    recognizer = speech_recognition.Recognizer()
    with speech_recognition.Microphone() as source:
        audio_input = recognizer.listen(source)
        try:
            data = recognizer.recognize_google(audio_input)
            return data
        except speech_recognition.UnknownValueError:
            generate_audio("I didn't catch what you said.")
        except speech_recognition.RequestError as error:
            generate_audio("I'm having trouble connecting: {}.".format(error))
    return False

    # temporary input in audioless space
    # search_query = input()
    # return search_query

# filename for output
filename = "query_response.mp3"

def generate_audio(string):
    """
    The generate function utilizes Google's Text-to-Speech API to turn strings into audio
    messages that are played back to the user.  The function also utilizes Pygame, a class
    that contains the functionality (via pygame.mixer) to play stored .mp3 files.  The TTS
    stores the .mp3 result into the directory and then pygame.mixer plays it.

    :param string: The string output for TTS
    """

    audio = gTTS(string)
    audio.save("../audio_files/" + filename)

    mixer.init()
    mixer.music.load("../audio_files/" + filename)
    mixer.music.play()

    # temporary output in audioless space
    # print(string)


import numpy as np
import pandas as pd
import xarray as xr
import scipy as sp

import json
import matplotlib.pyplot as plt
import os
from astropy.convolution import convolve, Box1DKernel, Gaussian1DKernel
import astropy.units as u
from scipy import optimize
import glob

from matplotlib.ticker import StrMethodFormatter

import virga.justdoit as vj
from .justdoit import inputs, opannection, mean_regrid, u, input_xarray, copy

from bokeh.palettes import Cividis
from multiprocessing import Pool

import dynesty
from dynesty import utils as dyfunc

class GridFitter():
    """
    Top level grid fitter

    Currently our GridFitter has these requirements for xarray models:

    Required `coords`:
    - wavelength
    - pressure

    Required `data_vars`:
    - transit_depth

    Parameters
    ----------
    grid_name : str
        Grid name so that users can keep track of inputs

    model_dir : str
        Location of model grid. Should be a directory that points to
        several files in the PICASO xarray format.
    to_fit : str
        parameter to fit, default is transit_depth. other common is flux
    """
    def __init__(self, grid_name, model_dir,to_fit='transit_depth', grid_dimensions=False, verbose=True):
        self.verbose=verbose

        self.grids = []
        self.list_of_files = {}
        self.grid_params = {}
        self.overview = {}
        self.wavelength={}
        self.temperature={}
        self.pressure={}
        self.spectra={}

        #adds first grid
        self.add_grid(grid_name, model_dir, to_fit=to_fit)

    def find_grid(self, grid_name, model_dir):
        """
        Makes sure the grid exists with proper nc files. Then, adds the file directory to self.list_of_files

        """
        if not os.path.isdir(model_dir):
            raise Exception(f'Path to models entered does not exist: {model_dir}')
        else:
            self.list_of_files[grid_name] = glob.glob(os.path.join(model_dir,"*.nc"))
            nfiles = len(self.list_of_files[grid_name])
            if nfiles<=1:
                raise Exception("Oops! It looks like you only have 1 or less files with the extension '.nc'")
            else:
                if self.verbose: print(f'Total number of models in grid is {nfiles}')

    def add_grid(self,grid_name,model_dir,to_fit='transit_depth'):
        #loads in grid info
        self.grids += [grid_name]
        self.find_grid(grid_name, model_dir)
        self.load_grid_params(grid_name,to_fit=to_fit)

    def add_data(self, data_name, wlgrid_center,wlgrid_width,y_data,e_data):
        """
        Adds data to class

        Parameters
        ----------
        data_name : str
            Create a distinguisher for the dataset to test
        wlgrid_center : array
            array of wavelength centers
        wlgrid_width : array
            array of wavelength bins
        y_data : array
            data to be comapred to spectrum. CautioN!! make sure that y_data and material pulled to spectra are the same units.
        e_data : array
            measurement error associated y_data
        """
        self.data =  getattr(self, 'data',{data_name: []})
        self.data[data_name] = {'wlgrid_center': wlgrid_center,
                                'wlgrid_width':wlgrid_width,
                                 'y_data':y_data,
                                 'e_data':e_data}
    def as_dict(self):
        """
        get class in dictionary form to easily search
        """
        return {
        'list_of_files':self.list_of_files,
        'spectra_w_offset':self.best_fits,
        'rank_order':self.rank,
        'grid_params':self.grid_params,
        'offsets': getattr(self, 'offsets',0), #,
        'chi_sqs': self.chi_sqs,
        'posteriors': self.posteriors
        }

    def load_grid_params(self,grid_name,to_fit='transit_depth'):
        """
        This will read the grid parameters and set the array of parameters

        Parameters
        ----------
        grid_name : str
            Name of grid for bookkeeping
        to_fit : str
            Default is transit_depth but also could be flux or any other xarray parameter
            you are interested in fitting.

        Returns
        -------
        None
            Creates self.overview, and self.grid_params
        """
        possible_params = {'planet_params': ['rp','mp','tint', 'heat_redis','p_reference','logkzz','mh','cto','p_quench','rainout','teff','logg','m_length'],
                           'stellar_params' : ['rs','logg','steff','feh','ms'],
                           'cld_params': ['opd','ssa','asy','p_cloud','haze_eff','fsed']}

        #define possible grid parameters
        self.grid_params[grid_name] = {i:{j:np.array([]) for j in possible_params[i]} for i in possible_params.keys()}
        #define possible grid parameters
        self.overview[grid_name] = {i:{j:np.array([]) for j in possible_params[i]} for i in possible_params.keys()}

        #how many possible files
        number_files = len(self.list_of_files[grid_name])

        #loop through grid files to get parameters
        for ct, filename in enumerate(self.list_of_files[grid_name]):

            ds = xr.open_dataset(filename)

            if ct == 0:
                nwave = len(ds['wavelength'].values)
                npress = len(ds['pressure'].values)

                spectra_grid = np.zeros(shape=(number_files,nwave))
                temperature_grid = np.zeros(shape=(number_files,npress))
                pressure_grid = np.zeros(shape=(number_files,npress))
                wavelength = ds['wavelength'].values

            #start filling out grid parameters
            #seems like we need to save these?????
            temperature_grid[ct,:] = ds['temperature'].values[:]
            pressure_grid[ct,:] = ds['pressure'].values[:]
            spectra_grid[ct,:] = ds[to_fit].values[:]

            # Read all the paramaters in the Xarray so that User can gain insight into the
            # grid parameters
            for iattr in possible_params.keys():#loops through e.g. planet_params, stellar_params,
                if iattr in ds.attrs:
                    attr_dict = json.loads(ds.attrs[iattr])
                    for ikey in possible_params[iattr]:

                        self.grid_params[grid_name][iattr][ikey] = np.append(
                                                        self.grid_params[grid_name][iattr][ikey],
                                                        _get_xarray_attr(attr_dict, ikey))

        #now count how many are in each and pop
        for iattr in possible_params.keys():#loops through e.g. planet_params, stellar_params,
            if iattr in ds.attrs:
                for ikey in possible_params[iattr]:
                    uni_values = np.unique(self.grid_params[grid_name][iattr][ikey])
                    if (len(uni_values)==1):
                        if ('not specified' in str(uni_values[0])):
                            self.overview[grid_name][iattr][ikey] = 'Not specified in attrs.'
                            self.grid_params[grid_name][iattr].pop(ikey)
                        elif ('None' in str(uni_values[0])):
                            self.overview[grid_name][iattr][ikey] = f'Not used in grid.'
                            self.grid_params[grid_name][iattr].pop(ikey)
                        else:
                            self.overview[grid_name][iattr][ikey] = uni_values[0]
                            self.grid_params[grid_name][iattr].pop(ikey)
                    else:
                        self.overview[grid_name][iattr][ikey] = uni_values

            else:
                #e.g. if no stellar_params were included for a brown dwarf grid
                self.overview[grid_name].pop(iattr)
                self.grid_params[grid_name].pop(iattr)

        #for iattr in possible_params.keys():
        #    if len(self.grid_params[grid_name][iattr].keys())==0:
        #        self.grid_params[grid_name].pop(iattr)

        cnt_params = 0
        for iattr in self.overview[grid_name].keys():#loops through e.g. planet_params, stellar_params,
            for ikey in   self.overview[grid_name][iattr]:
                if isinstance(self.overview[grid_name][iattr][ikey],np.ndarray):
                    cnt_params += 1
                    if self.verbose:
                        print(f'For {ikey} in {iattr} grid is: {self.overview[grid_name][iattr][ikey]}')

        self.overview[grid_name]['num_params'] = cnt_params

        #lastly save wavelength, temperature, spectra
        self.wavelength[grid_name] = wavelength
        self.pressure[grid_name] = pressure_grid
        self.temperature[grid_name] = temperature_grid
        self.spectra[grid_name] = spectra_grid

    def fit_all(self):
        for i in self.grids:
            for j in self.data.keys() :
                self.fit_grid(i, j)

    def fit_grid(self,grid_name, data_name,dof='ndata', offset=True):
        """
        Fits grids given model and data. Retrieves posteriors of fit parameters.

        Parameters
        ----------
        grid_name : str
            grid name that was specified in GridFiter or add_grid
        data_name : str
            data name that was specified before in add_data
        dof : str
            used for chi square. if dof=='ndata' then chi square is computed as chi2/len(data).
            otherwise it computes as chi2/(len(data) - numparameters)
        offset : bool
            Fit for an offset (e.g. in transit spectra)

        To Dos
        ------
        - make general to fpfs_thermal
        """
        #number of models
        nmodels = len(self.list_of_files[grid_name])

        wlgrid_center = self.data[data_name]['wlgrid_center']
        y_data = self.data[data_name]['y_data']
        e_data = self.data[data_name]['e_data']

        #get chi_sqrs if it already exists
        self.chi_sqs =  getattr(self, 'chi_sqs',{grid_name: {data_name:np.zeros(shape=(nmodels))}})
        #get best fit dicts if it already exists
        self.best_fits =  getattr(self, 'best_fits',{grid_name:{data_name:np.zeros(shape=(nmodels,len(wlgrid_center)))}})
        #get rank order
        self.rank =  getattr(self, 'rank',{grid_name:{data_name:np.zeros(shape=(nmodels))}})

        #get posetiors
        self.posteriors =  getattr(self, 'posteriors',{grid_name:{data_name:{}}})

        #make sure nothing exiting is overwritten
        self.chi_sqs[grid_name] = self.chi_sqs.get(grid_name, {data_name:np.zeros(shape=(nmodels))})
        self.best_fits[grid_name] = self.best_fits.get(grid_name, {data_name:np.zeros(shape=(nmodels,len(wlgrid_center)))})
        self.rank[grid_name] = self.rank.get(grid_name, {data_name:np.zeros(shape=(nmodels))})
        self.posteriors[grid_name] = self.posteriors.get(grid_name, {data_name:{}})

        #make sure nothing existing is overwritten
        self.chi_sqs[grid_name][data_name] = self.chi_sqs[grid_name].get(data_name, np.zeros(shape=(nmodels)))
        self.best_fits[grid_name][data_name]  = self.best_fits[grid_name].get(data_name,np.zeros(shape=(nmodels,len(wlgrid_center))))
        self.rank[grid_name][data_name]  = self.rank[grid_name].get(data_name,np.zeros(shape=(nmodels)))
        self.posteriors[grid_name][data_name]  = self.posteriors[grid_name].get(data_name,{})

        if offset:

            self.offsets =  getattr(self, 'offsets',{grid_name:{data_name:np.zeros(nmodels) }})
            self.offsets[grid_name] = self.offsets.get(grid_name, {data_name:np.zeros(shape=(nmodels))})
            self.offsets[grid_name][data_name] = self.offsets.get(data_name,np.zeros(shape=(nmodels)))
            #self.overview[grid_name]['num_params'] = self.overview[grid_name]['num_params'] + 1

        numparams = self.overview[grid_name]['num_params']

        def shift_spectrum(waves,shift):
            return flux_in_bin+shift

        #can be parallelized
        for index in range(nmodels):
            xw , flux_in_bin = mean_regrid(self.wavelength[grid_name],self.spectra[grid_name][index,:],newx= wlgrid_center)

            if offset:
                popt, pcov = optimize.curve_fit(shift_spectrum, wlgrid_center, y_data,p0=[-0.001])
                shift = popt[0]
            else:
                shift = 0
            if dof == 'ndata': numparams=0
            self.chi_sqs[grid_name][data_name][index]= chi_squared(y_data,e_data,flux_in_bin+shift,numparams)

            self.best_fits[grid_name][data_name][index,:] = flux_in_bin+shift
            if offset: self.offsets[grid_name][data_name][index] = shift

        self.rank[grid_name][data_name] = self.chi_sqs[grid_name][data_name].argsort()

        #finally compute the posteriors
        for iattr in self.grid_params[grid_name].keys():
            for ikey in self.grid_params[grid_name][iattr].keys():
                self.posteriors[grid_name][data_name][ikey] = self.get_posteriors(grid_name, data_name, ikey)

    def print_best_fit(self, grid_name, data_name, verbose=True):
        """
        Print out table of best fit parameters

        Parameters
        ----------
        grid_name : str
            grid name or string of single grid name to plot
        data_name : str
            data name or string of single
        """
        best_fits = {}
        for iattr in self.grid_params[grid_name].keys():
            for ikey in self.grid_params[grid_name][iattr].keys():
                single_best_fit = self.grid_params[grid_name][iattr][ikey][self.rank[grid_name][data_name]][0]
                if verbose: print(f'{ikey}={single_best_fit}')
                best_fits[ikey] = single_best_fit
        return best_fits

    def plot_best_fit(self, grid_names, data_names, plot_kwargs={}):
        """

        Parameters
        ----------
        grid_names : list, str
            List of grid names or string of single grid name to plot
        data_names : list, str
            List of data names or string of single
        plot_kwargs : dict
            key word arguments for matplotlib plt
        """
        if isinstance(grid_names ,str):grid_names=[grid_names]
        if isinstance(data_names ,str):data_names=[data_names]

        x='''
        AA
        ..
        BB
        '''
        fig = plt.figure(figsize=(18,10))
        plt.style.use('seaborn-paper')
        plt.rcParams['figure.figsize'] = [7, 4]           # Figure dimensions
        plt.rcParams['figure.dpi'] = 300
        plt.rcParams['image.aspect'] = 1.2                       # Aspect ratio (the CCD is quite long!!!)
        plt.rcParams['lines.linewidth'] = 1
        plt.rcParams['lines.markersize'] = 3
        plt.rcParams['lines.markeredgewidth'] = 0

        cmap = plt.cm.magma
        #cmap.set_bad('k',1.)

        plt.rcParams['image.cmap'] = 'magma'                   # Colormap.
        plt.rcParams['image.origin'] = 'lower'
        plt.rcParams['font.family'] = 'sans-serif'
        plt.rcParams['font.serif'] = 'DejaVu Sans'
        plt.rcParams['mathtext.fontset'] = 'stixsans'
        #plt.rcParams['axes.prop_cycle'] = \
        #plt.cycler(color=["tomato", "dodgerblue", "gold", 'forestgreen', 'mediumorchid', 'lightblue'])
        plt.rcParams['figure.dpi'] = 300
        colors=["xkcd:salmon", "dodgerblue", "sandybrown", 'cadetblue', 'orchid', 'lightblue']
        plt.rcParams["axes.prop_cycle"] = plt.cycler(color=colors)

        ax = fig.subplot_mosaic(x,gridspec_kw={
                # set the height ratios between the rows
                "height_ratios": [1,0.00001,0.2],
                # set the width ratios between the columns
                "width_ratios": [1,1]})

        all_data_waves = np.concatenate([self.data[i]['wlgrid_center'] for i in self.data.keys()])
        ax['A'].set_xlim(np.min(all_data_waves)-0.1,np.max(all_data_waves)+0.1)
        ax['B'].set_xlim(np.min(all_data_waves)-0.1,np.max(all_data_waves)+0.1)
        #ax['A'].set_ylim(np.min(rprs_data2)-0.01*np.min(rprs_data2),np.max(rprs_data2)+0.01*np.max(rprs_data2))

        #colors = ['tomato', 'dodgerblue','forestgreen','green','orchid','slateblue']
        ii=0
        for igrid in grid_names:
            for idata in data_names:
                cycler = ax['A']._get_lines.prop_cycler
                color = next(cycler)['color']

                wlgrid_center = self.data[idata]['wlgrid_center']
                y_data = 100*self.data[idata]['y_data']
                e_data = 100*self.data[idata]['e_data']
                best_fit = 100*self.best_fits[igrid][idata][self.rank[igrid][idata],:][0,:]
                chi1 = self.chi_sqs[igrid][idata][self.rank[igrid][idata]][0]

                ax['A'].plot(wlgrid_center,best_fit,color,linewidth=2,label=r"Best Fit "+igrid+"+"+idata+", ${\chi}_{\\nu}$$^2$= "+ str(np.round(chi1,2)))

                ax['B'].plot(wlgrid_center,(y_data-best_fit)/e_data,"o",color=color,markersize=5)
                if ii==0:ax['B'].plot(wlgrid_center,0*y_data,"k")

                ii+=1

        for i,idata in enumerate(data_names):
            wlgrid_center = self.data[idata]['wlgrid_center']
            y_data = 100*self.data[idata]['y_data']
            e_data = 100*self.data[idata]['e_data']
            ax['A'].errorbar(wlgrid_center,y_data,yerr=e_data,fmt="o",color=Cividis[7][i],label=idata+" Reduction",markersize=5)

        ax['B'].set_xlabel(plot_kwargs.get('xlabel',r"wavelength [$\mu$m]"),fontsize=20)
        ax['A'].set_ylabel(plot_kwargs.get('ylabel',r"transit depth [%]"),fontsize=20)

        ax['A'].minorticks_on()
        ax['A'].tick_params(axis='y',which='major',length =20, width=3,direction='in',labelsize=20)
        ax['A'].tick_params(axis='y',which='minor',length =10, width=2,direction='in',labelsize=20)
        ax['A'].tick_params(axis='x',which='major',length =20, width=3,direction='in',labelsize=20)
        ax['A'].tick_params(axis='x',which='minor',length =10, width=2,direction='in',labelsize=20)

        ax['B'].minorticks_on()
        ax['B'].tick_params(axis='y',which='major',length =20, width=3,direction='in',labelsize=20)
        ax['B'].tick_params(axis='y',which='minor',length =10, width=2,direction='in',labelsize=20)
        ax['B'].tick_params(axis='x',which='major',length =20, width=3,direction='in',labelsize=20)
        ax['B'].tick_params(axis='x',which='minor',length =10, width=2,direction='in',labelsize=20)

        ax['B'].set_ylabel("${\delta}/N$",fontsize=20)

        ax['A'].legend(fontsize=16)

        return fig,ax

    def get_posteriors(self, grid_name, data_name, parameter):
        """
        Get posteriors (x,y) given a grid name, data name and parameter specified in grid_params

        Parameters
        ----------
        grid_names : list, str
            List of grid names or string of single grid name to plot
        data_names : list, str
            List of data names or string of single
        parameter : str
            Name of parameter to get the posterior of (e.g. mh or tint)
        """
        parameter_sort = _finditem(self.grid_params[grid_name], parameter)
        if isinstance(parameter_sort, type(None)):
            raise Exception(f'Parameter {parameter} not found in grid {grid_name}')

        chi_sq = self.chi_sqs[grid_name][data_name]

        parameter_grid =np.unique(parameter_sort)

        prob_array = np.exp(-chi_sq/2.0)
        alpha = 1.0/np.sum(prob_array)
        prob_array = prob_array*alpha

        prob= np.array([])

        for m in parameter_grid:
            wh = np.where(parameter_sort == m)
            prob = np.append(prob,np.sum(prob_array[wh]))

        return parameter_grid,prob

    def plot_posteriors(self, grid_name, data_name,parameters, fig=None, ax=None,
                       x_label_style={}, x_axis_type={}, label=''):
        """
        Plots posteriors for a given parameter set

        Parameters
        ----------
        grid_names : str
            string of single grid name to plot
        data_names : str
            data names or string of single
        parameters : list, str
            Name or list of parameter(s) to get the posterior of (e.g. mh or tint)
        x_label_style : dict
            dictionary with elements of parameters for stylized x axis labels
        x_axis_type : dict
            dictionry with 'linear' 'log' arguments for the x axis.
        labels : list
            how to label the data
        """
        if isinstance(parameters, str): parameters=[parameters]

        if label == '':
            legend_label = grid_name + ' ' + data_name
        else:
            legend_label = label

        if fig == None:
            if ax == None:
                plt.style.use('seaborn-paper')
                plt.rcParams['figure.figsize'] = [7, 4]           # Figure dimensions
                plt.rcParams['figure.dpi'] = 300
                plt.rcParams['image.aspect'] = 1.2                       # Aspect ratio (the CCD is quite long!!!)
                plt.rcParams['lines.linewidth'] = 1
                plt.rcParams['lines.markersize'] = 3
                plt.rcParams['lines.markeredgewidth'] = 0

                cmap = plt.cm.magma
                #cmap.set_bad('k',1.)

                plt.rcParams['image.cmap'] = 'magma'                   # Colormap.
                plt.rcParams['image.origin'] = 'lower'
                plt.rcParams['font.family'] = 'sans-serif'
                plt.rcParams['font.serif'] = 'DejaVu Sans'
                plt.rcParams['mathtext.fontset'] = 'stixsans'
                #plt.rcParams['axes.prop_cycle'] = \
                #color = plt.cycler()
                colors=["xkcd:salmon", "dodgerblue", "sandybrown", 'cadetblue', 'orchid', 'lightblue']
                plt.rcParams["axes.prop_cycle"] = plt.cycler(color=colors)
                plt.rcParams['figure.dpi'] = 300
                nrow = 2
                ncol = int(np.ceil(len(parameters)/nrow))
                fig,ax = plt.subplots(nrows=nrow,ncols=ncol,figsize=(30,20))

        nrow=ax.shape[0]
        ncol=ax.shape[1]
        #colors=["tomato", "dodgerblue", "gold", 'forestgreen', 'mediumorchid', 'lightblue']
        iparam = -1
        for irow in range(nrow):
            for icol in range(ncol):
                iparam+=1
                if icol==0: ax[irow,icol].set_ylabel("Probability",fontsize=50)
                if iparam > len(parameters)-1:
                    try:
                        fig.delaxes(ax[irow,icol])
                        continue
                    except:
                        continue
                else:
                    get_post = _finditem(self.posteriors[grid_name][data_name], parameters[iparam])
                    if isinstance(get_post, type(None)):
                        xgrid,yprob = [0,0,0],[0,0,0]
                    else:
                        xgrid,yprob = get_post

                    ax[irow,icol].set_ylim(1e-2,1)

                    if x_axis_type.get(parameters[iparam],'linear') == 'log':
                        xgrid = np.log10(xgrid)
                    cycler = ax[irow,icol]._get_lines.prop_cycler
                    col = next(cycler)['color']
                    ax[irow,icol].bar(xgrid, yprob,
                        width=[np.mean(abs(np.diff(xgrid)))/2]*len(xgrid),
                        color=col,edgecolor=col,
                        linewidth=5,label=legend_label,alpha=0.2 )
                    ax[irow,icol].tick_params(axis='both',which='major',length =40, width=3,direction='in',labelsize=30)
                    ax[irow,icol].tick_params(axis='both',which='minor',length =10, width=2,direction='in',labelsize=30)

                    label = x_label_style.get(parameters[iparam],parameters[iparam])
                    ax[irow,icol].set_xlabel(label,fontsize=50)

                    ax[irow,icol].legend(fontsize=20)
        return fig, ax

def detection_test(fitter, molecule, min_wavelength, max_wavelength,
                   grid_name, data_name,
                   filename, molecule_baseline=None,baseline_wavelength=[],
                   model_full=None,
                   opa_kwargs={},plot=True):
    """
    Computes the detection significance of a molecule given a grid name, data name,
    filename
    """
    wlgrid_center = fitter.data[data_name]['wlgrid_center']
    y_data = fitter.data[data_name]['y_data']
    e_data = fitter.data[data_name]['e_data']

    index = fitter.list_of_files[grid_name].index(filename)

    shift = fitter.offsets[grid_name][data_name][index]

    xr_data = xr.load_dataset(filename)

    opa = opannection(**opa_kwargs)
    case = input_xarray(xr_data, opa)
    og_atmo = copy.deepcopy(case.inputs['atmosphere']['profile'])

    if isinstance(model_full,type(None)):
        model_full = xr_data.data_vars['transit_depth']
        wavelength = xr_data.coords['wavelength']
        wavelength, model_full = mean_regrid(wavelength,model_full,newx=wlgrid_center)
        model_full = model_full + shift

    #
    double_gauss =False
    if isinstance(molecule_baseline,str):
        molecule = [molecule, molecule_baseline]
        double_gauss = True
        if len(baseline_wavelength)==2:
            min_wavelength_add = sorted(baseline_wavelength)[0]
            max_wavelength_add = sorted(baseline_wavelength)[1]
        else:
            min_wavelength_add = min_wavelength
            max_wavelength_add = max_wavelength

    case.atmosphere(df = og_atmo,exclude_mol=molecule, delim_whitespace=True)
    df= case.spectrum(opa, full_output=True,calculation='transmission') #note the new last key
    wno, model_exclude  = df['wavenumber'] , df['transit_depth']
    wno, model_exclude = mean_regrid(wno,model_exclude,newx=np.sort(1e4/wlgrid_center))
    model_exclude = model_exclude + shift
    out = pd.DataFrame({
        'wno':wno,
        'wavelength':1e4/wno,
        'model_exclude':model_exclude})
    out = out.sort_values(by='wavelength')
    model_exclude = out['model_exclude']
    wavelength = out['wavelength']

    if plot:
        fig,ax = plt.subplots(nrows=3,ncols=1,figsize=(15,10))
        ax[0].plot(wlgrid_center, model_full,color='blue',label='Full Model')
        ax[0].plot(wlgrid_center, model_exclude,color='red',label=f'Without {molecule}')
        ax[0].errorbar(wlgrid_center, y_data, yerr=e_data,fmt='ok')
        ax[0].set_xlabel('wavelength [microns]')
        ax[0].set_ylabel('transit depth')
        ax[0].legend(fontsize=12)

    residual_model = model_full-model_exclude
    residual_data = y_data-model_exclude
    if plot:
        ax[1].plot(wlgrid_center, residual_model, color='blue',label='Residual Model')
        ax[1].errorbar(wlgrid_center, residual_data, yerr=e_data,fmt='ok',label='Residual in data')
        ax[1].set_xlabel('wavelength [microns]')
        ax[1].set_ylabel('delta transit depth')
        ax[1].legend(fontsize=12)

    #defining gaussian model-params are centeral wavlenegth, width, amplitude, and a constant "DC" offset
    def model_gauss(wlgrid, lam0, sig, Amp,cst):
        return (Amp*np.exp(-(wlgrid-lam0)**2/sig**2)+cst)/1e6

    def model_double_gauss(wlgrid, lam01, sig1, Amp1,cst1,lam02, sig2, Amp2,cst2):
        return ((Amp1*np.exp(-(wlgrid-lam01)**2/sig1**2)+cst1)/1e6 +
                (Amp2*np.exp(-(wlgrid-lam02)**2/sig2**2)+cst2)/1e6    )
    #TK ADD IN DOUBLE LOGLIKE AND DOUBLE PRIOR
    #likelihood function
    def loglike_gauss(theta):
        logAmp, lam0,logsig,cst=theta #fitting for the "log" amplitude and witdths b/c why not...could try linear to see if it affects answer
        mod=model_gauss(wlgrid_center, lam0, 10**logsig, 10**logAmp,cst) #evaluating model
        lnl=-0.5*np.sum((residual_data-mod)**2/e_data**2) #the equation for -1/2 chi-square....
        return lnl
    def loglike_double_gauss(theta):
        logAmp1, lam01,logsig1,cst1,logAmp2, lam02,logsig2,cst2=theta #fitting for the "log" amplitude and witdths b/c why not...could try linear to see if it affects answer
        mod=model_double_gauss(wlgrid_center, lam01, 10**logsig1, 10**logAmp1, cst1,
                                       lam02, 10**logsig2, 10**logAmp2, cst2) #evaluating model
        lnl=-0.5*np.sum((residual_data-mod)**2/e_data**2) #the equation for -1/2 chi-square....
        return lnl

    #prior transform
    def prior_transform_gauss(theta):
        logAmp, lam0,logsig,cst=theta
        logAmp=-1+(4.5+1)*logAmp
        lam0=min_wavelength+(max_wavelength-min_wavelength)*lam0
        logsig=-2+(1+2)*logsig
        cst=-200+(400)*cst
        return logAmp, lam0,logsig,cst
    def prior_transform_double_gauss(theta):
        logAmp1, lam01,logsig1,cst1,logAmp2, lam02,logsig2,cst2=theta
        logAmp1=-1+(4.5+1)*logAmp1
        lam01=min_wavelength+(max_wavelength-min_wavelength)*lam01
        logsig1=-2+(1+2)*logsig1
        cst1=-200+(400)*cst1
        logAmp2=-1+(4.5+1)*logAmp2
        lam02=min_wavelength_add+(max_wavelength_add-min_wavelength_add)*lam02
        logsig2=-2+(1+2)*logsig2
        cst2=-200+(400)*cst2
        return logAmp1, lam01,logsig1,cst1,logAmp2, lam02,logsig2,cst2

    Nproc=4  #number of processors for multi processing--best if you can run on a 12 core+ node or something
    Nlive=500 #number of nested sampling live points

    #setting up multi-threading and sampler
    #pool = Pool(processes=Nproc)
    results = {}
    models = []
    if double_gauss:
        models += ['double']
        Nparam=8  #number of parameters--make sure it is the same as what is in prior and loglike
        results['double'] = dynesty.NestedSampler(loglike_double_gauss, prior_transform_double_gauss, ndim=Nparam,
                                            bound='multi', sample='auto', nlive=Nlive)#,
                                            #pool=pool, queue_size=Nproc)
    #run single for comparison
    Nparam = 4
    models += ['single']
    results['single'] = dynesty.NestedSampler(loglike_gauss, prior_transform_gauss, ndim=Nparam,
                                        bound='multi', sample='auto', nlive=Nlive)#,
                                        #pool=pool, queue_size=Nproc)
    keys = list(results.keys())
    for dsampler in keys:
        results[dsampler].run_nested()
        #GAUSS RESULTS
        results[f'dres_{dsampler}'] = results[dsampler].results #results
        ##grabbing the final evidence--will be used for bayes factor (see Dynesty documnetation)
        results[f'logZ_{dsampler}'] = results[f'dres_{dsampler}'].logz[-1]
        samples, weights = results[f'dres_{dsampler}'].samples, np.exp(results[f'dres_{dsampler}'].logwt - results[f'dres_{dsampler}'].logz[-1])
        results[f'samp_{dsampler}'] = dyfunc.resample_equal(samples, weights)

    #flat line test
    def model_line(wlgrid,cst):
        #flat line slope = 0
        return (cst+wlgrid*0. )/1e6

    #loglike with
    def loglike_line(theta):
        cst=theta
        mod=model_line(wlgrid_center, cst)
        lnl=-0.5*np.sum((residual_data-mod)**2/e_data**2)
        return lnl

    #prior cube
    def prior_transform_line(theta):
        cst=theta
        cst=-200+(2000)*cst
        return cst

    Nparam=1
    results['line'] = dynesty.NestedSampler(loglike_line, prior_transform_line, ndim=Nparam,
                                        bound='multi', sample='auto', nlive=Nlive#,
                                        #pool=pool, queue_size=Nproc
                                    )

    results['line'].run_nested()
    results['dres_line'] = results['line'].results
    results['logZ_line'] = results['dres_line'].logz[-1]
    samples, weights = results['dres_line'].samples, np.exp(results['dres_line'].logwt - results['dres_line'].logz[-1])
    results['samp_line'] = dyfunc.resample_equal(samples, weights)

    if plot:
        ax[2].errorbar(wlgrid_center, residual_data,yerr=e_data,fmt='ob',ms=3,label='Residual Data')

        samp_gauss = results['samp_single']
        for i in range(samp_gauss.shape[0]):
            logAmp, lam0,logsig,cst=samp_gauss[i,:]
            mod=model_gauss(wlgrid_center, lam0, 10**logsig, 10**logAmp,cst)
            ax[2].plot(wlgrid_center, mod,alpha=0.01,color='purple')

        ax[2].plot(wlgrid_center, mod,alpha=0.5,color='purple')

        if double_gauss:
            samp_gauss = results['samp_double']
            for i in range(samp_gauss.shape[0]):
                logAmp1, lam01,logsig1,cst1,logAmp2, lam02,logsig2,cst2=samp_gauss[i,:]
                mod=model_double_gauss(wlgrid_center, lam01, 10**logsig1, 10**logAmp1, cst1,
                                   lam02, 10**logsig2, 10**logAmp2, cst2)
                ax[2].plot(wlgrid_center, mod,alpha=0.01,color='orange')

            ax[2].plot(wlgrid_center, mod,alpha=0.5,color='orange')

        samp_line = results['samp_line']
        for i in range(samp_line.shape[0]):
            cst=samp_line[i,:]
            mod=model_line(wlgrid_center, cst)
            ax[2].plot(wlgrid_center, mod,alpha=0.01,color='grey')

        ax[2].plot(wlgrid_center, mod,alpha=0.5,color='grey',label='Constant Fit Ensemble')

        ax[2].set_xlabel('wavelength [microns]')
        ax[2].set_ylabel('Delta Transit Depth')
        ax[2].legend(fontsize=12)

    results['sigma_single_v_line'],results['lnB_single_v_line']= sigma(
                                        results['logZ_single'], results['logZ_line'])

    if double_gauss:
        results['sigma_double_v_single'],results['lnB_double_v_single']= sigma(
                                    results['logZ_double'], results['logZ_single'])
    return results

def chi_squared(data,data_err,model,numparams):
    """
    Compute reduced chi squared assuming DOF = ndata_pts - num parameters
    """

    chi_squared = np.sum(((data-model)/(data_err))**2)/(len(data)-(numparams))

    return chi_squared

def plot_atmosphere(location,bf_filename,gas_names=None,fig=None,ax=None,linestyle=None,color=None,label=None):

    f = os.path.join(location, bf_filename)

    if os.path.isfile(f):

        ds = xr.open_dataset(f)

        temp = ds['temperature'].values[:]
        pressure = ds['pressure'].values[:]
        gas_vmr = np.zeros(shape=(len(gas_names),len(pressure)))

        try: # see if clouds are here
            pressure_cld = ds['pressure_cld'].values[:]
            wno_cld = ds['wno_cld'].values[:]
            wno_cld = ds['wno_cld'].values[:]
            wno_cld = ds['wno_cld'].values[:]
            opd_cld = ds['opd'].values
            asy_cld = ds['asy'].values
            ssa_cld = ds['ssa'].values
        except:
            pressure_cld = 0
            wno_cld = 0
            wno_cld = 0
            wno_cld = 0
            opd_cld = 0
            asy_cld = 0
            ssa_cld = 0

        for igas,gases in zip(range(0,len(gas_names)),gas_names):

            try: # see if clouds are here
                gas_vmr[igas,:]= ds[gases].values[:]

            except:
                gas_vmr[igas,:]= 0.0

        if fig == None:
            if ax == None:
                plt.style.use('seaborn-paper')
                plt.rcParams['figure.figsize'] = [7, 4]           # Figure dimensions
                plt.rcParams['figure.dpi'] = 300
                plt.rcParams['image.aspect'] = 1.2                       # Aspect ratio (the CCD is quite long!!!)
                plt.rcParams['lines.linewidth'] = 1
                plt.rcParams['lines.markersize'] = 3
                plt.rcParams['lines.markeredgewidth'] = 0

                cmap = plt.cm.get_cmap('tab20b', len(gas_names))
                #cmap.set_bad('k',1.)

                plt.rcParams['image.cmap'] = 'magma'                   # Colormap.
                plt.rcParams['image.origin'] = 'lower'
                plt.rcParams['font.family'] = 'serif'
                plt.rcParams['font.serif'] = 'DejaVu Sans'
                plt.rcParams['mathtext.fontset'] = 'stixsans'
                plt.rcParams['axes.prop_cycle'] = \
                plt.cycler(color=["xkcd:salmon", "dodgerblue", "sandybrown", 'cadetblue', 'orchid', 'lightblue'])
                plt.rcParams['figure.dpi'] = 300
                fig,ax = plt.subplots(nrows=1,ncols=3,figsize=(30,10))

        cmap = plt.cm.get_cmap('tab20b', len(gas_names))

        ax[0].set_ylim(1000,1e-6)
        ax[0].set_xlim(500,3500)

        ax[0].semilogy(temp,pressure,linewidth=3,linestyle=linestyle,color=color,label=label)
        ax[0].minorticks_on()
        ax[0].tick_params(axis='both',which='major',length =40, width=3,direction='in',labelsize=30)
        ax[0].tick_params(axis='both',which='minor',length =10, width=2,direction='in',labelsize=30)
        ax[0].legend(fontsize=15)
        ax[0].set_xlabel(r"Temperature [K]",fontsize=30)
        ax[0].set_ylabel(r"Pressure [Bars]",fontsize=30)

        ax[1].set_ylim(1000,1e-6)
        ax[1].set_xlim(1e-8,1)

        #ax[1].set_ylabel(r"Pressure [Bars]",fontsize=50)

        for igas,gases in zip(range(0,len(gas_names)),gas_names):

            ax[1].loglog(gas_vmr[igas,:],pressure,linewidth=3,linestyle=linestyle,color=cmap(igas),label=gases+" "+label)

        ax[1].minorticks_on()
        ax[1].tick_params(axis='both',which='major',length =40, width=3,direction='in',labelsize=30)
        ax[1].tick_params(axis='both',which='minor',length =10, width=2,direction='in',labelsize=30)
        ax[1].set_xlabel(r"VMR",fontsize=30)

        ax[1].legend(fontsize=12)

        ax[2].set_ylim(1000,1e-6)
        ax[2].set_xlim(1e-5,500)

        #ax[2].set_ylabel(r"Pressure [Bars]",fontsize=50)
        if np.sum(opd_cld) != 0:
            ax[2].loglog(opd_cld[:,150],pressure_cld,linewidth=2,linestyle=linestyle,color=color,label=label)

        ax[2].minorticks_on()
        ax[2].tick_params(axis='both',which='major',length =40, width=3,direction='in',labelsize=30)
        ax[2].tick_params(axis='both',which='minor',length =10, width=2,direction='in',labelsize=30)
        ax[2].set_xlabel(r"Cloud OPD (1 ${\mu}$m)",fontsize=30)
        ax[2].legend(fontsize=15)

    else:
        print("Filename or directory is not correct.")
        fig,ax=0

    # function to show best fit atmospheric abundances

    return fig,ax

#equations in Trotta 2008
def sigma(lnz1,lnz2):
    """
    Author: Mike Line (mrline@asu.edu)

    Computes equatiosn from Trotta 2008
    https://ui.adsabs.harvard.edu/abs/2008ConPh..49...71T/abstract

    Tests model preference from model 1 vs model 2
    Returns Bayes Factor (Eqn. 21) and sigma significance (Table 2)

    Parameters
    ----------
    lnz1 : float
        evidence model 1
    lnz2 : float
        evidence model 2

    Returns
    -------
    sigma, bayes factor
    """

    # This is the python version of sigma.pro

    lnB = lnz1 - lnz2
    logp = np.arange(-300.00,0.00,.1) #reverse order
    logp = logp[::-1] # original order
    P = 10.0**logp
    Barr = -1./(np.exp(1)*P*np.log(P))

    sigma = np.arange(0.1,100.10,.01)
    p_p = sp.special.erfc(sigma/np.sqrt(2.0))
    B = np.exp(lnB)
    pvalue = 10.0**np.interp(np.log10(B),np.log10(Barr),np.log10(P))
    sig = np.interp(pvalue,p_p[::-1],sigma[::-1])

    return sig , lnB

def _finditem(obj, key):
    if key in obj: return obj[key]
    for k, v in obj.items():
        if isinstance(v,dict):
            item = _finditem(v, key)
            if item is not None:
                return item

def _get_xarray_attr(attr_dict, parameter):
    not_found_msg = "{parameter} not specified"
    #we assume clear if no fsed parameter specified
    if parameter =='fsed':
        not_found_msg='clear'
    param = attr_dict.get(parameter,not_found_msg)
    if isinstance(param, dict):
        param_flt = param.get('value',param)
        #get unit
        #if isinstance(param.get('unit',np.nan),str):
        #    try:
        #        param_unit = u.Unit(param.get('unit'))
        #        param = param_flt*param_unit
        #    except ValueError:
        #        param = param_flt
        #        pass
        #else:
        #    param = param_flt
        param = param_flt

    if isinstance(param, str):
        if len(param.split(' ')) > 1:
            #float value
            try:
                param_flt = float(param.split(' ')[0])
            except ValueError:
                param_flt = np.nan
                pass
            param = param_flt
            #unit value
            #if not np.isnan(param_flt):
            #    try:
            #        param_unit = u.Unit(''.join(param.split(' ')[1:]))
            #        param = param_flt*param_unit
            #    except ValueError:
            #        param = param_flt
            #        pass
    return param


from pyomo.core import Param, value

def create_expected_value_instance(average_instance,
                                   scenario_tree,
                                   scenario_instances,
                                   verbose=False):

    rootnode = scenario_tree._stages[0]._tree_nodes[0]
    ScenCnt = len(rootnode._scenarios)

    for p in average_instance.component_map(Param, active=True):

        average_parameter_object = getattr(average_instance, p)

        for index in average_parameter_object:
            average_value = 0.0
            for scenario in rootnode._scenarios:
                scenario_parameter_object = getattr(scenario_instances[scenario._name], p)
                average_value += value(scenario_parameter_object[index])
            average_value = average_value / float(len(scenario_instances))
            average_parameter_object[index] = average_value

def fix_ef_first_stage_variables(ph, scenario_tree, expected_value_instance):

    if ph._verbose:
        print("Fixing first stage variables at mean instance solution values.\n")

    stage = ph._scenario_tree._stages[0]
    root_node = stage._tree_nodes[0] # there should be only one root node!
    for variable_name, index_template in stage._variable_templates.iteritems():

        variable_indices = root_node._variable_indices[variable_name]
        for index in variable_indices:
            for scen in root_node._scenarios:
                inst = ph._instances[scen._name]
                print("HEYYYY fix varstatus !!!!!xxxxxx\n")
                #if getattr(inst, variable_name)[index].status != VarStatus.unused:
                if 1 == 1:
                    print("variable_name= %s\n" % variable_name)
                    fix_value = getattr(expected_value_instance, variable_name)[index].value
                    getattr(inst, variable_name)[index].fix(fix_value)


import ray

from forge.blade.core.env import HarvestEnv
from forge.trinity import Sword

@ray.remote#(memory=10 ** 10)
class Realm:
    def __init__(self, config, args, idx):
        self.env = HarvestEnv(num_agents=5)
        self.horizon = config.HORIZON
        self.config = config
        self.sword = Sword(config, args)
        self.idx = idx
        self.step = 0

    def recvSwordUpdate(self, update):
        if update is None:
            return
        self.sword.recvUpdate(update)

    def run(self, update):
        self.recvSwordUpdate(update)
        for epoch in range(self.config.EPOCHS):
            obs = self.env.reset()
            rewards = self.env.getInitialRewards()
            for i in range(self.horizon):
                self.step += 1
                agents = self.env.agents
                actions = {key: self.sword.decide(agents[key], obs[key], rewards[key],
                                                  (i + 1) % self.config.LSTM_PERIOD == 0,
                                                  i + epoch * self.horizon, epoch) for key in agents.keys()}
                obs, rewards, dones, info, = self.env.step(actions)
              #  commonReward = 0
              #  for key in obs.keys():
              #      commonReward += rewards[key]

                for key in obs.keys():
                    annID = agents[key].annID
                    self.sword.buffer[annID]['reward'][i + epoch * self.horizon] = rewards[key]
            for agent in self.env.agents.values():
                self.sword.collectRollout(agent.agent_id + str(epoch), agent, self.step, epoch)
        self.sword.backward()
        logs = self.sword.sendLogUpdate()
        buf = self.sword.dispatchBuffer()
        return self.idx, buf, logs


import matplotlib.pyplot as plt
import os
import torch

from utils.util import project_path, ensure_dir

v = 1
act = "relu"
npl = [784, 10]

model_path = "imgs/mnist/mnist_2021.02.05_20.46.43_w=1_v=1_i=normal_npl=[784, 10]_act=relu_eta=0.1_wd=0.0_epochs=100000/model_best.pth"
model_path = os.path.join(project_path, model_path)
plots_path = os.path.join(project_path, "imgs/mnist/layers/ae_relu_init=noraml_uniform_wd=0")
ensure_dir(plots_path)

model = torch.load(model_path)
model.to("cpu")

w = None
if v == 1:
    w = model.weights[0].T
    w = w.detach()
elif v == 2:
    w = model.seq.ll_1.weight.data

for i in range(10):
    title = f"weights for digit {i}, npl:{npl}"
    print(title)
    plt.title(title)
    plt.imshow(w[i].reshape((28, 28)), cmap=plt.get_cmap('gray'))
    plt.savefig(os.path.join(plots_path, f"{i}.png"), dpi=300)
    plt.close()

    title = f"weights for digit {i}, npl:{npl}"
    print(title)
    plt.title(title)
    plt.imshow(w[i].reshape((28, 28)))
    plt.savefig(os.path.join(plots_path, f"g_{i}.png"), dpi=300)
    plt.close()

# (x_train, y_train), (x_valid, y_valid), (x_test, y_test) = load_mnist()
# train_mean, train_std = standardize_inplace(x_train, [x_valid, x_test])
#
# for i in range(50):
#     plt.title(f"i:{i} digit {y_train[i]}")
#     plt.imshow(x_train[i], cmap=plt.get_cmap('gray'))
#     plt.savefig(os.path.join(plots_path, f"s_MNIST_TRAIN_{i}.png"), dpi=300)
#     # plt.show()
#     plt.close()


import numpy as np
import matplotlib.pyplot as plt
import scipy.linalg as sl
import scipy as sp

"""原理
#         RANSAC算法的输入是一组观测数据，一个可以解释或者适应于观测数据的参数化模型，一些可信的参数。
#         RANSAC通过反复选择数据中的一组随机子集来达成目标。被选取的子集被假设为局内点，并用下述方法进行验证：
#         1.
#         首先我们先随机假设一小组局内点为初始值。然后用此局内点拟合一个模型，此模型适应于假设的局内点，所有的未知参数都能从假设的局内点计算得出。
#         2.
#         用1中得到的模型去测试所有的其它数据，如果某个点适用于估计的模型，认为它也是局内点，将局内点扩充。
#         3.
#         如果有足够多的点被归类为假设的局内点，那么估计的模型就足够合理。
#         4.
#         然后，用所有假设的局内点去重新估计模型，因为此模型仅仅是在初始的假设的局内点估计的，后续有扩充后，需要更新。
#         5.
#         最后，通过估计局内点与模型的错误率来评估模型。  整个这个过程为迭代一次，此过程被重复执行固定的次数，每次产生的模型有两个结局：  1、要么因为局内点太少，还不如上一次的模型，而被舍弃，  2、要么因为比现有的模型更好而被选用。
#

"""

class LinearLeastsquareModel:
    # 最小二乘求线性解,用于RANSAC的输入模型
    def __init__(self,input_columns,output_columns,debug = False):
        self.input_columns = input_columns
        self.output_columns = output_columns
        self.debug = debug

    # 将输入x，y进行按a0 + b0x = y 进行堆叠，写成[1，x] *[a,b]^T = [y]矩阵格式
    # np.vstack按垂直方向（行顺序）堆叠数组构成一个新的数组
    def fit(self,data):
        A = np.vstack([data[:,i] for i in self.input_columns]).T
        B = np.vstack([data[:,i] for i in self.output_columns]).T
        x, resids, rank, s = sp.linalg.lstsq(A,B) #residues:残差和，x：返回最小二乘法的k/b值
        return x #返回最小平方和向量

    def get_error(self,data,model):
        A = np.vstack([data[:, i] for i in self.input_columns]).T
        B = np.vstack([data[:, i] for i in self.output_columns]).T
        # 计算的y值,B_fit = model.k*A + model.b
        B_fit = sp.dot(A,model)
        err_per_point =np.sum((B-B_fit)**2,axis=1)
        return err_per_point

def ransac_model(data,model,n,k,t,d,debug= False,return_all = False):
    # 输入:
    #     data - 样本点
    #     model - 假设模型:事先自己确定
    #     n - 生成模型所需的最少样本点
    #     k - 最大迭代次数
    #     t - 阈值:作为判断点满足模型的条件
    #     d - 拟合较好时,需要的样本点最少的个数,当做阈值看待
    # 输出:
    #     bestfit - 最优拟合解（返回nil,如果未找到）
    # 输出：
    # best_model —— 跟数据最匹配的模型参数（如果没有找到好的模型，返回null）
    # best_consensus_set —— 估计出模型的数据点
    # best_error —— 跟数据相关的估计出的模型错误
    # ————————————————

    # maybe_inliers =  # 从数据集中随机选择n个点
    # maybe_model =  # 适合于maybe_inliers的模型参数
    # consensus_set = maybe_inliers
    # for (  # 每个数据集中不属于maybe_inliers的点 ）
    #         if ( 如果点适合于maybe_model，且错误小于t ）
    #         将点添加到consensus_set
    #         if （ consensus_set中的元素数目大于d ）
    #         已经找到了好的模型，现在测试该模型到底有多好
    #         better_model = 适合于consensus_set中所有点的模型参数
    #         this_error = better_model究竟如何适合这些点的度量
    #         if ( this_error < best_error)
    # 我们发现了比以前好的模型，保存该模型直到更好的模型出现
    # best_model =  better_model
    # best_consensus_set = consensus_set
    # best_error =  this_error
    # 增加迭代次数
    # 返回 best_model, best_consensus_set, best_error
    iterations = 0
    bestfit = None
    best_inlier_idxs = None #内群
    better_point_set = None #

    betterr = np.inf
    while(iterations < k):
        #从数据中随机选择n个点作为内群点maybe_idxs,其他测试点test_idxs
        #这里注意，需要每次都将数据打乱
        if n < data.shape[0]:
            all_idxs = np.arange(data.shape[0]) #生成特定补偿的排列
            np.random.shuffle(all_idxs)
            maybe_idxs = all_idxs[:n]
            test_idxs = all_idxs[n:]

        else:
            print(f"设定的内群点数目大于输入点数目 {n} > {data.shape[0]}" )
            break
        #内群随机点
        maybe_inliners = data[maybe_idxs,:]
        test_points = data[test_idxs,:]

        # 适合于内群点maybe_inliers的模型参数k/b
        maybe_model = model.fit(maybe_inliners)
        #计算其他点误差的最小平方和
        test_err = model.get_error(test_points,maybe_model)

        # 如果点适合于maybe_model，且错误小于t，这里consensus_set/also_inliers是等同的，这里只是学习
        consensus_set = test_points[test_err<t]
        also_idxs =  test_idxs[test_err<t]
        also_inliers = data[also_idxs, :]
        # print(also_inliners)

        if debug:
            print ('test_err.min()',test_err.min())
            print ('test_err.max()',test_err.max())
            print ('numpy.mean(test_err)',np.mean(test_err))
            print ('iteration %d:len(alsoinliers) = %d' %(iterations, len(also_inliers)) )

        #         将点添加到consensus_set
        if (len(also_inliers) > d):
            # betterdata = np.concatenate((maybe_inliners,consensus_set))
            betterdata = np.concatenate((maybe_inliners, also_inliers))
            better_model = model.fit(betterdata)
            better_err = model.get_error(betterdata,better_model)
            thiserr = np.mean(better_err)
            if thiserr < betterr :
                bestfit = better_model
                betterr = thiserr
                best_point_set = betterdata
                best_inlier_idxs = np.concatenate((maybe_idxs, also_idxs))  # 更新局内点,将新点加入

        iterations += 1
    if bestfit is None:
        raise ValueError("did't meet fit acceptance criteria")
    if return_all:
        return bestfit,{'inliers':best_inlier_idxs},best_point_set
    else:
        return bestfit

def test():
    # 生成理想数据
    n_samples = 500  # 样本个数
    n_inputs = 1  # 输入变量个数
    n_outputs = 1  # 输出变量个数

    # 随机生成0-20之间的500个数据:行向量x
    A_exact = 20 * np.random.random((n_samples,n_inputs))
    # print(f"%s A_exact.shape:",A_exact.shape) #500*1

    #生成斜率K
    perfect_fit =20 * np.random.normal(size=(n_inputs,n_outputs))

    #由y = kx生成直线上的点 500*1
    B_exact = sp.dot(A_exact,perfect_fit)

    #生成局外点和噪声点,这里是噪声 500 * 1行向量,代表Xi/Yi
    A_noisy = A_exact + np.random.normal(size=A_exact.shape)
    B_noisy = B_exact + np.random.normal(size=B_exact.shape)
    # print(f"%s A_noisy:", A_noisy[range(2), :])
    # print(f"%s A_noisy:", B_noisy[range(2), :])

    if 1:
        #局外干扰点,打乱原数组顺序，添加局外干扰点
        n_outliners = 100
        all_idxs = np.arange(A_noisy.shape[0])
        np.random.shuffle(all_idxs) #将all_idxs打乱
        outliner_idxs = all_idxs[:n_outliners]#取前100个
        A_noisy[outliner_idxs] = 20 * np.random.normal(size=(n_outliners,n_outputs))
        B_noisy[outliner_idxs] = 30 * np.random.normal(size=(n_outliners, n_outputs))
        #print("%s A_noise.shape:",A_noise.shape)

    #setup-model
    all_data = np.hstack((A_noisy,B_noisy)) #形式([Xi,Yi]....) shape:(500,2)
    # print(f"%s all_data:", all_data[range(2), :])
    input_columns = range(n_inputs)
    output_columns = [n_inputs+i for i in range(n_outputs)]
    debug = False

    #使用最小二乘法生成模型
    model = LinearLeastsquareModel(input_columns,output_columns,debug=debug) #类的实例化:用最小二乘生成已知模型

    #生成最小二乘法的model, 将x，y代入y =ax+b中求得a，b，都存储到第一个返回值里 linear_fit，这是最小二乘法的拟合K/b
    linear_fit,residues,rank,s = sp.linalg.lstsq(all_data[:,input_columns],all_data[:,output_columns])

    #由Ransac算法计算拟合的K值
    ransac_fit, ransac_data, ransac_point =   ransac_model(all_data, model, 50, 3000, 5e3, 200,debug = debug, return_all = True)

    if 1:
        import pylab

        sort_idxs = np.argsort(A_exact[:, 0])
        A_col0_sorted = A_exact[sort_idxs]  # 秩为2的数组

        #没有看懂为什么从小到大排序，有啥区别
        # A_col0_sorted = A_exact

        if 1:
            pylab.plot(A_noisy[:, 0], B_noisy[:, 0], 'k.', label='data')  # 散点图

            #展示方式1
            pylab.plot( A_noisy[ransac_data['inliers'], 0], B_noisy[ransac_data['inliers'], 0], 'bx',
                       label="RANSAC data")

            # 展示方式2
            # pylab.plot(ransac_point[:, 0], ransac_point[:, 1], 'x',
            #            label="RANSAC another data")

        else:
            pylab.plot(A_noisy[non_outlier_idxs, 0], B_noisy[non_outlier_idxs, 0], 'k.', label='noisy data')
            pylab.plot(A_noisy[outlier_idxs, 0], B_noisy[outlier_idxs, 0], 'r.', label='outlier data')

        pylab.plot(A_col0_sorted[:, 0],
                   np.dot(A_col0_sorted, ransac_fit)[:, 0],
                   color="red", label='RANSAC fit')
        pylab.plot(A_col0_sorted[:, 0],
                   np.dot(A_col0_sorted, perfect_fit)[:, 0],
                   color="yellow", label='exact system')
        pylab.plot(A_col0_sorted[:, 0],
                   np.dot(A_col0_sorted, linear_fit)[:, 0],
                   label='linear fit')
        pylab.legend()
        pylab.show()

if __name__ == "__main__":
    test()


from Motion_Detector import df
from bokeh.plotting import figure , show, output_file
from bokeh.models import HoverTool, ColumnDataSource

#Convert df to String

df["Start_string"] = df["Start"].dt.strftime("%Y-%M-%D %H:%M:%S")
df["End_string"] = df["End"].dt.strftime("%Y-%M-%D %H:%M:%S")

# Collecting the data Source

cds = ColumnDataSource(df)

# Creating the Graph

p = figure(x_axis_type = "datetime" , sizing_mode='scale_width', height = 100 , width = 500 , title = "Motion Graph")
p.yaxis.minor_tick_line_color = None
p.ygrid[0].ticker.desired_num_ticks = 1
p.title.text_font_size = '20pt'
p.title.align = 'center'

#Creating the Hover Effect Over the Graph

hover = HoverTool(tooltips = [("Start", "@Start_string"), ("End", "@End_string")])
p.add_tools(hover)

q= p.quad(left = "Start", right = "End", bottom = 0 , top = 1, color = "green", source = cds)
output_file("Graph2.html")
show(p)