text:
import numpy as np
import modin.pandas as pd
import pytest

from greykite.common.constants import ADJUSTMENT_DELTA_COL
from greykite.common.constants import END_DATE_COL
from greykite.common.constants import METRIC_COL
from greykite.common.constants import START_DATE_COL
from greykite.common.constants import TIME_COL
from greykite.common.features.adjust_anomalous_data import adjust_anomalous_data
from greykite.common.testing_utils import generate_anomalous_data
from greykite.common.testing_utils import generic_test_adjust_anomalous_data

@pytest.fixture
def data():
    return generate_anomalous_data()

def test_adjust_anomalous_data(data):
    """ Tests various scenarios for ``adjust_anomalous_data``."""
    df_raw = data["df"]
    anomaly_df = data["anomaly_df"]

    # Adjusts for `"y"` for `"MOBILE"` dimension.
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict={
            METRIC_COL: value_col,
            "platform": "MOBILE"},
        filter_by_value_col=None,
        adjustment_method="add")
    adj_values = pd.Series([np.nan, np.nan, 2., 6., 7., 8., 6., 7., 8., 9.])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Adjusts for `"y"` for `"MOBILE"` dimension and vertical `"sales"` or `"ads"`,
    # adjustment_method = `"subtract"`
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict={
            "platform": "MOBILE",
            "vertical": ["sales", "ads"]},
        filter_by_value_col=METRIC_COL,
        adjustment_method="subtract")

    adj_values = pd.Series([np.nan, np.nan, 2., 0., 1., 2., 6., 7., 8., 9.])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Adjusts for `"y"` for `"MOBILE"` dimension and vertical `"sales"`.
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict={
            METRIC_COL: value_col,
            "platform": "MOBILE",
            "vertical": "sales"},
        filter_by_value_col=None)

    adj_values = pd.Series([0., 1., 2., 6., 7., 8., 6., 7., 8., 9.])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Adjusts for `"z"`.
    # Since for `"z"` only Desktop is impacted, no change is expected.
    value_col = "z"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col="z",
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict={
            METRIC_COL: value_col,
            "platform": "MOBILE"},
        filter_by_value_col=None)

    adj_values = pd.Series([20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Adjusts for `"z"` with DESKTOP dimension passed.
    # Since for `"z"` Desktop is impacted, changes are expected.
    value_col = "z"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col="z",
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict={
            METRIC_COL: value_col,
            "platform": "DESKTOP"},
        filter_by_value_col=None)

    adj_values = pd.Series([20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 22.0, 23.0, np.nan])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Adjusts for `"y"` with no additional dimensions filter.
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col="y",
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict={METRIC_COL: value_col},
        filter_by_value_col=None)

    adj_values = pd.Series([np.nan, np.nan, 2.0, 6.0, 7.0, 8.0, 6.0, 7.0, 8.0, 9.0])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Same as above, using `filter_by_value_col`
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col="y",
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict=None,
        filter_by_value_col=METRIC_COL)

    adj_values = pd.Series([np.nan, np.nan, 2.0, 6.0, 7.0, 8.0, 6.0, 7.0, 8.0, 9.0])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # This does not use the metric column to match or dimensions.
    # However it does use the adjustment delta column.
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=ADJUSTMENT_DELTA_COL,
        filter_by_dict=None,
        filter_by_value_col=None)

    adj_values = pd.Series([np.nan, np.nan, 2.0, 6.0, 7.0, 8.0, 6.0, 2.0, 3.0, np.nan])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # This does not match on metric column or dimensions.
    # Therefore all the anomalies will be used to adjust `"y"`
    # Also does not use adjustment delta, therefore every anomaly is mapped to `np.nan`
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=None,
        filter_by_dict=None,
        filter_by_value_col=None)

    adj_values = pd.Series([np.nan, np.nan, 2.0, np.nan, np.nan, np.nan, 6.0,
                            np.nan, np.nan, np.nan])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Same as above, filter criteria are always met
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=None,
        filter_by_dict={
            METRIC_COL: ("y", "z"),
            "platform": {"MOBILE", "DESKTOP"},
            "vertical": ["ads", "sales"],
        },
        filter_by_value_col=None)

    adj_values = pd.Series([np.nan, np.nan, 2.0, np.nan, np.nan, np.nan, 6.0,
                            np.nan, np.nan, np.nan])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Successful date conversion for comparison.
    # The provided dates are strings in `anomaly_df_format`, datetime in `df_raw`.
    anomaly_df_format = anomaly_df.copy()
    anomaly_df_format[START_DATE_COL] = ["1/1/2018", "1/4/2018", "1/8/2018", "1/10/2018"]
    anomaly_df_format[END_DATE_COL] = ["1/2/2018", "1/6/2018", "1/9/2018", "1/10/2018"]
    value_col = "y"
    adj_df_info = adjust_anomalous_data(
        df=df_raw,
        time_col=TIME_COL,
        value_col=value_col,
        anomaly_df=anomaly_df_format,
        start_date_col=START_DATE_COL,
        end_date_col=END_DATE_COL,
        adjustment_delta_col=None,
        filter_by_dict={
            METRIC_COL: ("y", "z"),
            "platform": {"MOBILE", "DESKTOP"},
            "vertical": ["ads", "sales"],
        },
        filter_by_value_col=None)

    adj_values = pd.Series([np.nan, np.nan, 2.0, np.nan, np.nan, np.nan, 6.0,
                            np.nan, np.nan, np.nan])
    generic_test_adjust_anomalous_data(
        value_col=value_col,
        adj_df_info=adj_df_info,
        adj_values=adj_values)

    # Checks failure to convert date
    anomaly_df_format[START_DATE_COL] = ["999/999/2018", "1/4/2018", "1/8/2018", "1/10/2018"]
    anomaly_df_format[END_DATE_COL] = ["999/999/2019", "1/6/2018", "1/9/2018", "1/10/2018"]
    with pytest.warns(
            UserWarning,
            match=r"Dates could not be parsed by `pandas.to_datetime`, using string comparison "
                  r"for dates instead. Error message:\nUnknown string format: 999/999/2018"):
        value_col = "y"
        adjust_anomalous_data(
            df=df_raw,
            time_col=TIME_COL,
            value_col=value_col,
            anomaly_df=anomaly_df_format,
            start_date_col=START_DATE_COL,
            end_date_col=END_DATE_COL,
            adjustment_delta_col=None,
            filter_by_dict={
                METRIC_COL: ("y", "z"),
                "platform": {"MOBILE", "DESKTOP"},
                "vertical": ["ads", "sales"],
            },
            filter_by_value_col=None)

    # Checks the bad column name exception
    expected_match = "`df` cannot include this column name"
    df_bad_col_name = df_raw.copy()
    df_bad_col_name["adjusted_y"] = np.nan
    with pytest.raises(ValueError, match=expected_match):
        value_col = "y"
        adjust_anomalous_data(
            df=df_bad_col_name,
            time_col=TIME_COL,
            value_col=value_col,
            anomaly_df=anomaly_df,
            start_date_col=START_DATE_COL,
            end_date_col=END_DATE_COL,
            adjustment_delta_col=None,
            filter_by_dict=None,
            filter_by_value_col=None)

    # Checks the bad timestamps exception where a start date is after end date
    # Test is done by inputting the column names for
    # ``start_date_col`` and ``end_date_col`` in reverse order.
    expected_match = "End Time:"
    with pytest.raises(ValueError, match=expected_match):
        value_col = "y"
        adjust_anomalous_data(
            df=df_raw,
            time_col=TIME_COL,
            value_col=value_col,
            anomaly_df=anomaly_df,
            start_date_col=END_DATE_COL,
            end_date_col=START_DATE_COL,
            adjustment_delta_col=None,
            filter_by_dict=None,
            filter_by_value_col=None)

    expected_match = "Column 'device' was requested"
    with pytest.raises(ValueError, match=expected_match):
        value_col = "y"
        adjust_anomalous_data(
            df=df_raw,
            time_col=TIME_COL,
            value_col=value_col,
            anomaly_df=anomaly_df,
            start_date_col=START_DATE_COL,
            end_date_col=END_DATE_COL,
            adjustment_delta_col=None,
            filter_by_dict={"device": "iPhone"},
            filter_by_value_col=None)

max_stars_repo_name: briancpark/greykite

max_issues_repo_name: briancpark/greykite

max_forks_repo_name: briancpark/greykite


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns # used for plot interactive graph. I like it most for plot
from sklearn.model_selection import train_test_split # to split the data into two parts
from sklearn import preprocessing
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from keras.models import Model, Sequential
from keras.layers import Dense, Input, Activation
from keras.models import model_from_json
from keras import optimizers
from keras import backend as K
from keras.utils import plot_model

class ParkerAnalysis:

    def __init__(self):
        pass

    def run(self):
        pdata = pd.read_csv("./data/parker_sleeping.csv", header=0)
        #pdata = pd.read_csv("./data/data.csv")

        pdata.drop("id", axis=1, inplace=True)
        pdata.drop("Counter", axis=1, inplace=True)
        print(pdata)
        data_cols = list(pdata.columns[1:8])
        print(pdata.head())
        print(pdata.describe())
        fig, ax = plt.subplots()
        ax.axis((0,6,0,3000))
        ax.plot(pdata)
        ax.legend(pdata.columns.values.tolist())
        plt.show()

    def do_correlation_matrix(self, drop_cols):
        pdata = pd.read_csv("./data/data.csv", header=0)
        for d in drop_cols:
            pdata.drop(d, axis=1, inplace=True)

        data_cols = list(pdata.columns[0:11])
        corr = pdata[data_cols].corr()  # .corr is used for find corelation
        g = sns.clustermap(corr, cbar=True, square=True, annot=True, fmt='.2f', annot_kws={'size': 8},
                    cmap='coolwarm', figsize=(8, 8))
        plt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)
        plt.setp(g.ax_heatmap.xaxis.get_majorticklabels(), rotation=90)
        plt.show()
        g.savefig("pretty_map")

    def do_data_scaling_an_normalization(self):
        pdata = pd.read_csv("./data/parker_sleeping.csv", header=0)
        pdata.drop("Day", axis=1, inplace=True)
        pdata.drop("Counter", axis=1, inplace=True)
        pdata.drop("Bed", axis=1, inplace=True)
        pdata.drop("Sunshine",  axis=1, inplace=True)
        print("Data:")
        print(pdata)

        pretty_printer = lambda x: str.format('{:.2f}', x)

        nd_normalized = preprocessing.normalize(pdata, norm="l2")

        min_max_scaler = preprocessing.MinMaxScaler()
        nd_scaled = min_max_scaler.fit_transform(pdata)

        fig, (ax1, ax2, ax3) = plt.subplots(3, sharex=True)
        ax1.axis((0, 6, 0, 3000))
        ax1.set_title("Raw Data")
        ax1.plot(pdata)

        ax2.set_title("Normalized")
        ax2.axis((0,6,0, 0.2))
        ax2.plot(nd_normalized)

        ax3.set_title("Scaled")
        ax3.axis((0, 6, 0, 1))
        ax3.plot(nd_scaled)

        ax1.legend(pdata.columns.values.tolist())
        ax2.legend(pdata.columns.values.tolist())
        ax3.legend(pdata.columns.values.tolist())

        plt.show()

    def preprocess_data(self, pdata, preserve):
        preserved = pdata["{0}".format(preserve)]
        pdata.drop("{0}".format(preserve), axis=1, inplace=True)

        nd_normalized = preprocessing.normalize(pdata, norm="l2")
        min_max_scaler = preprocessing.MinMaxScaler()
        nd_scaled = min_max_scaler.fit_transform(nd_normalized)

        # fig, (ax1,ax2,ax3) = plt.subplots(3)
        # box1 = ax1.get_position()
        # ax1.set_title("Raw Data")
        # ax1.plot(pdata)
        #
        # box2 = ax2.get_position()
        # ax2.set_title("Normalized")
        # ax2.plot(nd_normalized)
        #
        # box3 = ax3.get_position()
        # ax3.set_title("Scaled")
        # ax3.plot(nd_scaled)
        #
        # plt.show()
        preprocessed_data = pd.DataFrame(data=nd_scaled, columns=pdata.columns, dtype='float')
        preprocessed_data["{0}".format(preserve)] = preserved.values

        return preprocessed_data

    def do_machine_learning_random_forest(self):
        data = pd.read_csv("./data/data.csv", header=0)
        data.drop("Unnamed: 32", axis=1, inplace=True)
        data.drop("id", axis=1, inplace=True)
        data['diagnosis'] = data['diagnosis'].map({'M': 1, 'B': 0})
        data = self.preprocess_data(data, preserve="diagnosis")
        prediction_var = ['fractal_dimension_mean',
                          'smoothness_mean',
                          'symmetry_mean',
                          'radius_mean',
                          'texture_mean',
                          'compactness_mean']

        train, test = train_test_split(data, test_size=0.3)
        train_X = train[prediction_var]
        train_y = train.diagnosis
        test_X = test[prediction_var]
        test_y = test.diagnosis

        model = RandomForestClassifier(n_estimators=100)

        model.fit(train_X, train_y.astype(int))

        prediction = model.predict(test_X)
        accuracy = metrics.accuracy_score(prediction, test_y)
        plt.show()
        print("Calculation complete.  Random Forest Accuracy: {0}".format(accuracy))
        return accuracy

    def do_neural_network_estimation(self):
        data = pd.read_csv("./data/data.csv", header=0)
        data.drop("Unnamed: 32", axis=1, inplace=True)
        data.drop("id", axis=1, inplace=True)
        data['diagnosis'] = data['diagnosis'].map({'M': 1, 'B': -1})
        data = self.preprocess_data(data, "diagnosis")
        prediction_var = ['fractal_dimension_mean',
                          'smoothness_mean',
                          'symmetry_mean',
                          'radius_mean',
                          'texture_mean',
                          'compactness_mean']
        train, test = train_test_split(data, test_size=0.3)

        train_X = train[prediction_var]
        train_y = train.diagnosis
        test_X = test[prediction_var]
        test_y = test.diagnosis

        model = Sequential()
        model.add(Dense(6, input_shape=(6,), activation='relu'))
        model.add(Dense(100, activation='softmax'))
        model.add(Dense(1, activation='tanh'))
        model.compile(optimizer='rmsprop',
                      loss='mean_squared_error')
        model.fit(train_X.values, train_y.values, batch_size=100, epochs=2000, verbose=False)

        correct_m = 0
        correct_b = 0
        type_I = 0
        type_II = 0

        for x in range(0,len(test_X.values)):

            i = np.reshape(test_X.values[x], (1, 6))
            estimate = model.predict(i)

            if estimate > 0 and test_y.values[x] > 0:
                correct_m = correct_m + 1
            if estimate < 0 and test_y.values[x] < 0:
                correct_b = correct_b + 1
            if estimate > 0 and test_y.values[x] < 0:
                type_I = type_I + 1
            if estimate < 0 and test_y.values[x] > 0:
                type_II = type_II + 1

        a = (correct_m + correct_b) / len(test_X.values)
        print("\n\n{0},{1},{2},{3},{4}\n\n".format(correct_b, correct_m, type_I, type_II, a))

if __name__ == "__main__":
    pa = ParkerAnalysis()
    #pa.do_correlation_matrix(["id", "Unnamed: 32"])

    #for x in range(0,500):

    print(pa.do_machine_learning_random_forest())

    #print("correct_b, correct_m, type_I, type_II, a")
    #for x in range(0,25):
    #    pa.do_neural_network_estimation()

ext: py

reason: import numpy


text:
import os
import glob
import time
import shutil
import pylightcurve as plc

from astropy.io import fits
from urllib.request import urlretrieve

from iraclis import __version__
from iraclis.errors import *
from iraclis.classes import *
from iraclis.images1_reduction import *
from iraclis.images2_calibration import *
from iraclis.images3_photometry import *
from iraclis.lightcurves1_fitting import *

def process_visit(parameters_file=None, data_directory=None, procedure=None, par_string=None):

    print('')
    print('Iraclis {0}'.format(__version__))

    # user inputs

    variables.from_parameters_file(parameters_file, data_directory)

    # for developers use only

    variables.overwrite(procedure, par_string)

    # load the parameters file

    directory = os.path.abspath(variables.data_directory.value)

    if not os.path.isdir(directory):
        raise IraclisFileError('No such directory: ' + directory)
    else:
        print('')
        print('Processing directory: ', os.path.split(directory)[1])
        print('                   @: ', os.path.split(directory)[0])
        print('')

    # set paths

    raw_data_directory = os.path.join(directory, variables.raw_data_directory.value)
    reduced_data_directory = os.path.join(directory, variables.reduced_data_directory.value)
    output_directory = os.path.join(directory, variables.output_directory.value)
    output_directory_copy = os.path.join(directory, variables.output_directory_copy.value)
    figures_directory = variables.figures_directory.value
    extraction_figures_directory = os.path.join(output_directory, '{0}_extraction'.format(figures_directory))
    fitting_figures_directory = os.path.join(output_directory, '{0}_fitting'.format(figures_directory))
    lc_file = os.path.join(output_directory, '{0}.pickle'.format(variables.light_curve_file.value))
    fit_file = os.path.join(output_directory, '{0}.pickle'.format(variables.fitting_file.value))
    splitted_data_directory = os.path.join(directory, variables.splitted_data_directory.value)
    splitted_reduced_data_directory = os.path.join(splitted_data_directory,
                                                   variables.reduced_data_directory.value)

    # set process steps and detect inconsistencies

    if variables.fitting_spectrum.value and not variables.fitting_white.value:
        if not os.path.isfile(fit_file):
            print('\nResetting Fitting-white to True ...')
            variables.fitting_white.set(True)

    if (variables.fitting_white.value and not variables.extraction.value
       and not variables.splitting_extraction.value):
        if not os.path.isfile(lc_file):
            print('\nResetting Extraction to True ...')
            variables.extraction.set(True)

    if variables.extraction.value and not variables.reduction.value:
        if not os.path.isdir(reduced_data_directory):
            print('\nResetting Reduction to True ...')
            variables.reduction.set(True)

    if variables.splitting_extraction.value and not variables.splitting.value:
        if not os.path.isdir(splitted_data_directory):
            print('\nResetting Splitting to True ...')
            variables.splitting.set(True)

    if variables.splitting_extraction.value and variables.extraction.value:
            print('\nResetting Splitting extraction to False ...')
            variables.splitting_extraction.set(False)

    if variables.splitting.value and not variables.reduction.value:
        if not os.path.isdir(reduced_data_directory):
            print('\nResetting Reduction to True ...')
            variables.reduction.set(True)

    if variables.reduction.value:
        if not os.path.isdir(raw_data_directory):
            print('\nMoving raw images in {} ...'.format(variables.raw_data_directory.value))
            os.mkdir(raw_data_directory)
            for fits_file in glob.glob(os.path.join(directory, '*.fits')):
                shutil.move(fits_file, os.path.join(raw_data_directory, os.path.split(fits_file)[1]))

    # reduction and calibration

    if variables.reduction.value:

        # load raw images

        print('Loading raw images ...')
        data_set = DataSet(raw_data_directory)

        data_set = timing(data_set)
        data_set = bias(data_set)
        data_set = linearity(data_set)
        data_set = dark(data_set)
        data_set = gain(data_set)
        data_set = sky(data_set)
        data_set = calibration(data_set)
        data_set = flat(data_set)
        data_set = bpcr(data_set)

        # save reduced images

        print('Saving reduced frames in {} ...'.format(os.path.split(reduced_data_directory)[1]))
        data_set.save(reduced_data_directory)
        del data_set
        variables.save(os.path.join(reduced_data_directory, 'parameters.txt'))
        print('')

    # splitting

    if variables.splitting.value:

        print('Loading raw images ...')
        raw_data_set = DataSet(raw_data_directory)

        # load reduced images

        print('Loading reduced images ...')
        data_set = DataSet(reduced_data_directory)

        # split images

        if os.path.isdir(splitted_data_directory):
            backup = '{0}_{1}'.format(splitted_data_directory, time.strftime('%y-%m-%d_%H-%M-%S'))
            shutil.copytree(splitted_data_directory, backup)
            shutil.rmtree(splitted_data_directory)

        os.mkdir(splitted_data_directory)

        total_samples = raw_data_set.spectroscopic_images[0][0].header[variables.total_samples.keyword] - 1

        for sample in range(total_samples):

            print('Splitting sample {0}:'.format(sample + 1))

            for i in range(len(raw_data_set.spectroscopic_images)):

                frame = plc.copy_fits(data_set.spectroscopic_images[i])
                frame2 = plc.copy_fits(raw_data_set.spectroscopic_images[i])
                new_frame = [fits.PrimaryHDU(header=frame[0].header, data=frame[0].data),
                             fits.ImageHDU(header=frame2[1 + sample * 5].header, data=frame2[1 + sample * 5].data),
                             fits.ImageHDU(header=frame2[2 + sample * 5].header, data=frame2[2 + sample * 5].data),
                             fits.ImageHDU(header=frame2[3 + sample * 5].header, data=frame2[3 + sample * 5].data),
                             fits.ImageHDU(header=frame2[4 + sample * 5].header, data=frame2[4 + sample * 5].data),
                             fits.ImageHDU(header=frame2[5 + sample * 5].header, data=frame2[5 + sample * 5].data),
                             fits.ImageHDU(header=frame2[6 + sample * 5].header, data=frame2[6 + sample * 5].data),
                             fits.ImageHDU(header=frame2[7 + sample * 5].header, data=frame2[7 + sample * 5].data),
                             fits.ImageHDU(header=frame2[8 + sample * 5].header, data=frame2[8 + sample * 5].data),
                             fits.ImageHDU(header=frame2[9 + sample * 5].header, data=frame2[9 + sample * 5].data),
                             fits.ImageHDU(header=frame2[10 + sample * 5].header, data=frame2[10 + sample * 5].data),
                             fits.ImageHDU(header=frame['SKYAREA'].header, data=frame['SKYAREA'].data,
                                           name='SKYAREA'),
                             fits.ImageHDU(header=frame['SCANMAP'].header, data=frame['SCANMAP'].data,
                                           name='SCANMAP'),
                             fits.ImageHDU(header=frame['WMAP'].header, data=frame['WMAP'].data, name='WMAP'),
                             fits.ImageHDU(header=frame['NWMAP'].header, data=frame['NWMAP'].data, name='NWMAP'),
                             fits.ImageHDU(header=frame['BPCRMAP'].header, data=frame['BPCRMAP'].data,
                                           name='BPCRMAP'),
                             ]

                data_set.spectroscopic_images[i] = fits.HDUList(new_frame)

            data_set = bias(data_set)
            data_set = linearity(data_set)
            data_set = dark(data_set, splitting=True)
            data_set = gain(data_set)
            data_set = sky(data_set, splitting=True)
            data_set = calibration(data_set, splitting=True)
            data_set = flat(data_set)
            data_set = bpcr(data_set, splitting=True)
            data_set.save('{0}_{1}'.format(splitted_reduced_data_directory, str(sample + 1).zfill(2)))

        del data_set
        del raw_data_set

        print('')

    # create a backup if output directory exists

    if (variables.extraction.value or variables.splitting_extraction.value or
            variables.fitting_white.value or variables.fitting_spectrum.value):

        if os.path.isdir(output_directory):
            backup = '{0}_{1}'.format(output_directory, time.strftime('%y-%m-%d_%H-%M-%S'))
            shutil.copytree(output_directory, backup)
        else:
            os.mkdir(output_directory)

    # extraction of light curves

    if variables.extraction.value or variables.splitting_extraction.value:

        if variables.extraction.value:

            # load reduced images

            print('Loading reduced images ...')
            data_set = DataSet(reduced_data_directory)

            # use the photometry function to extract the light curves

            data_set, light_curve = photometry(data_set)

        else:

            print('Loading reduced images ...')
            data_set = DataSet(reduced_data_directory)

            # use the photometry function to extract the light curves

            data_set, light_curve = photometry(data_set)

            star_y_position_array = variables.y_star_array.custom_from_dictionary(light_curve)
            y_shift_error_array = variables.y_shift_error_array.custom_from_dictionary(light_curve)
            scan_length_array = variables.scan_length_array.custom_from_dictionary(light_curve)
            scan_length_error_array = variables.scan_length_error_array.custom_from_dictionary(light_curve)

            # load reduced splitted images

            print('Loading splitted reduced images ...')
            data_set = DataSet(splitted_data_directory)

            # use the split_photometry function to extract the light curves

            data_set, light_curve = split_photometry(data_set)

            star_y_position_array.to_dictionary(light_curve)
            y_shift_error_array.to_dictionary(light_curve)
            scan_length_array.to_dictionary(light_curve)
            scan_length_error_array.to_dictionary(light_curve)

        # save extraction results

        print('Saving extracted light-curves in {} ...'.format(str(os.sep).join(lc_file.split(os.sep)[-2:])))
        plc.save_dict(light_curve, lc_file)

        # plot extraction diagnostics and results

        print('Saving extraction plots in {} ...'.format(os.path.split(extraction_figures_directory)[1]))
        plot_photometry(data_set, light_curve, extraction_figures_directory)

        del data_set

        variables.save(os.path.join(output_directory, 'parameters.txt'))

        print('')

    # fitting the white and the spectral light curves

    if variables.fitting_white.value:

        light_curve = plc.open_dict(lc_file)
        light_curve = fitting(light_curve, fitting_spectrum=False)

        print('Saving fitting results in {} ...'.format(str(os.sep).join(fit_file.split(os.sep)[-2:])))
        plc.save_dict(light_curve, fit_file)

        print('Saving fitting plots in {} ...'.format(os.path.split(fitting_figures_directory)[1]))
        plot_fitting(light_curve, fitting_figures_directory)

        variables.save(os.path.join(output_directory, 'parameters.txt'))

        print('')

    # fitting the spectral light curves

    if variables.fitting_spectrum.value:

        light_curve = plc.open_dict(lc_file)
        fitted_white_light_curve = plc.open_dict(fit_file)
        light_curve = fitting(light_curve, fitted_white_light_curve=fitted_white_light_curve,
                              fitting_spectrum=variables.fitting_spectrum.value)

        print('Saving fitting results in {} ...'.format(str(os.sep).join(fit_file.split(os.sep)[-2:])))
        plc.save_dict(light_curve, fit_file)

        print('Saving fitting plots in {} ...'.format(os.path.split(fitting_figures_directory)[1]))
        plot_fitting(light_curve, fitting_figures_directory)

        variables.save(os.path.join(output_directory, 'parameters.txt'))

        print('')

    # copy results
    # create a backup if output directory exists

    if variables.output_directory_copy.value != 'False':

        if output_directory_copy != output_directory:

            if os.path.isdir(output_directory_copy):
                backup = '{0}_{1}'.format(output_directory_copy, time.strftime('%y-%m-%d_%H-%M-%S'))
                shutil.copytree(output_directory_copy, backup)
                shutil.rmtree(output_directory_copy)

            print('Copying {} to {} ...'.format(os.path.split(output_directory)[1],
                                                os.path.split(output_directory_copy)[1]))
            if os.path.isdir(output_directory):
                shutil.copytree(output_directory, output_directory_copy)

            print('')

    print('Processed directory: ', os.path.split(directory)[1])
    print('                  @: ', os.path.split(directory)[0])
    print('')

def run_test():
    dataset_files = [
        'icy021ljq_raw.fits',
        'icy021l6q_raw.fits',
        'icy021m2q_raw.fits',
        'icy021kuq_raw.fits',
        'icy021lgq_raw.fits',
        'icy021kmq_raw.fits',
        'icy021k1q_raw.fits',
        'icy021kxq_raw.fits',
        'icy021lrq_raw.fits',
        'icy021l0q_raw.fits',
        'icy021lyq_raw.fits',
        'icy021ksq_raw.fits',
        'icy021kfq_raw.fits',
        'icy021llq_raw.fits',
        'icy021jzq_raw.fits',
        'icy021ltq_raw.fits',
        'icy021kkq_raw.fits',
        'icy021laq_raw.fits',
        'icy021lkq_raw.fits',
        'icy021kaq_raw.fits',
        'icy021m3q_raw.fits',
        'icy021ktq_raw.fits',
        'icy021l7q_raw.fits',
        'icy021lfq_raw.fits',
        'icy021klq_raw.fits',
        'icy021kyq_raw.fits',
        'icy021lsq_raw.fits',
        'icy021k0q_raw.fits',
        'icy021lxq_raw.fits',
        'icy021kgq_raw.fits',
        'icy021lmq_raw.fits',
        'icy021luq_raw.fits',
        'icy021k6q_raw.fits',
        'icy021kjq_raw.fits',
        'icy021l4q_raw.fits',
        'icy021m0q_raw.fits',
        'icy021kwq_raw.fits',
        'icy021lhq_raw.fits',
        'icy021kbq_raw.fits',
        'icy021k3q_raw.fits',
        'icy021l9q_raw.fits',
        'icy021kzq_raw.fits',
        'icy021lpq_raw.fits',
        'icy021leq_raw.fits',
        'icy021koq_raw.fits',
        'icy021kdq_raw.fits',
        'icy021lnq_raw.fits',
        'icy021l2q_raw.fits',
        'icy021k8q_raw.fits',
        'icy021kqq_raw.fits',
        'icy021kiq_raw.fits',
        'icy021lcq_raw.fits',
        'icy021k5q_raw.fits',
        'icy021lvq_raw.fits',
        'icy021m1q_raw.fits',
        'icy021kvq_raw.fits',
        'icy021l5q_raw.fits',
        'icy021liq_raw.fits',
        'icy021kcq_raw.fits',
        'icy021lqq_raw.fits',
        'icy021k2q_raw.fits',
        'icy021l8q_raw.fits',
        'icy021ldq_raw.fits',
        'icy021knq_raw.fits',
        'icy021keq_raw.fits',
        'icy021loq_raw.fits',
        'icy021jxq_flt.fits',
        'icy021lzq_raw.fits',
        'icy021kpq_raw.fits',
        'icy021l3q_raw.fits',
        'icy021k9q_raw.fits',
        'icy021lbq_raw.fits',
        'icy021lwq_raw.fits',
        'icy021jyq_raw.fits',
        'icy021k4q_raw.fits',
    ]

    desktop = os.path.join(os.path.expanduser('~'), 'Desktop')
    if not os.path.isdir(desktop):
        desktop = glob.glob(os.path.join(os.path.expanduser('~'), '*', 'Desktop'))[0]
    if not os.path.isdir(desktop):
        desktop = os.path.expanduser('~')

    destination = os.path.join(desktop, 'iraclis_test_dataset_hatp26b')

    if not os.path.isdir(destination):
        os.mkdir(destination)

    if not os.path.isdir(os.path.join(destination, 'raw_data')):
        for num, dataset_file in enumerate(dataset_files):
            print('{0}/{1}: '.format(num + 1, len(dataset_files)), dataset_file)
            if not os.path.isfile(os.path.join(destination, dataset_file)):
                urlretrieve('https://mast.stsci.edu/portal/Download/file/HST/product/{0}'.format(
                    dataset_file), os.path.join(destination, dataset_file))

    parameters_file = os.path.join(destination, 'iraclis_test_dataset_hatp26b_parameters.txt')
    w = open(parameters_file, 'w')
    w.write('\n'.join([
        'data_directory                     {0}          '.format(destination),
        '# directory path                                ',
        'output_directory_copy              False        ',
        '# directory name/False                          ',
        '                                                ',
        'reduction                          True         ',
        '# True/False                                    ',
        'splitting                          False        ',
        '# True/False                                    ',
        'extraction                         True         ',
        '# True/False                                    ',
        'splitting_extraction               False        ',
        '# True/False                                    ',
        'fitting_white                      True         ',
        '# True/False                                    ',
        'fitting_spectrum                   True         ',
        '# True/False                                    ',
        '                                                ',
        'target_x_offset                    0.0          ',
        '# number                                        ',
        'target_y_offset                    0.0          ',
        '# number                                        ',
        'aperture_lower_extend              -20.0        ',
        '# number                                        ',
        'aperture_upper_extend              20.0         ',
        '# number                                        ',
        'extraction_method                  gauss        ',
        '# gauss/integral                                ',
        'extraction_gauss_sigma             47.0         ',
        '# number                                        ',
        '                                                ',
        'method                             claret       ',
        '# claret/linear/quad/sqrt                       ',
        '                                                ',
        'white_lower_wavelength             default      ',
        '# number/default                                ',
        'white_upper_wavelength             default      ',
        '# number/default                                ',
        'white_ldc1                         default      ',
        '# number/default                                ',
        'white_ldc2                         default      ',
        '# number/default                                ',
        'white_ldc3                         default      ',
        '# number/default                                ',
        'white_ldc4                         default      ',
        '# number/default                                ',
        '                                                ',
        '# Comment: When the above parameters are set '
        'to default, the white light-curve limits will '
        'be 10880.0 - 16800.0 Angstroms for G141 and '
        '8000.0 - 11500.0 Angstroms for G102.            ',
        '                                                ',
        'bins_file                          default_low  ',
        '# file path/default_high/default_low/default_'
        'vlow                                            ',
        '                                                ',
        '# Comment: A bins file should contain 4 '
        'columns: '
        '1 - bins lower edge in Angstrom, '
        '2 - bins upper edge in Angstrom, '
        '3 - first limb darkening coefficient, '
        '4 - second limb darkening coefficient, '
        '5 - third limb darkening coefficient, '
        '6 - forth limb darkening coefficient. '
        'An example of a bins file can be found in '
        'iraclis_test_dataset_hatp26b_bins.txt           ',
        '                                                ',
        'planet                             auto         ',
        '# name - no spaces/auto                         ',
        'star_teff                          auto         ',
        '# number/auto                                   ',
        'star_logg                          auto         ',
        '# number/auto                                   ',
        'star_meta                          auto         ',
        '# number/auto                                   ',
        'rp_over_rs                         auto         ',
        '# number/auto                                   ',
        'fp_over_fs                         auto         ',
        '# number/auto                                   ',
        'period                             auto         ',
        '# number/auto                                   ',
        'sma_over_rs                        auto         ',
        '# number/auto                                   ',
        'eccentricity                       auto         ',
        '# number/auto                                   ',
        'inclination                        auto         ',
        '# number/auto                                   ',
        'periastron                         auto         ',
        '# number/auto                                   ',
        'mid_time                           auto         ',
        '# number/auto                                   ',
        '                                                ',
        '# Comment: You can set any of the above 12 '
        'parameters to auto, to use the data from '
        'a built-in catalogue.                           ',
        '                                                ',
        'apply_up_down_stream_correction    False        ',
        '# True/False                                    ',
        'exclude_initial_orbits             1            ',
        '# number                                        ',
        'exclude_final_orbits               0            ',
        '# number                                        ',
        'exclude_initial_orbit_points       0            ',
        '# number                                        ',
        '                                                ',
        'mcmc_iterations                    300000       ',
        '# number                                        ',
        'mcmc_walkers                       50           ',
        '# number                                        ',
        'mcmc_burned_iterations             200000       ',
        '# number                                        ',
        'spectral_mcmc_iterations           60000        ',
        '# number                                        ',
        'spectral_mcmc_walkers              50           ',
        '# number                                        ',
        'spectral_mcmc_burned_iterations    10000        ',
        '# number                                        ',
        '                                                ',
        'first_orbit_ramp                   True         ',
        '# True/False                                    ',
        'second_order_ramp                  False        ',
        '# True/False                                    ',
        'mid_orbit_ramps                    True         ',
        '# True/False                                    ',
        '                                                ',
        'fit_ldc1                           False        ',
        '# True/False                                    ',
        'fit_ldc2                           False        ',
        '# True/False                                    ',
        'fit_ldc3                           False        ',
        '# True/False                                    ',
        'fit_ldc4                           False        ',
        '# True/False                                    ',
        'fit_inclination                    False        ',
        '# True/False                                    ',
        'fit_sma_over_rs                    False        ',
        '# True/False                                    ',
        'fit_mid_time                       True         '
    ]))
    w.close()

    os.system('iraclis -p {0}'.format(parameters_file))

max_stars_repo_name: ucl-exoplanets/Iraclis

max_issues_repo_name: ucl-exoplanets/Iraclis

max_forks_repo_name: ucl-exoplanets/Iraclis

reason: from astropy


text:
#!/usr/bin/env python

from VISA_Driver import VISA_Driver
import numpy as np

__version__ = "0.0.1"

class Error(Exception):
    pass

class Driver(VISA_Driver):
    """ This class implements the Agilent 5230 PNA driver"""

    def performOpen(self, options={}):
        """Perform the operation of opening the instrument connection"""
        # init meas param dict
        self.dMeasParam = {}
        # calling the generic VISA open to make sure we have a connection
        VISA_Driver.performOpen(self, options=options)
        # do perform get value for acquisition mode

    def performSetValue(self, quant, value, sweepRate=0.0, options={}):
        """Perform the Set Value instrument operation. This function should
        return the actual value set by the instrument"""
        # update visa commands for triggers
        if quant.name in ('S11 - Enabled', 'S21 - Enabled', 'S12 - Enabled',
                          'S22 - Enabled'):
            if self.getModel() in ('E5071C',):
                # new trace handling, use trace numbers, set all at once
                lParam = ['S11', 'S21', 'S12', 'S22']
                dParamValue = dict()
                for param in lParam:
                    dParamValue[param] = self.getValue('%s - Enabled' % param)
                dParamValue[quant.name[:3]] = value
                # add parameters, if enabled
                self.dMeasParam = dict()
                for (param, enabled) in dParamValue.items():
                    if enabled:
                        nParam = len(self.dMeasParam)+1
                        self.writeAndLog(":CALC:PAR%d:DEF %s" %
                                         (nParam, param))
                        self.dMeasParam[param] = nParam
                # set number of visible traces
                self.writeAndLog(":CALC:PAR:COUN %d" % len(self.dMeasParam))
            else:
                # get updated list of measurements in use
                self.getActiveMeasurements()
                param = quant.name[:3]
                # old-type handling of traces
                if param in self.dMeasParam:
                    # clear old measurements for this parameter
                    for name in self.dMeasParam[param]:
                        self.writeAndLog("CALC:PAR:DEL '%s'" % name)
                # create new measurement, if enabled is true
                if value:
                    newName = 'LabC_%s' % param
                    self.writeAndLog("CALC:PAR:EXT '%s','%s'" % (newName, param))
                    # show on PNA screen
                    iTrace = 1 + ['S11', 'S21', 'S12', 'S22'].index(param)
    #                sPrev = self.askAndLog('DISP:WIND:CAT?')
    #                if sPrev.find('EMPTY')>0:
    #                    # no previous traces
    #                    iTrace = 1
    #                else:
    #                    # previous traces, add new
    #                    lTrace = sPrev[1:-1].split(',')
    #                    iTrace = int(lTrace[-1]) + 1
                    self.writeAndLog("DISP:WIND:TRAC%d:FEED '%s'" % (iTrace, newName))
                    # add to dict with list of measurements
                    self.dMeasParam[param] = [newName]
        elif quant.name in ('Wait for new trace',):
            # do nothing
            pass
        else:
            # run standard VISA case
            value = VISA_Driver.performSetValue(self, quant, value, sweepRate, options)
        return value

    def performGetValue(self, quant, options={}):
        """Perform the Get Value instrument operation"""
        # check type of quantity
        if quant.name in ('S11 - Enabled', 'S21 - Enabled', 'S12 - Enabled',
                          'S22 - Enabled'):
            # update list of channels in use
            self.getActiveMeasurements()
            # get selected parameter
            param = quant.name[:3]
            value = (param in self.dMeasParam)
        elif quant.name in ('S11', 'S21', 'S12', 'S22'):
            # check if channel is on
            if quant.name not in self.dMeasParam:
                # get active measurements again, in case they changed
                self.getActiveMeasurements()
            if quant.name in self.dMeasParam:
                if self.getModel() in ('E5071C',):
                    # new trace handling, use trace numbers
                    self.writeAndLog("CALC:PAR%d:SEL" % self.dMeasParam[quant.name])
                else:
                    # old parameter handing, select parameter (use last in list)
                    sName = self.dMeasParam[quant.name][-1]
                    self.writeAndLog("CALC:PAR:SEL '%s'" % sName)
                # if not in continous mode, trig from computer
                bWaitTrace = self.getValue('Wait for new trace')
                bAverage = self.getValue('Average')
                # wait for trace, either in averaging or normal mode
                if bWaitTrace:
                    ### OLD ###
                    if bAverage:
                        # averageing trigger
                        # self.writeAndLog(':TRIG:AVER')

                        # set channels 1-4 to set event when average complete (bit 1 start)
                        self.writeAndLog(':SENS:AVER:CLE;:STAT:OPER:AVER1:ENAB 30;:ABOR;:SENS:AVER:CLE;')
                    else:
                        # self.writeAndLog(':ABOR;:INIT:CONT OFF;:INIT:IMM;')
                        self.writeAndLog('*OPC')
                        self.writeAndLog(':ABOR;:INIT:CONT ON')
                        self.writeAndLog('*OPC')
                    # wait some time before first check
                    tWait = float(self.askAndLog('SENS:SWE:TIME?'))
                    self.wait(tWait)
                    self.wait(0.03)
                    bDone = False
                    stb = 0
                    while (not bDone) and (not self.isStopped()):
                        # check if done
                        if bAverage:
                            sAverage = self.askAndLog('STAT:OPER:AVER1:COND?')
                            bDone = int(sAverage)>0
                        else:

                            stb = int(self.askAndLog('*ESR?'))
                            bDone = (stb & 1) > 0
                        if not bDone:
                            self.wait(0.1)
                    # if stopped, don't get data
                    if self.isStopped():
                        self.writeAndLog('*CLS;:INIT:CONT ON;')
                        return []
                # get data as float32, convert to numpy array
                if self.getModel() in ('E5071C',):
                    # new trace handling, use trace numbers
                    self.write(':FORM:DATA REAL32;:CALC:SEL:DATA:SDAT?', bCheckError=False)
                else:
                    # old parameter handing
                    self.write(':FORM REAL,32;CALC:DATA? SDATA', bCheckError=False)
                sData = self.read(ignore_termination=True)
                if bWaitTrace and not bAverage:
                    self.writeAndLog(':INIT:CONT ON;')
                # strip header to find # of points
                i0 = sData.find(b'#')
                # raise ValueError(sData)
                nDig = int(sData[i0+1:i0+2])
                nByte = int(sData[i0+2:i0+2+nDig])
                nData = int(nByte/4)
                nPts = int(nData/2)
                #
                # get data to numpy array
                try:
                    vData = np.frombuffer(sData[(i0+2+nDig):(i0+2+nDig+nByte)],
                                      dtype='>f', count=nData)#nData
                except ValueError:
                    Roger = open('C:\\Users\\Discord\\Desktop\\data.txt', 'w')
                    Roger.write(str(nDig)+ '\n'+ str(nByte)+ '\n'+str(nData)+ '\n'+str(nPts)+'\n')#.decode(encoding='UTF-8',errors='strict'))
                    nDig = int(sData[i0+1:i0+2])
                    nByte = int(sData[i0+2:i0+2+nDig])
                    nData = int(nByte/4)
                    nPts = int(nData/2)
                    Roger.write(str(i0)+ '\n'+ str(nDig)+ '\n'+ str(nByte)+ '\n'+str(nData)+ '\n'+str(nPts))
                    Roger.close()

                # data is in I0,Q0,I1,Q1,I2,Q2,.. format, convert to complex
                # raise ValueError(len(vData))
                # self.wait(1)
                # nPts = int(1)
                mC = vData.reshape((nPts,2))
                vComplex = mC[:,0] + 1j*mC[:,1]
                # get start/stop frequencies
                startFreq = self.readValueFromOther('Start frequency')
                stopFreq = self.readValueFromOther('Stop frequency')
                sweepType = self.readValueFromOther('Sweep type')
                # if log scale, take log of start/stop frequencies
                logX = (sweepType == 'Log')
                value = quant.getTraceDict(vComplex, x0=startFreq, x1=stopFreq,
                                           logX=logX)
            else:
                # not enabled, return empty array
                value = quant.getTraceDict([])
        elif quant.name in ('Wait for new trace',):
            # do nothing, return local value
            value = quant.getValue()
        else:
            # for all other cases, call VISA driver
            value = VISA_Driver.performGetValue(self, quant, options)
        return value

    def getActiveMeasurements(self):
        """Retrieve and a list of measurement/parameters currently active"""
        # proceed depending on model
        if self.getModel() in ('E5071C',):
            # in this case, meas param is just a trace number
            self.dMeasParam = {}
            # get number or traces
            nTrace = int(self.askAndLog(":CALC:PAR:COUN?"))
            # get active trace names, one by one
            for n in range(nTrace):
                sParam = self.askAndLog(":CALC:PAR%d:DEF?" % (n+1))
                self.dMeasParam[sParam] = (n+1)
        else:
            sAll = self.askAndLog("CALC:PAR:CAT:EXT?")
            # strip "-characters
            sAll = sAll[1:-1]
            # parse list, format is channel, parameter, ...
            self.dMeasParam = {}
            lAll = sAll.split(',')
            nMeas = len(lAll)//2
            for n in range(nMeas):
                sName = lAll[2*n]
                sParam = lAll[2*n + 1]
                if sParam not in self.dMeasParam:
                    # create list with current name
                    self.dMeasParam[sParam] = [sName,]
                else:
                    # add to existing list
                    self.dMeasParam[sParam].append(sName)

if __name__ == '__main__':
    pass


text:
import numpy as np
import os, warnings
import ctypes
from scipy.sparse import csr_matrix, isspmatrix_csr
from . import _cpp_interface

__all__ = ["read_sparse", "write_sparse"]

### Main functions
def read_sparse(
        file, multilabel=False, has_qid=False, integer_labels=False,
        index1=True, sort_indices=True, ignore_zeros=True,
        min_cols=0, min_classes=0, limit_nrows=0, no_trailing_ws = False,
        use_int64=False, use_double=True, use_cpp=True, from_string=False
    ):
    """
    Read Sparse Matrix from Text File

    Read a labelled sparse CSR matrix in text format as used by libraries
    such as SVMLight, LibSVM, ThunderSVM, LibFM, xLearn, XGBoost, LightGBM, and more.

    The format is as follows:

    <label(s)> <column>:<value> <column>:<value> ...

    Example line (row):

    1 1:1.234 3:20

    This line denotes a row with label (target variable) equal to 1, a value
    for the first column of 1.234, a value of zero for the second column (which is
    missing), and a value of 20 for the third column.

    The labels might be decimal (for regression), and each row might contain more
    than one label (must be integers in this case), separated by commas *without*
    spaces inbetween - e.g.:

    1,5,10 1:1.234 3:20

    This line indicates a row with labels 1, 5, and 10 (for multi-class classification).
    If the line has no labels, it should still include a space before the features.

    The rows might additionally contain a 'qid' parameter as used in ranking algorithms,
    which should always lay inbetween the labels and the features and must be an integer - e.g.:

    1 qid:2 1:1.234 3:20

    The file might optionally contain a header as the first line with metadata
    (number of rows, number of columns, number of classes). Presence of a header will be
    automatically detected, and is recommended to include it for speed purposes. Datasets
    from the extreme classification repository (see references) usually include such a header.

    Lines might include comments, which start after a '#' character. Lines consisting
    of only a '#' will be ignored. When reading from a file, such file might have a
    BOM (information about encoding uses in Windows sytems), which will be automatically
    skipped.

    Some extra notes - this function:
        - Will not make any checks for negative column indices.
        - Will be able to read numeric values in scientific notation only if the E is capitalized.
        - Will fill missing labels with NAs when passing ``multilabel=False``.
        - Will fill with zeros (empty values) the lines that are empty (that is,
          they generate a row in the data), but will ignore (that is, will not generate
          a row in the data) the lines that start with '#'.

    On 64-bit Windows systems, if compiling the library with a compiler other than MinGW
    or MSVC, it will not be able to read files larger than 2GB and might crash the system
    if the file is larger.

    If the file contains a header, and this header denotes a larger number of columns
    or of labels than the largest index in the data, the resulting object will have
    this dimension set according to the header. The third entry in the header (number
    of classes/labels) will be ignored when passing ``multilabel=False``.

    The function uses different code paths when reading from a file or from a string,
    and there might be slight differences between the obtained results from them.
    For example, reading from a file might produce the desired output if the file
    uses tabs as separators instead of spaces (not supported by most other software
    and not standard), whereas reading from a string will not.
    If any such difference is encountered, please submit a bug report in the
    package's GitHub page.

    Parameters
    ----------
    file : str, None, or file connection
        Either a file path from which the data will be read, or a string
        containing the text from which the data will be read.
        In the latter case, must pass ``from_string=True``.
    multilabel : bool
        Whether the input file can have multiple labels per observation.
        If passing ``multilabel=False`` and it turns out to have multiple labels, will only
        take the first one for each row. If the labels are non-integers or have decimal point,
        the results will be invalid.
    has_qid : bool
        Whether the input file has 'qid' field (used for ranking). If passing
        ``False`` and the file does turns out to have 'qid', the features will not be read for any
        observations.
    integer_labels : bool
        Whether to output the observation labels as integers.
    index1 : bool
        Whether the input file uses numeration starting at 1 for the column
        numbers (and for the label numbers when passing ``multilabel=True``). This is usually
        the case for files downloaded from the repositories in the references. The function
        will check for whether any of the column indices is zero, and will ignore this
        option if so (i.e. will assume it is ``False``).
    sort_indices : bool
        Whether to sort the indices of the columns after reading the data.
        These should already be sorted in the files from the repositories in the references.
    ignore_zeros : bool
        Whether to avoid adding features which have a value of zero.
        If the zeros are caused due to numerical rounding in the software that wrote the
        input file, they can be post-processed by passing ``ignore_zeros=False`` and then
        something like 'X.data[X.data == 0] = 1e-8'.
    min_cols : int
        Minimum number of columns that the output ``X`` object should have,
        in case some columns are all missing in the input data.
    min_classes : int
        Minimum number of columns that the output ``y`` object should have,
        in case some columns are all missing in the input data. Only used when passing
        ``multilabel=True``.
    limit_nrows : int
        Maximum number of rows to read from the data. If there are more than this
        number of rows, it will only read the first 'limit_nrows' rows.
        If passing zero (the default), there will be no row limit.
    no_trailing_ws : bool
        Whether to assume that lines in the file will never have extra whitespaces
        at the end before a new line. Parsing large files with this option set to
        'True' can be 1.5x faster, but if the file does turn up to have e.g. extra
        spaces at the end of lines, the results will be incorrect.
    use_int64 : bool
        Whether to use 64-bit integers for column and label indices (when passing
        ``multilabel=True``). If passing ``False``, will use the machine's 'int' type
        (typically np.int32 but this could differ in non-standard CPU architectures).
        Using 'int' is faster and uses less memory, but cannot store values higher
        than the machine's 'INT_MAX' (typically 2^31-1, or around 2.2 billion).
    use_double : bool
        Whether to use C 'double' type (typically np.float64) for numeric values.
        If passing ``False``, will use C 'float' type (typically np.float32), which
        uses less memory but might be very slightly slower to parse. Most machine
        learning software for Python works with 'double' data.
    use_cpp : bool
        Whether to use C++ functions directly for file IO. If passing ``False``, will
        read the contents using Python's own functions into a string variable, from
        which the data will then be read. If passing ``True``, will parse from the file
        directly, which is faster and uses less memory. Using the C++ engine can have
        issues on Windows if the file is larger than 2GB and the library was compiled
        with something other than MSVC or MinGW.
    from_string : bool
        Whether to read the data from a string variable instead of a file.
        If passing ``from_string=True``, then ``file`` is assumed to be a variable with the
        data contents on it.

    Returns
    -------
    data : dict
        A dict with the following entries:
            - 'X' : the features, as a CSR Matrix from SciPy, with data of type ``np.float64``
              or ``np.float32`` depending on argument ``use_double``.
            - 'y' : the labels. If passing ``multilabel=False`` (the default), will be a vector
              (NumPy 1-d array, with dtype float64 when passing ``integer_labels=False``, or
              dtype `equal to the indices of 'X' when passing ``integer_labels=True``),
              otherwise will be a binary CSR Matrix (same dtype as the values of 'X').
            - 'qid' : the query IDs used for ranking, as an integer vector.
              This entry will **only** be present when passing ``has_qid=True``

    References
    ----------
    Datasets in this format can be found here:
        - LibSVM Data: https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets
        - Extreme Classification Repository: http://manikvarma.org/downloads/XC/XMLRepository.html

    The format is also described at the SVMLight webpage: http://svmlight.joachims.org
    """
    multilabel      =  bool(multilabel)
    has_qid         =  bool(has_qid)
    integer_labels  =  bool(integer_labels)
    index1          =  bool(index1)
    sort_indices    =  bool(sort_indices)
    ignore_zeros    =  bool(ignore_zeros)
    from_string     =  bool(from_string)

    min_cols = int(min_cols)
    min_classes = int(min_classes)
    limit_nrows = int(limit_nrows)
    if (min_cols < 0):
        raise ValueError("'min_cols' must be a non-negative integer.")
    if (min_classes < 0):
        raise ValueError("'min_classes' must be a non-negative integer.")
    if (limit_nrows < 0):
        raise ValueError("'limit_nrows' must be a non-negative integer.")

    if ("read" in dir(file)):
        file = file.read()
        from_string = True
    elif from_string and isinstance(file, bytes):
        file = file.decode()

    if not from_string:
        file = os.path.expanduser(file)
        if not os.path.exists(file):
            raise ValueError("Error: file does not exist.")

        if multilabel:
            read_func = _cpp_interface.read_multi_label_py
        else:
            read_func = _cpp_interface.read_single_label_py

    else: ## from_string
        if (not isinstance(file, str)) and (not isinstance(file, bytes)):
            raise ValueError("Error: 'file' is not a string.")

        if multilabel:
            read_func = _cpp_interface.read_multi_label_from_str_py
        else:
            read_func = _cpp_interface.read_single_label_from_str_py

    r = read_func(
            file,
            ignore_zero_valued = ignore_zeros,
            sort_indices = sort_indices,
            text_is_base1 = index1,
            assume_no_qid = not has_qid,
            use_int64 = use_int64,
            use_double = use_double,
            assume_trailing_ws = not no_trailing_ws,
            limit_nrows = limit_nrows
    )

    if len(r) == 0:
        raise ValueError("Error: could not read file successfully.")

    r["ncols"] = max(r["ncols"], min_cols)
    r["nclasses"] = max(r["nclasses"], min_classes)

    features = csr_matrix((r["values"], r["indices"], r["indptr"]),
                          shape=(r["nrows"], r["ncols"]))

    if multilabel:
        labels = csr_matrix((np.ones_like(r["indices_lab"], dtype=r["values"].dtype),
                             r["indices_lab"], r["indptr_lab"]),
                            shape=(r["nrows"], r["nclasses"]))
    else:
        labels = r["labels"]
        if integer_labels:
            if use_int64:
                labels = labels.astype(np.int64)
            else:
                labels = labels.astype(ctypes.c_int)

    if (not has_qid) and (r["indptr"].shape[0] > 1) and (r["indptr"][0] == r["indptr"][-1]):
        warnings.warn("Data has empty features. Perhaps the file has 'qid' field?")

    if not has_qid:
        return {"X":features, "y":labels}
    else:
        return {"X":features, "y":labels, "qid":r["qid"]}

def write_sparse(
        file, X, y, qid=None, integer_labels=True,
        index1=True, sort_indices=True, ignore_zeros=True,
        add_header=False, decimal_places=8,
        use_cpp=True, append=False, to_string=False
    ):
    """
    Write Sparse Matrix in Text Format

    Write a labelled sparse matrix into text format as used by software
    such as SVMLight, LibSVM, ThunderSVM, LibFM, xLearn, XGBoost, LightGBM, and others - i.e.:

    <labels(s)> <column:value> <column:value> ...

    For more information about the format and usage examples, see the documentation for 'read_sparse'.

    Can write labels for regression, classification (binary, multi-class, and multi-label),
    and ranking (with 'qid'), but note that most software that makes use of this data
    format supports only regression and binary classification.

    Note
    ----
    Be aware that writing sparse matrices to text is not a lossless operation
    - that is, some information might be lost due to numeric precision, and metadata such
    as row and column names will not be saved. It is recommended to use ``save_npz`` or similar
    for saving data between Python sessions, or to use binary formats for passing between
    different software such as Python->R.

    Note
    ----
    The option ``ignore_zeros`` is implemented heuristically, by comparing
    'abs(x) >= 10^(-decimal_places)/2', which might not match exactly with
    the rounding that is done implicitly in string conversions in the libc/libc++
    functions - thus there might still be some corner cases of all-zeros written into
    features if the (absolute) values are very close to the rounding threshold.

    Note
    ----
    The function uses different code paths when writing to a file or to a string,
    and there might be slight differences between the generated texts from them.
    If any such difference is encountered, please submit a bug report in the
    package's GitHub page.

    Parameters
    ----------
    file : str or None
        Output file path into which to write the data.
        Will be ignored when passing ``to_string=True``.
    X : CSR(n_samples, n_labels)
        Sparse data to write. Can be a sparse matrix from SciPy, a dense array from NumPy,
        or a DataFrame from Pandas, but will be converted to a CSR matrix if it isn't already.

        If ``X`` is a vector (1-d NumPy array), will be assumed to
        be a row vector and will thus write one row only.
    y : array(n_samples,) or CSR(n_samples, n_labels)
        Labels for the data. Can be passed as a vector
        if each observation has one label, or as a sparse or dense matrix (same format as ``X``)
        if each observation can have more than 1 label. In the latter case, only the non-missing
        column indices will be written, while the values are ignored.
    qid : None or array(n_samples,)
        Secondary label information used for ranking algorithms. Must be an integer vector
        if passed. Note that not all software supports this.
    integer_labels : bool
        Whether to write the labels as integers. If passing ``False``, they will
        have a decimal point regardless of whether they are integers or not. If the file is meant
        to be used for a classification algorithm, one should pass ``True`` here (the default).
         For multilabel classification, the labels will always be written as integers.
    index1 : bool
        Whether the column and label indices (if multi-label) should have numeration
        starting at 1. Most software assumes this is ``True``.
    sort_indices : bool
        Whether to sort the indices of ``X`` (and of ``y`` if multi-label) before
        writing the data. Note that this will cause in-place modifications if either ``X`` or ``y``
        are passed as CSR matrices.
    ignore_zeros : bool
        Whether to ignore (not write) features with a value of zero
        after rounding to the specified decimal places.
    add_header : bool
        Whether to add a header with metadata as the first line (number of rows,
        number of columns, number of classes). If passing ``integer_label=False`` and ``y`` is a
        vector, will write zero as the number of labels. This is not supported by most software.
    decimal_places : int
        Number of decimal places to use for numeric values. All values
        will have exactly this number of places after the decimal point. Be aware that values
        are rounded and might turn to zeros (will be skipped by default) if they are too small
        (one can do something like
        'X.data = np.where(X.data >= 0, X.data.clip(min=1e-8), X.data.clip(max=-1e-8))' to avoid this).
    use_cpp : bool
        Whether to use C++ functions directly for IO. If passing ``False``, will first write
        the output text into a Python string and then use Python's IO functions to write
        it to a file. If passing ``True``, will write to the file directly bypassing Python.
        Passing ``True`` is faster, but less resilient to errors.
    append : bool
        Whether to append text at the end of the file instead of overwriting or
        creating a new file. Ignored when passing ``to_string=True``.
    to_string : bool
        Whether to write the result into a string (which will be returned
        from the function) instead of into a file.

    Returns
    -------
    output : bool or str
        If passing ``to_string=False`` (the default), will return ``True`` if
        it completes successfully, or raise an error if it doesn't. If passing
        ``to_string=False``, will return the encoded matrix in text as a string
        variable.
    """
    if (X is None) or (y is None):
        raise ValueError("Must pass 'X' and 'y'.")

    decimal_places = int(decimal_places)
    if (decimal_places < 0):
        raise ValueError("'decimal_places' must be non-negative.")
    if (decimal_places > 20):
        warnings.warn("'decimal_places' is greater than 20.")

    integer_labels  =  bool(integer_labels)
    index1          =  bool(index1)
    sort_indices    =  bool(sort_indices)
    ignore_zeros    =  bool(ignore_zeros)
    add_header      =  bool(add_header)

    write_str_to_file_conn = False
    return_string = to_string
    if ("write" in dir(file)):
        write_str_to_file_conn = True
        to_string = True
    if not use_cpp:
        to_string = True

    ### Note: this check should be made right here due to potential
    ### in-place modifications of the data which would render the input unusable
    if sort_indices:
        if isspmatrix_csr(y):
            y.sort_indices()
        if isspmatrix_csr(X):
            X.sort_indices()
        sort_indices = False

    indptr, indices, values, nrows, ncols = _process_X(X)
    indptr_lab, indices_lab, labels, nrows_y, nclasses = _process_y(y, add_header, integer_labels)
    qid = _process_qid(qid)
    if nrows != nrows_y:
        raise ValueError("'X' and 'y' must have the same number of rows.")
    if (qid.shape[0]) and (qid.shape[0] != nrows):
        raise ValueError("'X' and 'qid' must have the same number of rows.")

    indptr, indices, indptr_lab, indices_lab, qid = \
        _as_highest_dtype(
            indptr,
            indices,
            indptr_lab,
            indices_lab,
            qid
        )

    if not to_string:
        if (add_header) and (append) and (os.path.exists(file)):
            warnings.warn("Warning: adding header to existing file with 'append=TRUE'.")

        if indptr_lab is not None:
            success = _cpp_interface.write_multi_label_py(
                file,
                indptr,
                indices,
                values,
                indptr_lab,
                indices_lab,
                qid,
                ncols,
                nclasses,
                ignore_zero_valued = ignore_zeros,
                sort_indices = sort_indices,
                text_is_base1 = index1,
                add_header = add_header,
                decimal_places = decimal_places,
                append = append
            )
        else:
            success = _cpp_interface.write_single_label_py(
                file,
                indptr,
                indices,
                values,
                labels,
                qid,
                ncols,
                nclasses,
                ignore_zero_valued = ignore_zeros,
                sort_indices = sort_indices,
                text_is_base1 = index1,
                add_header = add_header,
                decimal_places = decimal_places,
                append = append
            )
        if not success:
            raise ValueError("Error: file write failed.")
        return success

    else: ### to_string=True
        if indptr_lab is not None:
            res = _cpp_interface.write_multi_label_to_str_py(
                indptr,
                indices,
                values,
                indptr_lab,
                indices_lab,
                qid,
                ncols,
                nclasses,
                ignore_zero_valued = ignore_zeros,
                sort_indices = sort_indices,
                text_is_base1 = index1,
                add_header = add_header,
                decimal_places = decimal_places,
                append = append
            )
        else:
            res = _cpp_interface.write_single_label_to_str_py(
                indptr,
                indices,
                values,
                labels,
                qid,
                ncols,
                nclasses,
                ignore_zero_valued = ignore_zeros,
                sort_indices = sort_indices,
                text_is_base1 = index1,
                add_header = add_header,
                decimal_places = decimal_places,
                append = append
            )

        if return_string:
            return res.decode()

        if write_str_to_file_conn:
            file.write(res)
            return True

        if use_cpp:
            with open(file, "a" if append else "w") as f:
                f.write(res)
            return True
        else:
            return res

### Helpers
dtypes_indices = [ctypes.c_int, np.int64, ctypes.c_size_t]
dtypes_values = [np.float32, np.float64]
dtypes_labels = dtypes_indices + dtypes_values

def _process_X(X):
    if not isspmatrix_csr(X):
        X = csr_matrix(X)
    return (
        X.indptr,
        X.indices,
        X.data if X.data.dtype in dtypes_values else X.data.astype(np.float64),
        X.shape[0], X.shape[1]
    )

def _process_y(y, add_header=False, integer_labels=True):
    if y.__class__.__name__ == "Categorical":
        y = y.codes
    if y.__class__.__name__ == "Series":
        y = y.to_numpy()
    if isinstance(y, list) or isinstance(y, tuple):
        y = np.array(y)
    if np.isscalar(y):
        y = np.array([y])

    if isinstance(y, np.ndarray):
        if (len(y.shape) == 0):
            y = y.reshape(-1)
        if (len(y.shape) > 2):
            raise ValueError("'y' cannot have dimension greater than 2.")
        ### will reach here when 'y' is a vector
        if (len(y.shape) == 1):
            if y.dtype not in dtypes_labels:
                if integer_labels:
                    y = y.astype(np.int64)
                else:
                    y = y.astype(np.float64)

            nclasses = 0
            if (add_header) and (integer_labels):
                if np.any(np.isinf(y)):
                    nclasses = 0
                elif np.any(y < 0):
                    nclasses = np.unique(y).shape[0]
                else:
                    nclasses = y.max()
                    if 0 in y:
                        nclasses += 1
                    nclasses = max(nclasses, 1)
                    if np.isinf(nclasses) or np.isnan(nclasses):
                        nclasses = 0
                    nclasses = int(nclasses)

            if (integer_labels) and (y.dtype not in dtypes_indices):
                y = y.astype(np.int64)

            return None, None, y, y.shape[0], nclasses

    if not isspmatrix_csr(y):
        y = csr_matrix(y)
    return (
        y.indptr,
        y.indices,
        None, y.shape[0], y.shape[1]
    )

def _process_qid(qid):
    if qid is None:
        return np.empty(0, np.int64)
    if qid.__class__.__name__ == "Categorical":
        qid = qid.codes
    if qid.__class__.__name__ == "Series":
        qid = qid.to_numpy()
    if np.isscalar(qid):
        qid = np.array([qid])
    if not isinstance(qid, np.ndarray):
        qid = np.array(qid)
    if len(qid.shape) != 1:
        warnings.warn("'qid' is not unidimensional, will reshape.")
        qid = qid.reshape(-1)
    return qid

def _as_highest_dtype(
        indptr,
        indices,
        indptr_lab,
        indices_lab,
        qid
    ):
    highest_int_type = ctypes.c_int
    def promote_dtype(dtype, curr_highest):
        if np.iinfo(dtype).max > np.iinfo(curr_highest).max:
            return dtype
        else:
            return curr_highest

    if indptr.dtype in dtypes_indices:
        highest_int_type = promote_dtype(indptr.dtype, highest_int_type)
    else:
        highest_int_type = promote_dtype(np.int64, highest_int_type)
    if indices.dtype in dtypes_indices:
        highest_int_type = promote_dtype(indices.dtype, highest_int_type)
    else:
        highest_int_type = promote_dtype(np.int64, highest_int_type)
    if indptr_lab is not None:
        if indptr_lab.dtype in dtypes_indices:
            highest_int_type = promote_dtype(indptr_lab.dtype, highest_int_type)
        else:
            highest_int_type = promote_dtype(np.int64, highest_int_type)
    if indices_lab is not None:
        if indices_lab.dtype in dtypes_indices:
            highest_int_type = promote_dtype(indices_lab.dtype, highest_int_type)
        else:
            highest_int_type = promote_dtype(np.int64, highest_int_type)
    if qid.shape[0]:
        if qid.dtype in dtypes_indices:
            highest_int_type = promote_dtype(qid.dtype, highest_int_type)
        else:
            highest_int_type = promote_dtype(np.int64, highest_int_type)

    if indptr.dtype != highest_int_type:
        indptr = indptr.astype(highest_int_type)
    if indices.dtype != highest_int_type:
        indices = indices.astype(highest_int_type)
    if indptr_lab is not None:
        if indptr_lab.dtype != highest_int_type:
            indptr_lab = indptr_lab.astype(highest_int_type)
    if indices_lab is not None:
        if indices_lab.dtype != highest_int_type:
            indices_lab = indices_lab.astype(highest_int_type)
    if qid.dtype != highest_int_type:
        qid = qid.astype(highest_int_type)

    return indptr, indices, indptr_lab, indices_lab, qid

max_stars_repo_name: david-cortes/readsparse

max_issues_repo_name: david-cortes/readsparse

max_forks_repo_name: david-cortes/readsparse

reason: import numpy,from scipy


text:
"""
utils
-----

Utility functions for data loading and cleaning.

Contents:
    _check_str_similarity,
    _check_str_args,
    graph_lda_topic_evals,
    english_names_list
"""

from difflib import SequenceMatcher

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from gensim import corpora
from gensim.models import CoherenceModel
from gensim.models.ldamulticore import LdaMulticore
from tqdm.auto import tqdm

def _check_str_similarity(str_1, str_2):
    """Checks the similarity of two strings."""
    return SequenceMatcher(None, str_1, str_2).ratio()

def _check_str_args(arguments, valid_args):
    """
    Checks whether a str argument is valid, and makes suggestions if not.
    """
    if isinstance(arguments, str):
        if arguments in valid_args:
            return arguments

        suggestions = []
        for v in valid_args:
            similarity_score = round(_check_str_similarity(str_1=arguments, str_2=v), 2)
            arg_and_score = (v, similarity_score)
            suggestions.append(arg_and_score)

        ordered_suggestions = sorted(suggestions, key=lambda x: x[1], reverse=True)

        print(f"'{arguments}' is not a valid argument for the given function.")
        print(f"The closest valid options to '{arguments}' are:")
        for item in ordered_suggestions[:5]:
            print(item)

        return

    elif isinstance(arguments, list):
        # Check arguments, and remove them if they're invalid.
        for a in arguments:
            _check_str_args(arguments=a, valid_args=valid_args)

        return arguments

def graph_lda_topic_evals(
    corpus=None,
    num_topic_words=10,
    topic_nums_to_compare=None,
    metrics=True,
    verbose=True,
    **kwargs,
):
    """
    Graphs metrics for the given models over the given number of topics.

    Parameters
    ----------
        corpus : list of lists (default=None)
            The text corpus over which analysis should be done.

        num_topic_words : int (default=10)
            The number of keywords that should be extracted.

        topic_nums_to_compare : list (default=None)
            The number of topics to compare metrics over.

            Note: None selects all numbers from 1 to num_topic_words.

        metrics : str or bool (default=True: all metrics)
            The metrics to include.

            Options:
                stability: model stability based on Jaccard similarity.

                coherence: how much the words associated with model topics co-occur.

        verbose : bool (default=True)
            Whether to show a tqdm progress bar for the query.

        **kwargs : keyword arguments
            Arguments correspoding to gensim.models.ldamulticore.LdaMulticore.

    Returns
    -------
        ax : matplotlib axis
            A graph of the given metrics for each of the given models based on each topic number.
    """
    assert (
        metrics == "stability" or metrics == "coherence" or metrics == True
    ), "An invalid value has been passed to the 'metrics' argument - please choose from 'stability', 'coherence', or True for both."

    if metrics == True:
        metrics = ["stability", "coherence"]

    if not isinstance(corpus[0], list):
        corpus = [c.split() for c in corpus]

    def jaccard_similarity(topic_1, topic_2):
        """
        Derives the Jaccard similarity of two topics.

        Notes
        -----
            Jaccard similarity:
                - A statistic used for comparing the similarity and diversity of sample sets.
                - J(A,B) = (A ∩ B)/(A ∪ B).
                - Goal is low Jaccard scores for coverage of the diverse elements.
        """
        # Fix for cases where there are not enough texts for clustering models.
        if topic_1 == [] and topic_2 != []:
            topic_1 = topic_2
        if topic_1 != [] and topic_2 == []:
            topic_2 = topic_1
        if topic_1 == [] and topic_2 == []:
            topic_1, topic_2 = ["_None"], ["_None"]
        intersection = set(topic_1).intersection(set(topic_2))
        num_intersect = float(len(intersection))

        union = set(topic_1).union(set(topic_2))
        num_union = float(len(union))

        return num_intersect / num_union

    plt.figure()

    dirichlet_dict = corpora.Dictionary(corpus)
    bow_corpus = [dirichlet_dict.doc2bow(text) for text in corpus]

    # Add an extra topic so that metrics can be calculated all inputs.
    if topic_nums_to_compare == None:
        topic_nums_to_compare = list(range(num_topic_words + 1)[1:])
    else:
        topic_nums_to_compare.append(topic_nums_to_compare[-1] + 1)

    LDA_models = {}
    LDA_topics = {}
    disable = not verbose
    for i in tqdm(
        iterable=topic_nums_to_compare, desc="LDA models ran", disable=disable
    ):
        LDA_models[i] = LdaMulticore(
            corpus=bow_corpus, id2word=dirichlet_dict, num_topics=i, **kwargs
        )

        shown_topics = LDA_models[i].show_topics(
            num_topics=i, num_words=num_topic_words, formatted=False
        )
        LDA_topics[i] = [[word[0] for word in topic[1]] for topic in shown_topics]

    LDA_stability = {}
    for i in range(0, len(topic_nums_to_compare) - 1):
        jaccard_sims = []
        for t1, topic1 in enumerate(  # pylint: disable=unused-variable
            LDA_topics[topic_nums_to_compare[i]]
        ):
            sims = []
            for t2, topic2 in enumerate(  # pylint: disable=unused-variable
                LDA_topics[topic_nums_to_compare[i + 1]]
            ):
                sims.append(jaccard_similarity(topic1, topic2))

            jaccard_sims.append(sims)

        LDA_stability[topic_nums_to_compare[i]] = jaccard_sims

    mean_stabilities = [
        np.array(LDA_stability[i]).mean() for i in topic_nums_to_compare[:-1]
    ]

    coherences = [
        CoherenceModel(
            model=LDA_models[i],
            texts=corpus,
            dictionary=dirichlet_dict,
            coherence="c_v",
        ).get_coherence()
        for i in topic_nums_to_compare[:-1]
    ]

    # Limit topic numbers to the number of keywords.
    coh_sta_diffs = [
        coherences[i] - mean_stabilities[i]
        for i in range(len(topic_nums_to_compare))[:-1]
    ]
    coh_sta_max = max(coh_sta_diffs)
    coh_sta_max_idxs = [i for i, j in enumerate(coh_sta_diffs) if j == coh_sta_max]

    # Choose less topics in case there's more than one max.
    ideal_topic_num_index = coh_sta_max_idxs[0]
    ideal_topic_num = topic_nums_to_compare[ideal_topic_num_index]

    ax = sns.lineplot(
        x=topic_nums_to_compare[:-1], y=mean_stabilities, label="Average Topic Overlap"
    )
    ax = sns.lineplot(
        x=topic_nums_to_compare[:-1], y=coherences, label="Topic Coherence"
    )

    ax.axvline(x=ideal_topic_num, label="Ideal Number of Topics", color="black")
    ax.axvspan(
        xmin=ideal_topic_num - 1, xmax=ideal_topic_num + 1, alpha=0.5, facecolor="grey"
    )

    # Set plot limits.
    y_max = max(max(mean_stabilities), max(coherences)) + (
        0.10 * max(max(mean_stabilities), max(coherences))
    )
    ax.set_ylim([0, y_max])
    ax.set_xlim([topic_nums_to_compare[0], topic_nums_to_compare[-1] - 1])

    ax.axes.set_title("Method Metrics per Number of Topics", fontsize=25)
    ax.set_ylabel("Metric Level", fontsize=20)
    ax.set_xlabel("Number of Topics", fontsize=20)
    plt.legend(fontsize=20)

    return ax


text:
from dataclasses import dataclass
from typing import Any, Callable, List, Tuple, Union

import albumentations as alb
import numpy as np
import torch
from albumentations import Compose, DualTransform
from albumentations.pytorch import ToTensorV2

from flash.core.data.io.input_transform import InputTransform
from flash.core.data.io.output_transform import OutputTransform
from flash.image.segmentation.input_transform import prepare_target, remove_extra_dimensions
from torch import nn, Tensor

COLOR_MEAN: float = 0.349977
COLOR_STD: float = 0.215829

class FlashAlbumentationsAdapter(nn.Module):
    """This is temporary class for Flash new release."""

    # mapping from albumentations to Flash
    TRANSFORM_INPUT_MAPPING = {"image": "input", "mask": "target"}

    def __init__(self, transform, mapping: dict = None, image_key: str = "input"):
        super().__init__()
        if not isinstance(transform, (list, tuple)):
            transform = [transform]
        self.transform = Compose(list(transform) + [ToTensorV2()])
        self._img_key = image_key
        if not mapping:
            mapping = self.TRANSFORM_INPUT_MAPPING
        self._mapping_rev = mapping
        self._mapping = {v: k for k, v in mapping.items()}

    @staticmethod
    def _image_transform(x: Tensor) -> np.ndarray:
        if x.ndim == 3 and x.shape[0] < 4:
            return x.permute(1, 2, 0).numpy()
        return x.numpy()

    @staticmethod
    def _to_numpy(t: Union[Tensor, np.ndarray]) -> np.ndarray:
        return t.numpy() if isinstance(t, Tensor) else t

    def forward(self, x: Any) -> Any:
        if isinstance(x, dict):
            x_ = {self._mapping.get(k, k): self._to_numpy(x[k]) for k in self._mapping if k in x and k != self._img_key}
            if self._img_key in self._mapping and self._img_key in x:
                x_.update({self._mapping[self._img_key]: self._image_transform(x[self._img_key])})
        else:
            x_ = {"image": self._image_transform(x)}
        x_ = self.transform(**x_)
        if isinstance(x, dict):
            x.update({self._mapping_rev.get(k, k): x_[k] for k in self._mapping_rev if k in x_})
        else:
            x = x_["image"]
        return x

class NormImage(DualTransform):
    def __init__(self, quantile: float = 0.01, norm: bool = True, always_apply=True, p=1):
        super().__init__(always_apply, p)
        self.quantile = quantile
        self.norm = norm

    def apply(self, img, **params):
        if self.quantile > 0:
            q_low, q_high = np.percentile(img, [self.quantile * 100, (1 - self.quantile) * 100])
            img = np.clip(img, q_low, q_high)
        if self.norm:
            v_min, v_max = np.min(img), np.max(img)
            img = (img - v_min) / float(v_max - v_min)
        return img

    def apply_to_mask(self, mask, **params):
        return mask

@dataclass
class TractFlashSegmentationTransform(InputTransform):
    # https://albumentations.ai/docs/examples/pytorch_semantic_segmentation

    image_size: Tuple[int, int] = (224, 224)
    color_mean: float = 0.349977
    color_std: float = 0.215829

    def train_per_sample_transform(self) -> Callable:
        return FlashAlbumentationsAdapter(
            [
                NormImage(always_apply=True),
                alb.Resize(*self.image_size),
                alb.VerticalFlip(p=0.5),
                alb.HorizontalFlip(p=0.5),
                alb.RandomRotate90(p=0.5),
                alb.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.03, rotate_limit=5, p=1.0),
                alb.GaussNoise(var_limit=(0.001, 0.005), mean=0, per_channel=False, p=1.0),
                alb.OneOf(
                    [
                        alb.GridDistortion(num_steps=5, distort_limit=0.05, p=1.0),
                        alb.ElasticTransform(alpha=1, sigma=50, alpha_affine=50, p=1.0),
                    ],
                    p=0.25,
                ),
                alb.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.8),
                alb.Normalize(mean=[self.color_mean] * 3, std=[self.color_std] * 3, max_pixel_value=1.0),
            ]
        )

    def per_sample_transform(self) -> Callable:
        return FlashAlbumentationsAdapter(
            [
                NormImage(always_apply=True),
                alb.Resize(*self.image_size),
                alb.Normalize(mean=[self.color_mean] * 3, std=[self.color_std] * 3, max_pixel_value=1.0),
            ]
        )

    def target_per_batch_transform(self) -> Callable:
        return prepare_target

    def predict_per_batch_transform(self) -> Callable:
        return remove_extra_dimensions

    def serve_per_batch_transform(self) -> Callable:
        return remove_extra_dimensions

def default_uncollate(batch: Any) -> List[Any]:
    """This function is used to uncollate a batch into samples. The following conditions are used.

    >>> import torch
    >>> from pprint import pprint
    >>> batch = {"input": torch.zeros([5, 3, 224, 224]), "target": torch.zeros([5, 3, 224, 224]),
    ...          "metadata": {
    ...             'size': [torch.tensor([266, 266, 266, 266, 266]), torch.tensor([266, 266, 266, 266, 266])],
    ...             'height': torch.tensor([266, 266, 266, 266, 266]),
    ...             'width': torch.tensor([266, 266, 266, 266, 266])
    ... }}
    >>> bbatch = default_uncollate(batch)
    >>> len(bbatch)
    5
    >>> print(bbatch[0].keys())
    dict_keys(['input', 'target', 'metadata'])
    >>> print(bbatch[0]["input"].size(), bbatch[0]["target"].size())
    torch.Size([3, 224, 224]) torch.Size([3, 224, 224])
    >>> pprint(bbatch[0]["metadata"])
    {'height': tensor(266),
     'size': (tensor(266), tensor(266)),
     'width': tensor(266)}
    """
    if isinstance(batch, dict):
        elements = [default_uncollate(element) for element in batch.values()]
        return [dict(zip(batch.keys(), element)) for element in zip(*elements)]
    if isinstance(batch, (list, tuple)):
        return list(zip(*batch))
    return list(batch)

class SemanticSegmentationOutputTransform(OutputTransform):
    def per_sample_transform(self, sample: Any) -> Any:
        resize = alb.Resize(*[s.item() for s in sample["metadata"]["size"]])
        sample["input"] = sample["input"].to(torch.float32).cpu().numpy()
        if sample["input"].ndim == 3:
            sample["input"] = np.rollaxis(sample["input"], 0, 3)
        sample["input"] = resize(image=sample["input"])["image"]
        sample["preds"] = [resize(image=pred)["image"] for pred in sample["preds"].to(torch.float32).cpu().numpy()]
        return super().per_sample_transform(sample)

    @staticmethod
    def uncollate(batch: Any) -> Any:
        return default_uncollate(batch)