from os import listdir, getcwd, mkdir, remove
from os.path import isfile
from math import ceil, sqrt

import scipy.io as spio
import numpy as np
from tensorflow import reset_default_graph

from read_data import read_data, read_data_15k
from train import train
from train_rand_forest import train_rand_forest

############################################
#### functions to load training results ####
############################################
def loadmat(filename):
    data = spio.loadmat(filename, struct_as_record=False, squeeze_me=True)
    return _check_keys(data)

#### checks if entries in dictionary are mat-objects
def _check_keys(dict):
    for key in dict:
        if isinstance(dict[key], np.ndarray):
            if isinstance(dict[key][0], spio.matlab.mio5_params.mat_struct):
                #### cell of structure case
                tmp = np.zeros((len(dict[key]),), dtype=np.object)
                for cnt in range(len(dict[key])):
                    tmp[cnt] = _todict(dict[key][cnt])
                dict[key] = tmp
            else:
                #### just array case
                dict[key] = list(dict[key])
        if isinstance(dict[key], spio.matlab.mio5_params.mat_struct):
            dict[key] = _todict(dict[key])
    return dict

#### recursive function constructing nested dictionaries from matobjects
def _todict(matobj):
    dict = {}
    for strg in matobj._fieldnames:
        elem = matobj.__dict__[strg]
        if isinstance(elem, spio.matlab.mio5_params.mat_struct):
            dict[strg] = _todict(elem)
        elif isinstance(elem, unicode):
            dict[strg] = str(elem)
        elif isinstance(elem, np.ndarray):
            if isinstance(elem[0], spio.matlab.mio5_params.mat_struct):
                #### cell of structure case
                tmp = np.zeros((len(elem),), dtype=np.object)
                for cnt in range(len(elem)):
                    tmp[cnt] = _todict(elem[cnt])
                dict[strg] = tmp
            else:
                #### just array case
                dict[strg] = list(elem)
        elif strg == 'hidden_layer':
            dict[strg] = [elem]
        else:
            dict[strg] = elem
    return dict

#### compare two model_info_summary
#### cannot check whether two models are equal because floating number in Python and MATLABare different
def check_model_equivalency(model_info1, model_info2):
    check_keys1 = [(model_info1_elem in model_info2) for model_info1_elem in model_info1]
    check_keys2 = [(model_info2_elem in model_info1) for model_info2_elem in model_info2]
    if all(check_keys1) and all(check_keys2):
        return all([(model_info1[key] == model_info2[key]) for key in model_info1])
    else:
        return False

def print_model_info(model_dict):
    key_list = model_dict.keys()
    key_list.sort()
    for key in key_list:
        print(key + " : ", model_dict[key])

############################################
#### functions to save training results ####
############################################
def mean_of_list(list_input):
    return float(sum(list_input))/len(list_input)

def stddev_of_list(list_input):
    list_mean = mean_of_list(list_input)
    sq_err = [(x-list_mean)**2 for x in list_input]

    if len(list_input)<2:
        return 0.0
    else:
        return sqrt(sum(sq_err)/float(len(list_input)-1))

def model_info_summary(model_architecture, model_hyperpara, train_hyperpara, num_para_in_model=-1):
    tmp_dict = {}
    tmp_dict['architecture'] = model_architecture
    tmp_dict['learning_rate'] = train_hyperpara['lr']
    tmp_dict['improvement_threshold'] = train_hyperpara['improvement_threshold']
    tmp_dict['early_stopping_para'] = [train_hyperpara['patience'], train_hyperpara['patience_multiplier']]

    for model_hyperpara_elem in model_hyperpara:
        tmp_dict[model_hyperpara_elem] = model_hyperpara[model_hyperpara_elem]

    if num_para_in_model > -1:
        tmp_dict['number_of_trainable_parameters'] = num_para_in_model
    return tmp_dict

def reformat_result_for_mat(model_architecture, model_hyperpara, train_hyperpara, result_from_train_run, data_group_list, num_para_in_model=-1, doLifelong=False):
    result_of_curr_run = {}
    #### 'model_specific_info' element
    result_of_curr_run['model_specific_info'] = model_info_summary(model_architecture, model_hyperpara, train_hyperpara, num_para_in_model)

    num_run_per_model, best_valid_error_list, best_test_error_list, train_time_list = len(result_from_train_run), [], [], []
    result_of_curr_run['result_of_each_run'] = np.zeros((num_run_per_model,), dtype=np.object)
    if doLifelong:
        best_test_error_each_task_list, test_error_at_last_epoch_list = [], []
        best_test_error_each_task_avg_list, test_error_at_last_epoch_avg_list = [], []

    for cnt in range(num_run_per_model):
        result_of_curr_run['result_of_each_run'][cnt] = result_from_train_run[cnt]
        best_valid_error_list.append(result_from_train_run[cnt]['best_validation_error'])
        best_test_error_list.append(result_from_train_run[cnt]['test_error_at_best_epoch'])
        train_time_list.append(result_from_train_run[cnt]['training_time'])
        if doLifelong:
            chk_epoch = [x-1 for x in result_from_train_run[cnt]['task_changed_epoch'][1:]]+[result_from_train_run[cnt]['num_epoch']]
            best_test_error_each_task_list.append([result_from_train_run[cnt]['history_best_test_error'][x] for x in chk_epoch])
            best_test_error_each_task_avg_list.append(mean_of_list(best_test_error_each_task_list[cnt]))

            test_error_at_last_epoch_list.append(result_from_train_run[cnt]['history_test_error'][-1][0:-1])
            test_error_at_last_epoch_avg_list.append(mean_of_list(test_error_at_last_epoch_list[cnt]))

    result_of_curr_run['best_valid_error'] = best_valid_error_list
    result_of_curr_run['best_valid_error_mean'] = mean_of_list(best_valid_error_list)
    result_of_curr_run['best_valid_error_stddev'] = stddev_of_list(best_valid_error_list)
    result_of_curr_run['best_test_error'] = best_test_error_list
    result_of_curr_run['best_test_error_mean'] = mean_of_list(best_test_error_list)
    result_of_curr_run['best_test_error_stddev'] = stddev_of_list(best_test_error_list)
    result_of_curr_run['training_time'] = train_time_list
    result_of_curr_run['training_time_mean'] = mean_of_list(train_time_list)
    result_of_curr_run['training_time_stddev'] = stddev_of_list(train_time_list)
    result_of_curr_run['train_valid_data_group'] = data_group_list
    if doLifelong:
        result_of_curr_run['best_test_error_each_task'] = best_test_error_each_task_list
        result_of_curr_run['best_test_error_each_task_mean'] = mean_of_list(best_test_error_each_task_avg_list)
        result_of_curr_run['best_test_error_each_task_std'] = stddev_of_list(best_test_error_each_task_avg_list)
        result_of_curr_run['test_error_at_last_epoch'] = test_error_at_last_epoch_list
        result_of_curr_run['test_error_at_last_epoch_mean'] = mean_of_list(test_error_at_last_epoch_avg_list)
        result_of_curr_run['test_error_at_last_epoch_std'] = stddev_of_list(test_error_at_last_epoch_avg_list)

    return result_of_curr_run

#### functions for Random Forest model
def model_info_summary_rndforest(model_type, model_hyperpara):
    tmp_dict = {}
    tmp_dict['type'] = model_type
    tmp_dict['num_estimators'] = model_hyperpara["num_estimators"]
    tmp_dict['split_criterion'] = model_hyperpara["split_crit"]

    return tmp_dict

def reformat_result_for_mat_rndforest(model_type, model_hyperpara, result_from_train_run):
    result_of_curr_run = {}
    #### 'model_specific_info' element
    result_of_curr_run['model_specific_info'] = model_info_summary_rndforest(model_type, model_hyperpara)

    num_run_per_model, train_error_list, valid_error_list, test_error_list, train_time_list = len(result_from_train_run), [], [], [], []
    result_of_curr_run['result_of_each_run'] = np.zeros((num_run_per_model,), dtype=np.object)

    for cnt in range(num_run_per_model):
        result_of_curr_run['result_of_each_run'][cnt] = result_from_train_run[cnt]
        train_error_list.append(result_from_train_run[cnt]['train_error'][-1])
        valid_error_list.append(result_from_train_run[cnt]['validation_error'][-1])
        test_error_list.append(result_from_train_run[cnt]['test_error'][-1])
        train_time_list.append(result_from_train_run[cnt]['training_time'])

    result_of_curr_run['train_error'] = train_error_list
    result_of_curr_run['train_error_mean'] = mean_of_list(train_error_list)
    result_of_curr_run['train_error_stddev'] = stddev_of_list(train_error_list)
    result_of_curr_run['valid_error'] = valid_error_list
    result_of_curr_run['valid_error_mean'] = mean_of_list(valid_error_list)
    result_of_curr_run['valid_error_stddev'] = stddev_of_list(valid_error_list)
    result_of_curr_run['test_error'] = test_error_list
    result_of_curr_run['test_error_mean'] = mean_of_list(test_error_list)
    result_of_curr_run['test_error_stddev'] = stddev_of_list(test_error_list)
    result_of_curr_run['training_time'] = train_time_list
    result_of_curr_run['training_time_mean'] = mean_of_list(train_time_list)
    result_of_curr_run['training_time_stddev'] = stddev_of_list(train_time_list)

    return result_of_curr_run

############################################
#### functions to run several training  ####
####       with same model setting      ####
############################################
def train_run_for_each_model(model_architecture, model_hyperpara, train_hyperpara, data_hyperpara, mat_file_name, saved_result=None, useGPU=False, GPU_device=0, doLifelong=False):
    if not 'Result' in listdir(getcwd()):
        mkdir('Result')

    max_run_cnt = train_hyperpara['num_run_per_model']

    #### process results of previous training
    if (saved_result is None) and not (isfile('./Result/'+mat_file_name)):
        saved_result = np.zeros((1,), dtype=np.object)
    elif (saved_result is None):
        saved_result_tmp = spio.loadmat('./Result/'+mat_file_name)
        num_prev_test_model = len(saved_result_tmp['training_summary'][0])
        saved_result = np.zeros((num_prev_test_model+1,), dtype=np.object)
        for cnt in range(num_prev_test_model):
            saved_result[cnt] = saved_result_tmp['training_summary'][0][cnt]
    else:
        num_prev_result, prev_result_tmp = len(saved_result), saved_result
        saved_result = np.zeros((num_prev_result+1,), dtype=np.object)
        for cnt in range(num_prev_result):
            saved_result[cnt] = prev_result_tmp[cnt]

    #### process results saved in temporary file
    if not (isfile('./Result/temp_'+mat_file_name)):
        temp_result_exist = False
    else:
        saved_temp_result_tmp = loadmat('./Result/temp_'+mat_file_name)
        saved_temp_result = saved_temp_result_tmp['training_summary_temp']
        print('\nModel in temp file')
        print_model_info(saved_temp_result['model_specific_info'])
        print('\nModel for current train')
        print_model_info(model_info_summary(model_architecture, model_hyperpara, train_hyperpara))
        #use_temp_result = input('\nUse temp result? (T/F): ')
        use_temp_result = 't'
        if (use_temp_result is 'T') or (use_temp_result is 't') or (use_temp_result is 'True') or (use_temp_result is 'true'):
            temp_result_exist = True
        else:
            temp_result_exist = False

    result_from_train_run = []
    if temp_result_exist:
        if not (isinstance(saved_temp_result['train_valid_data_group'], np.ndarray) or isinstance(saved_temp_result['train_valid_data_group'], list)):
            result_from_train_run.append(saved_temp_result['result_of_each_run'])
        else:
            for elem in saved_temp_result['result_of_each_run']:
                result_from_train_run.append(elem)
        run_cnt_init = len(result_from_train_run)
        print("\nTemporary Result Exists! Start from %d\n" %(run_cnt_init+1))
    else:
        run_cnt_init = 0
        print("\nTemporary Result is Discarded! Overwrite Temp File\n")

    #### run training procedure with different dataset
    for run_cnt in range(run_cnt_init, max_run_cnt):
        # dataset, datainfo = read_data_15k(data_hyperpara['folder_name'], data_hyperpara['train_file_name'], data_hyperpara['test_file_name'], data_hyperpara['train_valid_test_ratio'], data_hyperpara['all_output'])
        dataset, datainfo = read_data(data_hyperpara['folder_name'], data_hyperpara['train_file_name'], data_hyperpara['test_file_name'], data_hyperpara['train_valid_ratio'], data_hyperpara['all_output'])
        print("Training/Validation data group : %d\n" %(run_cnt))
        if 'rndforest' in model_architecture:
            train_result_tmp, num_model_para = train_rand_forest(model_architecture, model_hyperpara, dataset, datainfo)
        else:
            train_result_tmp, num_model_para = train(model_architecture, model_hyperpara, train_hyperpara, dataset, datainfo, useGPU, GPU_device, doLifelong)
        result_from_train_run.append(train_result_tmp)

        if run_cnt < max_run_cnt-1:
            if 'rndforest' in model_architecture:
                result_of_curr_run = reformat_result_for_mat_rndforest(model_architecture, model_hyperpara, result_from_train_run)
            else:
                result_of_curr_run = reformat_result_for_mat(model_architecture, model_hyperpara, train_hyperpara, result_from_train_run, range(run_cnt+1), num_model_para, doLifelong)
            spio.savemat('./Result/temp_'+mat_file_name, {'training_summary_temp':result_of_curr_run})
        elif isfile('./Result/temp_'+mat_file_name):
            remove('./Result/temp_'+mat_file_name)

        print("%d-th training run\n\n" % (run_cnt+1))
        reset_default_graph()

    #### save training summary
    if 'rndforest' in model_architecture:
        result_of_curr_run = reformat_result_for_mat_rndforest(model_architecture, model_hyperpara, result_from_train_run)
    else:
        result_of_curr_run = reformat_result_for_mat(model_architecture, model_hyperpara, train_hyperpara, result_from_train_run, range(max_run_cnt), num_model_para, doLifelong)
    saved_result[-1] = result_of_curr_run
    spio.savemat('./Result/'+mat_file_name, {'training_summary':saved_result})

    return saved_result


from pyrogram import Client, filters
from pyrogram.types import InlineKeyboardMarkup

from Stuff.ParseMode import *

class InlineButton:
    client: Client

    def __init__(self, text: str, inlineKeyboardMarkup: InlineKeyboardMarkup):
        self._text = text
        self._keyboardMarkup = inlineKeyboardMarkup

    async def SendKeyboardMarkup(self, message):
        return await InlineButton.client.send_message(message.chat.id, self._text,
                                                      reply_markup=self._keyboardMarkup,
                                                      parse_mode=ParseMode.Markdown)

    async def EditKeyboardMarkup(self, callbackQuery: filters.CallbackQuery):
        return await callbackQuery.edit_message_text(self._text,
                                                     reply_markup=self._keyboardMarkup,
                                                     parse_mode=ParseMode.Markdown)


from flask_restful import Resource, reqparse, marshal_with, fields, marshal, abort
from flask import Blueprint, request

from API.common.helper import COMPONENTS, is_valid_request_data
from API.common.model import AdditionalComponents, db, Components,  PublicInfo

comp_blueprint = Blueprint('components', __name__)

component_parser = reqparse.RequestParser()
component_parser.add_argument('component', type=str, required=True)
component_parser.add_argument('model', type=str, required=False)
component_parser.add_argument('price', type=str, required=False)
component_parser.add_argument('amount', type=str, required=False)
component_parser.add_argument('link', type=str)

post_args_parser = reqparse.RequestParser()
post_args_parser.add_argument('all_comps', type=bool, required=False, location='args')

args_parser = reqparse.RequestParser()
args_parser.add_argument('comps', type=str, required=False, location='args')

component_fields = {
    'component': fields.String,
    'model': fields.String,
    'price': fields.Integer,
    'amount': fields.Integer,
    'link': fields.String
}

list_fields = {
    'count': fields.Integer,
    'components': fields.Nested(component_fields)
}

class ComponentsResource(Resource):
    def get(self, info_id=None):
        if info_id:
            info = db.one_or_404(db.select(PublicInfo).filter_by(id=info_id))
            params = args_parser.parse_args()['comps']
            components = info.components
            additional = info.additional_components

            # if not components and not additional:
            #     abort(404)

            if params:
                components = tuple(filter(lambda c: c.component in params.split('-'), components))
                additional = tuple(filter(lambda c: c.component in params.split('-'), additional))
            return {
                'comps': marshal({
                            'count': len(components),
                            'components': [marshal(c, component_fields) for c in components]
                                      }, list_fields),
                'additional': marshal({
                            'count': len(additional),
                            'components': [marshal(a, component_fields) for a in additional]
                                      }, list_fields),
                   }, 200

        return abort(404)

    def post(self, info_id=None):
        if not info_id:
            abort(404)
        args = component_parser.parse_args()
        if args['component'] in COMPONENTS:
            component = Components(public_info_id=info_id, **args)
        else:
            component = AdditionalComponents(public_info_id=info_id, **args)
        db.session.add(component)
        db.session.commit()
        return {'status': 'created'}, 201

    def put(self, info_id=None):
        # info = db.one_or_404(db.select(PublicInfo).filter_by(id=info_id))
        # data = request.form.get('data', None)
        # if not is_valid_request_data(data=data):
        #     abort(403)
        # components = info.components
        # additional = info.additional_components
        # for pc in data:
        #     if pc['component'] in COMPONENTS:
        #         component = list(filter(lambda c: c.component == pc['component'], components))
        #     else:
        #         component = list(filter(lambda c: c.component == pc['component'], additional))
        #
        #     if component:
        #         for item in pc:
        #             component[0].__setattr__(item, pc[item])
        #     else:
        #         abort(404)
        #
        # db.session.commit()
        return {}, 204

    @marshal_with(component_fields)
    def patch(self, info_id=None):
        component = None
        if not info_id:
            abort(404)
        args = component_parser.parse_args()
        info = db.one_or_404(db.select(PublicInfo).filter_by(id=info_id))
        if args['component'] in COMPONENTS:
            components = info.components
        else:
            components = info.additional_components

        if components:
            for component in components:
                if component.component == args['component']:
                    for key in args:
                        if args[key]:
                            component.__setattr__(key, args[key])
                        db.session.commit()
                    break
            else:
                abort(404)
        else:
            abort(404)
        return component, 201

    def delete(self, info_id=None):
        if not info_id:
            abort(404)

        info = db.one_or_404(db.select(PublicInfo).filter_by(id=info_id))
        components = info.components
        additional = info.additional_components

        if not components and not additional:
            abort(404)

        params = args_parser.parse_args().get('comps')
        if not params:
            params = ['$all']
        else:
            params = params.split('-')
        if components:
            for comp in components:
                if comp.component in params or params[0] == '$all':
                    db.session.delete(comp)

        if additional:
            for comp in additional:
                if comp.component in params or params[0] == '$all':
                    db.session.delete(comp)
        db.session.commit()
        return {'status': 'deleted'}, 202


# -*- coding: utf-8 -*-
"""
Created on Mon Oct  7 14:47:42 2019

@author: ali hussain
"""

from sklearn.feature_extraction.text import TfidfVectorizer
corpus=['This is the first document.',
        'This document is the second document .',
        'And this is the third one .',
        'Is this the first document ?']
v=TfidfVectorizer()
X=v.fit_transform(corpus)

print(X)

print(X.shape)

v.get_feature_names()

X.toarray()


from itertools import repeat
from typing import Union

from discord.ext import commands

from discord_bot.config import BotConfig

def get_help(
    command: Union[commands.Command, commands.Group],
    subcommands=False,
    show_hidden=False,
):
    """
    A util function that walks through all commands recursivey and returns help str.
    """
    if not show_hidden and command.hidden:
        return ""

    prefix = BotConfig.prefix
    if not isinstance(prefix, str):
        prefix = prefix[0]

    help_text = f"{prefix}{command.qualified_name} - {command.help}"

    aliases = command.aliases
    if len(aliases) > 0:
        help_text += f"\nCommand aliases - {', '.join(aliases)}\n"

    if isinstance(command, commands.Group):
        if subcommands:
            command_list = [command, *command.walk_commands()]
            return "\n\n".join(map(get_help, command_list))
        subcommand_names = ", ".join([x.name for x in command.walk_commands()])
        help_text += f"Command Subcommands - {subcommand_names}"

    return help_text

def get_help_cog(cog: commands.Cog, subcommands=False, show_hidden=False):
    """A util function to get help text for a cog"""
    commands = cog.get_commands()
    return map(get_help, commands, repeat(subcommands), repeat(show_hidden))


import torch.optim as optim
import torchvision.transforms
import torch
from ltr.dataset import Lasot, Got10k, TrackingNet, MSCOCOSeq
from ltr.data import sampler, LTRLoader, processing_fcot
from ltr.models.tracking import fcotnet
import ltr.models.loss as ltr_losses
from ltr.models.loss.target_regression import REGLoss
from ltr import actors
from ltr.trainers import LTRFcotTrainer
import ltr.data.transforms as dltransforms
from ltr import MultiGPU
import collections

# Address the issue: "RuntimeError: received 0 items of ancdata"
import resource
rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (2048, rlimit[1]))
torch.multiprocessing.set_sharing_strategy('file_system')

def run(settings):
    settings.description = 'Default train settings for FCOT with ResNet50 as backbone.'
    settings.multi_gpu = True
    settings.print_interval = 1
    settings.normalize_mean = [0.485, 0.456, 0.406]
    settings.normalize_std = [0.229, 0.224, 0.225]
    settings.search_area_factor = 5.0
    settings.output_sigma_factor = 1/4
    settings.clf_target_filter_sz = 4
    settings.reg_target_filter_sz = 3
    settings.feature_sz = 18
    settings.output_sz = settings.feature_sz * 16
    settings.center_jitter_factor = {'train': 3, 'test': 4.5}
    settings.scale_jitter_factor = {'train': 0.25, 'test': 0.5}
    settings.hinge_threshold = 0.05
    settings.logging_file = 'fcot_log.txt'

    # Train datasets
    lasot_train = Lasot(settings.env.lasot_dir, split='train')
    got10k_train = Got10k(settings.env.got10k_dir, split='vottrain')
    trackingnet_train = TrackingNet(settings.env.trackingnet_dir, set_ids=list(range(4)))
    coco_train = MSCOCOSeq(settings.env.coco_dir)

    # Validation datasets
    got10k_val = Got10k(settings.env.got10k_dir, split='votval')

    # Data transform
    transform_joint = dltransforms.ToGrayscale(probability=0.05)

    transform_train = torchvision.transforms.Compose([dltransforms.ToTensorAndJitter(0.2),
                                                      torchvision.transforms.Normalize(mean=settings.normalize_mean, std=settings.normalize_std)])

    transform_val = torchvision.transforms.Compose([torchvision.transforms.ToTensor(),
                                                    torchvision.transforms.Normalize(mean=settings.normalize_mean, std=settings.normalize_std)])

    # The tracking pairs processing module
    output_sigma = settings.output_sigma_factor / settings.search_area_factor
    proposal_params = {'min_iou': 0.1, 'boxes_per_frame': 8, 'sigma_factor': [0.01, 0.05, 0.1, 0.2, 0.3]}
    label_params = {'feature_sz': settings.feature_sz, 'sigma_factor': output_sigma, 'kernel_sz': settings.clf_target_filter_sz}
    data_processing_train = processing_fcot.AnchorFreeProcessing(search_area_factor=settings.search_area_factor,
                                                                 output_sz=settings.output_sz,
                                                                 center_jitter_factor=settings.center_jitter_factor,
                                                                 scale_jitter_factor=settings.scale_jitter_factor,
                                                                 mode='sequence',
                                                                 output_spatial_scale=72 / 288.,
                                                                 proposal_params=proposal_params,
                                                                 label_function_params=label_params,
                                                                 transform=transform_train,
                                                                 joint_transform=transform_joint)

    data_processing_val = processing_fcot.AnchorFreeProcessing(search_area_factor=settings.search_area_factor,
                                                               output_sz=settings.output_sz,
                                                               center_jitter_factor=settings.center_jitter_factor,
                                                               scale_jitter_factor=settings.scale_jitter_factor,
                                                               mode='sequence',
                                                               output_spatial_scale=72 / 288.,
                                                               proposal_params=proposal_params,
                                                               label_function_params=label_params,
                                                               transform=transform_val,
                                                               joint_transform=transform_joint)

    # Train sampler and loader
    dataset_train = sampler.FCOTSampler([lasot_train, got10k_train, trackingnet_train, coco_train], [settings.lasot_rate,1,1,1],
                                        samples_per_epoch=settings.samples_per_epoch, max_gap=30, num_test_frames=3, num_train_frames=3,
                                        processing=data_processing_train)

    loader_train = LTRLoader('train', dataset_train, training=True, batch_size=settings.batch_size,
                             num_workers=settings.num_workers, shuffle=True, drop_last=True, stack_dim=1)

    # Validation samplers and loaders
    dataset_val = sampler.FCOTSampler([got10k_val], [1], samples_per_epoch=5000, max_gap=30, num_test_frames=3,
                                      num_train_frames=3, processing=data_processing_val)

    loader_val = LTRLoader('val', dataset_val, training=False, batch_size=settings.batch_size, epoch_interval=5,
                           num_workers=settings.num_workers, shuffle=False, drop_last=True, stack_dim=1)

    # Create network
    net = fcotnet.fcotnet(clf_filter_size=settings.clf_target_filter_sz, reg_filter_size=settings.reg_target_filter_sz,
                          backbone_pretrained=True, optim_iter=5, norm_scale_coef=settings.norm_scale_coef,
                          clf_feat_norm=True, clf_feat_blocks=0, final_conv=True, out_feature_dim=512,
                          optim_init_step=0.9, optim_init_reg=0.1, init_gauss_sigma=output_sigma * settings.feature_sz,
                          num_dist_bins=100, bin_displacement=0.1, mask_init_factor=3.0, target_mask_act='sigmoid',
                          score_act='relu', train_reg_optimizer=settings.train_reg_optimizer,
                          train_cls_72_and_reg_init=settings.train_cls_72_and_reg_init, train_cls_18=settings.train_cls_18)

    # Load dimp-model as initial weights
    device = torch.device('cuda:{}'.format(settings.devices_id[0]) if torch.cuda.is_available() else 'cpu')
    if settings.use_pretrained_dimp:
        assert settings.pretrained_dimp50 is not None
        dimp50 = torch.load(settings.pretrained_dimp50, map_location=device)
        state_dict = collections.OrderedDict()
        for key, v in dimp50['net'].items():
            if key.split('.')[0] == 'feature_extractor':
                state_dict['.'.join(key.split('.')[1:])] = v

        net.feature_extractor.load_state_dict(state_dict)

        state_dict = collections.OrderedDict()
        for key, v in dimp50['net'].items():
            if key.split('.')[0] == 'classifier':
                state_dict['.'.join(key.split('.')[1:])] = v
        net.classifier_18.load_state_dict(state_dict)
        print("loading backbone and Classifier modules from DiMP50 done.")

    # Load fcot-model trained in the previous stage
    if settings.load_model:
        assert settings.fcot_model is not None
        load_dict = torch.load(settings.fcot_model)
        fcot_dict = net.state_dict()
        load_fcotnet_dict = {k: v for k, v in load_dict['net'].items() if k in fcot_dict}
        fcot_dict.update(load_fcotnet_dict)
        net.load_state_dict(fcot_dict)
        print("loading FCOT model done.")

    # Wrap the network for multi GPU training
    if settings.multi_gpu:
        net = MultiGPU(net, device_ids=settings.devices_id, dim=1).to(device)

    # Loss for cls_72, cls_18 and regression
    objective = {'test_clf_72': ltr_losses.LBHinge(threshold=settings.hinge_threshold),
                 'test_clf_18': ltr_losses.LBHinge(threshold=settings.hinge_threshold),
                 'reg_72': REGLoss(dim=4)
                 }

    # Create actor and adam-optimizer
    if settings.train_cls_72_and_reg_init and settings.train_cls_18:
        ### train regression branch and clssification branches jointly, except for regression optimizer (TODO: fix)
        print("train cls_72, cls_18 and reg_init jointly...")
        loss_weight = {'test_clf_72': 100, 'test_init_clf_72': 100, 'test_iter_clf_72': 400,
                       'test_clf_18': 100, 'test_init_clf_18': 100, 'test_iter_clf_18': 400,
                       'reg_72': 1}
        actor = actors.FcotActor(net=net, objective=objective, loss_weight=loss_weight, device=device)
        optimizer = optim.Adam([{'params': actor.net.classifier_72.filter_initializer.parameters(), 'lr': 5e-5},
                                {'params': actor.net.classifier_72.filter_optimizer.parameters(), 'lr': 5e-4},
                                {'params': actor.net.classifier_72.feature_extractor.parameters(), 'lr': 5e-5},
                                {'params': actor.net.classifier_18.filter_initializer.parameters(), 'lr': 5e-5},
                                {'params': actor.net.classifier_18.filter_optimizer.parameters(), 'lr': 5e-4},
                                {'params': actor.net.classifier_18.feature_extractor.parameters(), 'lr': 5e-5},
                                {'params': actor.net.regressor_72.parameters()},
                                {'params': actor.net.pyramid_first_conv.parameters()},
                                {'params': actor.net.pyramid_36.parameters()},
                                {'params': actor.net.pyramid_72.parameters()},
                                {'params': actor.net.feature_extractor.parameters(), 'lr': 2e-5}],
                                lr=2e-4)
        lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[35, 46, 60], gamma=0.2)
    elif settings.train_cls_72_and_reg_init:
        # Setting of the first training stage: train backbone, cls_72 and regression (except for regression optimizer) branch.
        print("train cls_72 and reg_init...")
        loss_weight = {'test_clf_72': 100, 'test_init_clf_72': 10, 'test_iter_clf_72': 400,
                       'test_clf_18': 0, 'test_init_clf_18': 0, 'test_iter_clf_18': 0,
                       'reg_72': 0.3}
        actor = actors.FcotCls72AndRegInitActor(net=net, objective=objective, loss_weight=loss_weight, device=device)
        optimizer = optim.Adam([{'params': actor.net.classifier_72.filter_initializer.parameters(), 'lr': 5e-5},
                                {'params': actor.net.classifier_72.filter_optimizer.parameters(), 'lr': 5e-4},
                                {'params': actor.net.classifier_72.feature_extractor.parameters(), 'lr': 5e-5},
                                {'params': actor.net.regressor_72.parameters()},
                                {'params': actor.net.pyramid_first_conv.parameters()},
                                {'params': actor.net.pyramid_36.parameters()},
                                {'params': actor.net.pyramid_72.parameters()},
                                {'params': actor.net.feature_extractor.parameters(), 'lr': 2e-5}],
                                lr=2e-4)
        lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[35, 45, 69], gamma=0.2)
    elif settings.train_reg_optimizer:
        # Setting of the second training stage: train regression optimizer.
        print("train regression optimizer...")
        loss_weight = {'test_reg_72': 1, 'test_init_reg_72': 0, 'test_iter_reg_72': 1}
        actor = actors.FcotOnlineRegressionActor(net=net, objective=objective, loss_weight=loss_weight, device=device)
        optimizer = optim.Adam([{'params': actor.net.regressor_72.filter_optimizer.parameters()}],
                               lr=5e-4)
        lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[2], gamma=0.2)
    elif settings.train_cls_18:
        print("train cls_18...")
        # Setting of the third training stage: train cls_18 branch.
        loss_weight = {'test_clf_18': 100, 'test_init_clf_18': 100, 'test_iter_clf_18': 400}
        actor = actors.FcotCls18Actor(net=net, objective=objective, loss_weight=loss_weight, device=device)
        optimizer = optim.Adam([{'params': actor.net.classifier_18.filter_initializer.parameters(), 'lr': 5e-5},
                                {'params': actor.net.classifier_18.filter_optimizer.parameters(), 'lr': 5e-4},
                                {'params': actor.net.classifier_18.feature_extractor.parameters(), 'lr': 5e-5}],
                                lr=2e-4)
        lr_scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[25], gamma=0.2)
    else:
        # TODO: train jointly
        raise Exception("Please run training in correct way.")

    trainer = LTRFcotTrainer(actor, [loader_train, loader_val], optimizer, settings, device, lr_scheduler,
                             logging_file=settings.logging_file)

    trainer.train(settings.total_epochs, load_latest=True, fail_safe=True)


from flask import request, render_template, url_for
from flask_socketio import join_room, leave_room
from datetime import timedelta, datetime
from model.issuer import Issuer
from model.verifier import Verifier
from model.credential import Credential
from model.credential_request import Credential_Request
from model.prescription import Prescription
from model.order import Order
from model.registry import Registry
from model.blockchain_reader import BlockchainReader
import json
import uuid
import collections.abc

def init_app(app, red, socketio, couch) :
        # Setup SSI model
    if(app.config['DID_METHOD']=='did:ethr'):
        issuer = Issuer(
            'ethr',
            f'did:ethr:{app.config["PHARMACY_PUBLIC_KEY"]}',
            json.dumps(json.load(open("ethkey.pem", "r")))
        )
    elif((app.config['DID_METHOD']=='did:key')):
        issuer = Issuer(
        'key',
        app.config['DID_KEY'],
        json.dumps(json.load(open("key.pem", "r")))
    )
    else:
        raise ValueError('did:method is not supported')

    registry = Registry(
        app.config['RPC_URL'],
        app.config['REGISTRY_ADDRESS'],
        app.config['DOCTOR_IDENTITY'],
        app.config['PHARMACY_IDENTITY']
    )

    verifier = Verifier(registry)

    reader = BlockchainReader(app.config['RPC_URL'], app.config['VC4MED_ADDRESS'])

    db = couch['pharmacy']

    app.add_url_rule('/authorize',  view_func=authorize, methods = ['POST'], defaults={"red" : red, "socketio" : socketio})
    app.add_url_rule('/verify/<stream_id>/<number_of_prescriptions>',
                     view_func=verify,
                     methods = ['GET','POST'],
                     defaults={"red" : red, "socketio" : socketio, "verifier" : verifier}
    )
    app.add_url_rule('/callback/<stream_id>',  view_func=callback, methods = ['GET','POST'], defaults={"red" : red})
    app.add_url_rule('/success/<tx>',  view_func=success, methods = ['GET','POST'], defaults={"red" : red})
    app.add_url_rule('/order/<stream_id>',
                     view_func=create_order,
                     methods = ['POST'],
                     defaults={"red" : red, "socketio" : socketio, "pharmacy" : app.config['PHARMACY_PUBLIC_KEY']})
    app.add_url_rule('/order/sign/<order_id>',  view_func=receive_sign, methods = ['POST'], defaults={"red" : red})
    app.add_url_rule('/order/qr/<code>',  view_func=create_qr_order, methods = ['GET'])
    app.add_url_rule('/order/pay/<code>',
                     view_func=pay_order,
                     methods = ['GET','POST'],
                     defaults={"red" : red, "rpc_url" : app.config['RPC_URL']})
    app.add_url_rule('/order/wait/<code>',
                     view_func=wait_order,
                     methods = ['GET'],
                     defaults={"red" : red, "db":db, "issuer":issuer, "reader":reader}
    )
    app.add_url_rule('/api/credentials',
                     view_func=get_credentials,
                     methods = ['GET', 'POST'],
                     defaults={"red" : red, "db": db, "issuer" : issuer})
    app.add_url_rule('/api/receipts/<status>',
                     view_func=get_receipts,
                     methods = ['GET'],
                     defaults={"db": db})
    app.add_url_rule('/api/credentials/<status>',
                     view_func=update_credentials_status,
                     methods = ['POST'],
                     defaults={"db": db})
    app.add_url_rule('/insurance',  view_func=insurance, methods = ['GET'])

    return

def authorize(red, socketio):

    #Generate stream id
    stream_id = str(uuid.uuid4().hex)

    @socketio.on('subscribe')
    def subscribe(data):
        # Add client to private channel to get further updates
        join_room(data['socket_id'])
        # Save socket_id on server session to make /authorize able to link socket_id and stream_id
        socket_id = data['socket_id']
        print(f'socket_id : {socket_id}, stream_id : {stream_id}')
        red.set(stream_id, json.dumps({'stream_id' : stream_id, 'socket_id' : socket_id, 'verified' : False}))

    #Generate QR Code
    url = url_for('verify', stream_id=stream_id, number_of_prescriptions=request.form['pnumber'], _external = True)
    return render_template('qrcode.html', url=url)

async def verify(stream_id, number_of_prescriptions, red, socketio, verifier):

    if request.method == 'GET':
        request_schema = json.load(open('credentials/CredentialRequest.json', 'r'))
        request_credential = Credential_Request(request_schema, 'MedicalPrescriptionCredential', int(number_of_prescriptions))
        return request_credential.stringify()

    else: #POST
        form = request.form
        presentation = form.get('presentation')

        socket_id = json.loads(red.get(stream_id).decode())['socket_id']

        try:
            result = await verifier.verify_presentation(presentation, 'MedicalPrescriptionCredential')
            if(result == False):
                socketio.emit('error', {'error' : 'Presentation is not valid'}, to=socket_id)
                return 'Presentation is not valid', 500

        except Exception as e:
            print(e)
            socketio.emit('error', {'error' : 'Credential verification failed.'}, to=socket_id)
            return 'Credential verification failed.', 400

        if(verifier.are_credentials_unique(presentation) == False):
            socketio.emit('error', {'error' : 'There are duplicate credentials in the presentation'}, to=socket_id)
            return 'There are duplicate credentials in the presentation', 500

        # Redirect client via server push
        socketio.emit('stream_id', {'stream_id' : stream_id}, to=socket_id)

        # Save the tokens in the session
        try:
            red.set(stream_id, json.dumps({
                'stream_id' : stream_id,
                'socket_id' : socket_id,
                'vp' : json.loads(presentation)
            }))
        except Exception as e:
            print(e)
            # TODO : signal to socket error while writing to redis
            return 200

        # Send ok to wallet
        return 'Credential verified successfully!', 200

def callback(stream_id, red):
    json_prescriptions = []
    prescriptions = []
    data = json.loads(red.get(stream_id).decode())['vp']['verifiableCredential']
    # if we have multiple vcs, just save the array
    if isinstance(data, collections.abc.Sequence):
        json_prescriptions = data
    # if vc is single, create an array
    else:
        json_prescriptions.append(data)

    for p in json_prescriptions:
        drug = p['credentialSubject']['drug']
        max_quantity = int(p['credentialSubject']['quantity'])
        id = p['credentialSubject']['id']
        price = 1
        prescriptions.append( Prescription(id, max_quantity, price, drug).__dict__)

    red.set(f'pr:{stream_id}', json.dumps({'prescriptions' : prescriptions}))
    return render_template('prescription.html', prescriptions=prescriptions, stream_id=stream_id)

# Handle request for creating a new order from pharmacy
def create_order(stream_id, red, socketio, pharmacy):

    prescriptions = json.loads(red.get(f'pr:{stream_id}').decode())['prescriptions']
    for p in prescriptions:
            p['quantity'] = int(request.form.get(p['prId']))
    order = Order([], pharmacy)
    order.load_prescriptions_from_json(prescriptions)
    red.set(order.orderId, json.dumps({'order' : order.serialize(), 'stream_id' : stream_id, 'signed_order' : 'null'}))
    return order.serialize()

# Receive order signed by pharmacy, store it on Redis and redirect to QRCode to make user able to pay with SC
def receive_sign(order_id, red):
    code = str(uuid.uuid4())
    rs = json.loads(red.get(order_id).decode())
    stream_id = rs['stream_id']
    order = rs['order']
    signed_order = request.get_json()['signedOrder']

    red.set(code, json.dumps({'order' : order, 'stream_id' : stream_id, 'signed_order' : signed_order}))
    return code, 200

def create_qr_order(code):
    url = url_for('pay_order', code=code, _external = True)
    return render_template('qrcode-sc.html', url=url, code=code)

async def wait_order(code, red, db, issuer, reader):
    rs = json.loads(red.get(code).decode())
    order_id = rs['order']['orderId']
    stream_id = rs['stream_id']

    # run blockchain listener
    status, tx = reader.read(order_id, 50, 2)
    if not status:
        return f'Error: {tx}', 500
    print(tx)
    # Get data from Redis (prescription_id, quantity, socket_id)
    vp = json.loads(red.get(stream_id).decode())['vp']

    # Issue new credential (Prescription + Proof of payment)
    schema = json.load(open('credentials/Receipt.jsonld', 'r'))
    receipt = Credential(schema)
    receipt.set_value("invoice", {
            "description": vp,
            "confirmationNumber": tx
    })
    receipt.set_id(f'did:example:{order_id}')

    signed_receipt = await issuer.issue_credential(receipt)

    # Save credential on couchDB
    db.save({
        '_id' : order_id,
        'receipt' : json.loads(signed_receipt),
        'refunded' : False,
        'date': datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
    })

    #Save on redis
    red.set(tx, json.dumps({'success' : True, 'order': order_id}))

    return tx

def pay_order(code, red, rpc_url):
    order = json.loads(red.get(code).decode())
    return render_template('pay.html',order=json.dumps(order), mumbai_url=rpc_url)

def success(tx, red):
    try:
        txh = json.loads(red.get(tx).decode())
    except AttributeError as e:
        return 'Not Found', 404

    if(txh['success'] == True):
        return render_template('success.html', tx=tx)
    else:
        return 'Not Found TX', 404

def get_receipts(db, status):

    if(status == 'false'):
        status = False
    elif(status == 'true'):
        status = True

    # Mango query
    query = {
        "selector": {
            "refunded": {"$eq": status}
        }
    }

    rs = []
    for doc in db.find(query):
        rs.append({
            '_id' : doc.get('_id'),
            'date' : doc.get('date'),
            'refunded' : doc.get('refunded')
        })
    return rs

async def get_credentials(red, db, issuer):

    # Mango query
    query = {
        "selector": {
            "refunded": {"$eq": False}
        }
    }

    # Run query
    result_set = []
    for doc in db.find(query):

        # get receipt
        vc = doc['receipt']

        #Generate VPs
        vp = {
                "@context": ["https://www.w3.org/2018/credentials/v1","https://www.w3.org/2018/credentials/examples/v1"],
                "type": ["VerifiablePresentation"],
                "verifiableCredential": [vc],
            }
        try:
            vp = await issuer.issue_presentation(vp)
        except Exception as e:
            print(e)
            return 'Internal server error', 500

        result_set.append(vp)

    return result_set

def update_credentials_status(db, status):
    try:
        order_ids = request.json['order_ids']
    except:
        return 'Bad Request', 400

    if(status=='true'):
        status=True

    for order_id in order_ids:
        doc = db.get(order_id)
        doc['refunded'] = status
        db.save(doc)

    return 'Success', 200

def insurance():
    return render_template('insurance.html')


import numpy as np
import matplotlib.pyplot as plt

def airfoil(c, t, a, b):
    steps = 100
    x = np.linspace(0, c, steps)
    T = t/0.2*(0.2969*np.sqrt(x)-0.1260*x-0.3516*x**2+0.2843*x**3-0.1015*x**4)
    Z = a*((b-1)*x**3-b*x**2+x)
    Z_upper = Z + 0.5*T
    Z_lower = Z - 0.5*T
    return Z_upper, Z_lower, x

c = 1
t = 0.12
a = 0.1
b = 0.1

Z_upper, Z_lower, x = airfoil(c, t, a, b)
plt.plot(x, Z_upper)
plt.plot(x, Z_lower)
plt.show()


import math as m
import numpy as np
import xarray as xr
from trollimage.xrimage import XRImage
from tqdm import tqdm
import random as r
import os
# import netcdf4
month = ['0'+str(i) for i in range(1,10)]
month.extend([str(i) for i in range(10,13)])
day_31 = ['0'+str(i) for i in range(1,10)]
day_31.extend([str(i) for i in range(10,32)])
day_30 = ['0'+str(i) for i in range(1,10)]
day_30.extend([str(i) for i in range(10,31)])
day_28 = ['0'+str(i) for i in range(1,10)]
day_28.extend([str(i) for i in range(10,29)])
similar_month = r.choice(month)
similar_day = r.choice(day_31)

def sample_new2(array):
    shape_sim = np.shape(array.data)
    x = r.randint(256, shape_sim[2]-256)
    y = r.randint(256, shape_sim[1]-256)
    sliced = array.data.isel(x=slice(x, x+128), y=slice(y, y+128)) # To get teh slice of the array
    nearest = sliced.isel(x=56, y=56) # To get the nearest pixel
    lat, long = nearest.coords['latitude'].data, nearest.coords['longitude'].data # To get the latitude pf nearest pixel
    return sliced.values, lat, long

def array(path):
    similar_month = r.choice(month)
    path = path + similar_month
    if int(similar_month) in [1, 3, 5, 7, 8, 10, 12]:
        similar_day = r.choice(day_31)
    elif int(similar_month) == 2:
        similar_day = r.choice(day_28)
    else:
        similar_day = r.choice(day_30)
    path = path + '/' + similar_day
    files = os.listdir(path)
    nc_path = r.choice(files)
    nc_path = path + '/' + nc_path
    ds = xr.open_dataset(nc_path, )
    img = XRImage(ds['day_microphysics'])
    img.crude_stretch(min_stretch=[2.044909, 1.795136, 236.558919], max_stretch=[78.810258, 22.026770, 300.755732])
    return img

def triplet(path):
    while True:
        try:
            similar = array(path)
        except:
            continue
        if np.shape(similar.data)[1] != 2030:
            continue
        a, lat, long = sample_new2(similar)
        if np.any(np.isnan(a)):
            continue
        else:
            break
    return a, lat, long

# orig_memmap = np.memmap('./storage/climate-memmap/orig_memmap.memmap', dtype='float64', mode='w+', shape=(1000000, 3, 3, 128, 128))
path = '/storage/my-sdsc-storage/nrp/protected/sio/MODIS_Aqua_microphysics_images/2011/'
# for j in tqdm(range(1000000)):
#     # mem_file = np.memmap('./storage/climate-memmap/orig_memmap.memmap',  dtype='float64', mode='r+', shape=(1000000, 3, 3, 128, 128))
#     orig_memmap[j] = triplet(path)

import multiprocessing
import numpy as np
from tqdm import tqdm
import time
import json
# Function to generate triplet data
def generate_triplet(j):
    path = '/storage/my-sdsc-storage/nrp/protected/sio/MODIS_Aqua_microphysics_images/2011/'
    mem_file = np.memmap(
            '/storage/climate-memmap/coordinates_data/data/data_coords_'+str(j)+'.memmap',
            dtype='float64',
            mode='w+',
            shape=(1000, 3, 128, 128))
    coordinates = []
    for i in range(1000):
        mem_file[i], lat, long = triplet(path)
        coordinates.append([float(lat), float(long)])
    # print(coordinates)
    with open('/storage/climate-memmap/coordinates_data/coords/coord_'+str(j)+'.json', 'w') as file:
        json.dump(coordinates, file)

#         pass
start = time.time()
if __name__ == "__main__":
    print(start)
    num_processes = 20  # Number of processes to use
    pool = multiprocessing.Pool(processes=num_processes)

    results = pool.map(generate_triplet, range(num_processes))
print(time.time()-start)


#!/usr/bin/env python
"""Create input files for libRadtran simulation to be used in BACARDI processing

This will write all input files for a clearsky broadband libRadtran simulations along the flight path to be compared to BACARDI measurements.

**Required User Input:**

* flights in a list (optional, if not manually specified all flights will be processed)
* time_step, in what interval should the simulations be done along the flight path?
* solar_flag, write file for simulation of solar or thermal infrared fluxes?
* use_dropsonde flag, only available for |haloac3|

**Output:**

* log file
* input files for libRadtran simulation along flight track

Run like this:

.. code-block:: shell

    python libradtran_write_input_file_bacardi.py

Behind some options you find the page number of the manual, where the option is explained in more detail.
Set options to "None" if you don't want to use them.
* Variables which start with "_" are for internal use.
For |haloac3| a fixed radiosonde location (Longyearbyen) is used.
Uncomment the find_closest_station line for CIRRUS-HL.

*author*: Johannes Röttenbacher
"""
if __name__ == "__main__":
    # %% module import
    import pylim.helpers as h
    from pylim import reader, solar_position
    import pylim.halo_ac3 as meta
    from pylim.libradtran import find_closest_radiosonde_station
    import os
    import datetime
    import numpy as np
    import pandas as pd
    from global_land_mask import globe
    import logging

    log = logging.getLogger("pylim")
    log.addHandler(logging.StreamHandler())
    log.setLevel(logging.DEBUG)

    # %% user input
    campaign = "halo-ac3"
    all_flights = [key for key in meta.transfer_calibs.keys()] if campaign == "cirrus-hl" else list(meta.flight_names.values())
    all_flights = all_flights[18:19]  # select single flight if needed
    time_step = pd.Timedelta(minutes=1)
    solar_flag = True  # True for solar wavelength range, False for terrestrial wavelength range
    use_dropsonde = True if campaign == "halo-ac3" else False

    # %% run for all flights
    for flight in all_flights:
        date = flight[9:17] if campaign == "halo-ac3" else flight[7:15]
        # set paths
        _base_dir = h.get_path("base", campaign=campaign)
        _libradtran_dir = h.get_path("libradtran", flight, campaign)
        _bahamas_dir = h.get_path("bahamas", flight, campaign)
        _bahamas_file = [f for f in os.listdir(_bahamas_dir) if f.endswith(".nc")][0]
        input_path = f"{_libradtran_dir}/wkdir/{'solar' if solar_flag else 'thermal'}"
        h.make_dir(input_path)  # create directory
        radiosonde_dir = f"{_base_dir}/../0{1 if campaign == 'halo-ac3' else 2}_soundings/RS_for_libradtran"
        # only needed for HALO-AC3
        dropsonde_path = f"{_base_dir}/../01_soundings/RS_for_libradtran/Dropsondes_HALO/Flight_{date}"
        solar_source_path = f"{_base_dir}/../00_tools/0{5 if campaign == 'cirrus-hl' else 8}_libradtran"

        bahamas_ds = reader.read_bahamas(f"{_bahamas_dir}/{_bahamas_file}")
        timestamp = bahamas_ds.time[0]
        while timestamp < bahamas_ds.time[-1]:
            bahamas_ds_sel = bahamas_ds.sel(time=timestamp)
            lat, lon, alt, pres, temp = bahamas_ds_sel.IRS_LAT.values, bahamas_ds_sel.IRS_LON.values, bahamas_ds_sel.IRS_ALT.values, bahamas_ds_sel.PS.values, bahamas_ds_sel.TS.values
            is_on_land = globe.is_land(lat, lon)  # check if location is over land
            zout = alt / 1000  # page 127; aircraft altitude in km
            # need to create a time zone aware datetime object to calculate the solar azimuth angle
            dt_timestamp = datetime.datetime.fromtimestamp(timestamp.values.astype('O') / 1e9, tz=datetime.timezone.utc)

            # define radiosonde station or dropsonde
            if use_dropsonde:
                dropsonde_files = [f for f in os.listdir(dropsonde_path)]
                dropsonde_files.sort()  # sort dropsonde files
                # read out timestamps from file
                dropsonde_times = [int(f[9:14].replace(" ", "")) for f in dropsonde_files]
                # convert to date time to match with BAHAMAS/INS timestamp
                dropsonde_times = pd.to_datetime(dropsonde_times, unit="s", origin=pd.to_datetime(date))
                # find index of the closest dropsonde to BAHAMAS/INS timestamp
                time_diffs = dropsonde_times - timestamp.values
                idx = np.asarray(np.nonzero(time_diffs == time_diffs.min()))[0][0]
                dropsonde_file = dropsonde_files[idx]
                radiosonde = f"{dropsonde_path}/{dropsonde_file} H2O RH"
            else:
                # radiosonde_station = find_closest_radiosonde_station(lat, lon)
                radiosonde_station = "Longyearbyen_01004"  # standard for HALO-(AC)3
                station_nr = radiosonde_station[-5:]
                if campaign == "halo-ac3":
                    radiosonde = f"{radiosonde_dir}/Radiosonde_for_libradtran_{station_nr}_{dt_timestamp:%Y%m%d}_12.dat H2O RH"
                else:
                    radiosonde = f"{radiosonde_dir}/{radiosonde_station}/{dt_timestamp:%m%d}_12.dat H2O RH"

            # get time in decimal hours
            decimal_hour = dt_timestamp.hour + dt_timestamp.minute / 60 + dt_timestamp.second / 60 / 60
            year, month, day = dt_timestamp.year, dt_timestamp.month, dt_timestamp.day
            if is_on_land:
                calc_albedo = 0.2  # set albedo to a fixed value (will be updated)
            else:
                # calculate cosine of solar zenith angle
                cos_sza = np.cos(np.deg2rad(
                    solar_position.get_sza(decimal_hour, lat, lon, year, month, day, pres, temp - 273.15)))
                # calculate albedo after Taylor et al 1996 for sea surface
                calc_albedo = 0.037 / (1.1 * cos_sza ** 1.4 + 0.15)

            # optionally provide libRadtran with sza (libRadtran calculates it itself as well)
            sza_libradtran = solar_position.get_sza(decimal_hour, lat, lon, year, month, day, pres, temp - 273.15)
            if sza_libradtran > 90:
                log.debug(f"Solar zenith angle for {dt_timestamp} is {sza_libradtran:.2f}!\n"
                          f"Skipping this timestamp and moving on to the next one.")
                # increase timestamp by time_step
                timestamp = timestamp + time_step
                continue

            # %% create input file
            _input_filename = f"{dt_timestamp:%Y%m%d_%H%M%S}_libRadtran.inp"
            _input_filepath = f"{input_path}/{_input_filename}"

            # %% set options for libRadtran run - atmospheric shell
            atmos_settings = dict(
                albedo=f"{calc_albedo:.4f}",
                altitude=0,  # page 80; ground height above sea level in km (0 for over ocean)
                # atmosphere_file="/opt/libradtran/2.0.4/share/libRadtran/data/atmmod/afglms.dat",  # page 81
                data_files_path="/opt/libradtran/2.0.4/share/libRadtran/data",  # location of internal libRadtran data
                latitude=f"N {lat:.6f}" if lat > 0 else f"S {-lat:.6f}",  # page 96
                longitude=f"E {lon:.6f}" if lon > 0 else f"W {-lon:.6f}",  # BAHAMAS: E = positive, W = negative
                mol_file=None,  # page 104
                mol_modify="O3 300 DU",  # page 105
                radiosonde=radiosonde,  # page 114
                time=f"{dt_timestamp:%Y %m %d %H %M %S}",  # page 123
                source=f"solar {solar_source_path}/NewGuey2003_BBR.dat" if solar_flag else "thermal",  # page 119
                sur_temperature="293.15",  # page 121; set to 20 degC for now
                # sza=f"{sza_libradtran:.4f}",  # page 122
                # verbose="",  # page 123
                # SMART wavelength range (179.5, 2225), BACARDI solar (290, 3600), BACARDI terrestrial (4000, 100000)
                wavelength="290 3600" if solar_flag else "4000 100000",
                zout=f"{zout:.3f}",  # page 127; altitude in km above surface altitude
            )

            # set options for libRadtran run - radiative transfer equation solver
            rte_settings = dict(
                rte_solver="fdisort2",
            )

            # set options for libRadtran run - post-processing
            postprocess_settings = dict(
                output_user="sza albedo zout edir edn eup",  # page 109
                output_process="integrate",  # page 108
            )

            # %% write input file
            with open(_input_filepath, "w") as ifile:
                ifile.write(f"# libRadtran input file generated with libradtran_write_input_file_bacardi.py "
                            f"({datetime.datetime.utcnow():%c UTC})\n")
                for settings, line in zip([atmos_settings, rte_settings, postprocess_settings],
                                          ["Atmospheric", "RTE", "Post Process"]):
                    ifile.write(f"\n# {line} Settings\n")
                    for key, value in settings.items():
                        if value is not None:
                            ifile.write(f"{key} {value}\n")

            # increase timestamp by time_step
            timestamp = timestamp + time_step
            # end of while