import os
import json
import numpy as np
import copy

#relative imports of Isotomics
import sys
import os
from pathlib import Path
cwd = Path().resolve()

sys.path.insert(1, os.path.join(cwd, 'Isotomics Scripts'))

import initializeMethionine as metInit
import fragmentAndSimulate as fas
import solveSystem as ss
import readInput as ri
import calcIsotopologues as ci

'''
This code is divided into the following sections:

1a) Precalculate isotopologues and save them for later. Here, do both the '5 frags' version with 13 sites and '8 frags' version with 16 sites.

2) Prepare to make the 'observed isotopologues' portion. Calclulate how many isotopologues are constrained assuming a perfect measurement relative to how many isotopologues exist.

3) Construct the figure for the 'isotopologues observed' plot

4) Generate a perfect dataset for 8 fragments, and compute how many beams we observe relative to how many we could observe
'''
'''
1a) Precalculate isotopologues and save them for later; this saves time.
'''
precalc = False
if precalc:
    #precalculate isotopologues of interest: 5 fragments
    deltasPreCalc = [0] * 13
    fragSubset = ['full','133','104','102','61','56']
    thisMethionine = metInit.initializeMethionine(deltasPreCalc, fragSubset, printHeavy = False)
    byAtom = ci.inputToAtomDict(thisMethionine['molecularDataFrame'], overrideIsoDict = {}, disable = False, M1Only = False)
    MN = ci.massSelections(byAtom, massThreshold = 4)
    overrideIsoDict = ci.pullOutIsotopologues(MN)

    with open(str(cwd) + '/Reference Files/Stored Isotopologues 5 Frags.json', 'w', encoding='utf-8') as f:
        json.dump(overrideIsoDict, f, ensure_ascii=False, indent=4)

    #precalculate isotopologues of interest: 8 fragments
    deltasPreCalc = [0] * 16
    fragSubset = ['full','133','104','102','87','74High','74Low','61','56']
    thisMethionine = metInit.initializeMethionine(deltasPreCalc, fragSubset, smallFrags = True, printHeavy = False)
    byAtom = ci.inputToAtomDict(thisMethionine['molecularDataFrame'], overrideIsoDict = {}, disable = False, M1Only = False)
    MN = ci.massSelections(byAtom, massThreshold = 4)
    overrideIsoDict = ci.pullOutIsotopologues(MN)

    with open(str(cwd) + '/Reference Files/Stored Isotopologues 8 Frags.json', 'w', encoding='utf-8') as f:
        json.dump(overrideIsoDict, f, ensure_ascii=False, indent=4)

with open(str(cwd) + '/Reference Files/Stored Isotopologues 5 Frags.json', 'r') as j:
    overrideIsoDict5 = json.loads(j.read())

with open(str(cwd) + '/Reference Files/Stored Isotopologues 8 Frags.json', 'r') as j:
    overrideIsoDict8 = json.loads(j.read())
'''
2) Prepare to make the 'observed isotopologues' portion. Calclulate how many isotopologues are constrained assuming a perfect measurement relative to how many isotopologues exist. Uses 5 fragments.
'''

#Initialize dictionaries to hold output
resultsAbbr = {}
deltasPerfect = [0] * 13
fragSubset = ['full','133','104','102','61','56']
thisMethionine = metInit.initializeMethionine(deltasPerfect, fragSubset,printHeavy = False)

#Simulate sample and standard datasets, then standard forward model.
predictedMeasurementSmp, MNDictSmp, FFSmp = metInit.simulateMeasurement(thisMethionine,
                                                        overrideIsoDict = overrideIsoDict5,
                                                       abundanceThreshold = 0,
                                                       massThreshold = 4, outputPath = 'Fig 5 Sample',
                                                       disableProgress = True)

predictedMeasurementStd, MNDictStd, FFStd = metInit.simulateMeasurement(thisMethionine,
                                                        overrideIsoDict = overrideIsoDict5,
                                                       abundanceThreshold = 0,
                                                       massThreshold = 4, outputPath = 'Fig 5 Standard',
                                                       disableProgress = True)

predictedMeasurementFMStd, MNDictFMStd, FFFMStd = metInit.simulateMeasurement(thisMethionine,
                                                        overrideIsoDict = overrideIsoDict5,
                                                       abundanceThreshold = 0,
                                                       massThreshold = 4,
                                                       disableProgress = True)

#Read in simulated sample and standard data, generated above.
standardJSON = ri.readJSON('Fig 5 Standard.json')
processStandard = ri.readComputedData(standardJSON, error = 0, theory = predictedMeasurementFMStd)

sampleJSON = ri.readJSON('Fig 5 Sample.json')
processSample = ri.readComputedData(sampleJSON, error = 0)
UValuesSmp = ri.readComputedUValues(sampleJSON, error = 0)

ri.checkSampleStandard(processSample, processStandard)
#solve M+1
isotopologuesDict = fas.isotopologueDataFrame(MNDictFMStd, thisMethionine['molecularDataFrame'])
OCorrection = ss.OValueCorrectTheoretical(predictedMeasurementFMStd, processSample, massThreshold = 4)
M1Results = ss.M1MonteCarlo(processStandard, processSample, OCorrection, isotopologuesDict,
                            thisMethionine['fragmentationDictionary'], perturbTheoryOAmt = 0,
                            N = 1, GJ = False, debugMatrix = False, disableProgress = True)

#Process results from M+N relative abundance space to U value space, using the UM+1 value calculated via the sub(s) in UMNSub.
processedResults = ss.processM1MCResults(M1Results, UValuesSmp, isotopologuesDict, thisMethionine['molecularDataFrame'], disableProgress = True,
                                        UMNSub = ['13C'])

#Update the dataframe with results.
ss.updateSiteSpecificDfM1MC(processedResults, thisMethionine['molecularDataFrame'])

M1Df = thisMethionine['molecularDataFrame'].copy()
M1Df['deltas'] = M1Df['VPDB etc. Deltas']

#Initialize dictionary for M+N solutions
MNSol = {'M2':0,'M3':0,'M4':0}
#Determine which substitutions are used for U Value scaling
UMNSubs = {'M2':['34S'],'M3':['34S15N'],'M4':['36S']}

for MNKey in MNSol.keys():
    Isotopologues = isotopologuesDict[MNKey]

    #Solve for specific M+N isotopologues
    results, comp, GJSol, meas = ss.MonteCarloMN(MNKey, Isotopologues, processStandard, processSample, OCorrection, thisMethionine['fragmentationDictionary'], N = 1, disableProgress = True)

    dfOutput = ss.checkSolutionIsotopologues(GJSol, Isotopologues, MNKey, numerical = False)
    #Many will be codependent, not individually constarined. Determine which isotopologues have actually been solved for.
    nullSpaceCycles = ss.findNullSpaceCycles(comp, Isotopologues)
    actuallyConstrained = ss.findFullyConstrained(nullSpaceCycles)

    #Scale results to U values, update the dataframe.
    processedResults = ss.processMNMonteCarloResults(MNKey, results, UValuesSmp, dfOutput, M1Df, MNDictFMStd, UMNSub = UMNSubs[MNKey], disableProgress = True)
    dfOutput = ss.updateMNMonteCarloResults(dfOutput, processedResults)
    MNSol[MNKey] = dfOutput.loc[dfOutput.index.isin(actuallyConstrained)].copy()

    if os.path.exists('Fig 5 Sample.json'):
        os.remove('Fig 5 Sample.json')
    if os.path.exists('Fig 5 Standard.json'):
        os.remove('Fig 5 Standard.json')

#Generate an abbreviated results file for easy use.
resultsAbbr = {'M1':{},'M2':{},'M3':{},'M4':{}}
resultsAbbr['M1']["Values"] = list(M1Df['Relative Deltas'])
resultsAbbr['M1']["Errors"] = list(M1Df['Relative Deltas Error'])

resultsAbbr['M2']['Identity'] = list(MNSol['M2'].index)
resultsAbbr['M2']['Values'] = list(MNSol['M2']['Clumped Deltas Relative'])
resultsAbbr['M2']['Errors'] = list(MNSol['M2']['Clumped Deltas Relative Error'])

resultsAbbr['M3']['Identity'] = list(MNSol['M3'].index)
resultsAbbr['M3']['Values'] = list(MNSol['M3']['Clumped Deltas Relative'])
resultsAbbr['M3']['Errors'] = list(MNSol['M3']['Clumped Deltas Relative Error'])

resultsAbbr['M4']['Identity'] = list(MNSol['M4'].index)
resultsAbbr['M4']['Values'] = list(MNSol['M4']['Clumped Deltas Relative'])
resultsAbbr['M4']['Errors'] = list(MNSol['M4']['Clumped Deltas Relative Error'])

'''
3) Construct the figure for the 'isotopologues observed' plot
'''
siteObs = {}
for MNKey in ['M2','M3','M4']:
    siteObs[MNKey] = {'Obs':0,'Possibly Observed':0}
    possiblyObservedIsotopologues = resultsAbbr[MNKey]['Identity']
    with open(str(cwd) + '/Read Process CSV/' + MNKey + 'ProcessedResults.json', 'r') as f:
        thisMNObs = json.load(f)

        for isotopologueID in possiblyObservedIsotopologues:
            siteObs[MNKey]['Possibly Observed'] += 1
            if isotopologueID in thisMNObs:
                siteObs[MNKey]['Obs'] += 1

    siteObs[MNKey]['Total Isotopologues'] = len(MNDictSmp[MNKey])

with open(str(cwd) + '/Read Process CSV/M1ProcessedResults.json', 'r') as f:
    thisMNObs = json.load(f)

M1Amt = len(thisMNObs['Mean'])
siteObs['M1'] = {'Obs':len(thisMNObs['Mean']),'Possibly Observed':len(resultsAbbr['M1']['Values']),'Total Isotopologues':len(thisMethionine['molecularDataFrame'])}

import matplotlib.pyplot as plt
import numpy as np
fig, axes = plt.subplots(nrows = 1, ncols = 4, figsize = (9,4), dpi = 150)
fig.set_facecolor('white')
blue3 = [0.7161860822760477,0.8332026143790849,0.916155324875048]
blue6 = [0.2909803921568628,0.5945098039215686,0.7890196078431373]
blue8 = [0.09019607843137256,0.39294117647058824,0.6705882352941177]
colors = [blue8, blue6, blue3]
labels = ['# Isotopologues Observed', '# Observable Isotopologues', '# Isotopologues Total']
for MNIdx, MNKey in enumerate(['M1','M2','M3','M4']):
    cAx = axes[MNIdx]
    nObs = siteObs[MNKey]['Obs']
    nPosObs =  siteObs[MNKey]['Possibly Observed']
    nTot = siteObs[MNKey]['Total Isotopologues']
    sizes = [nObs, nPosObs - nObs, nTot - nPosObs]

    print(MNKey)
    print(sizes)

    def actualValues(val):
        a  = np.round(val/100.*nTot, 0)
        return int(a)

    patches, texts, other = cAx.pie(sizes, colors = colors, autopct=actualValues, startangle = 90)
    #patches, texts = cAx.pie(sizes, colors = colors, startangle = 90)
    #cAx.set_title('M+' + MNKey[-1])

plt.legend(patches, labels, loc="best", bbox_to_anchor= [1.00,0.5])

plt.tight_layout()

#Output figure
fig.savefig(str(cwd) + '/Fig5_Percentage_Complete/Fig5Percent_Completion_Panel1.eps')

'''
4) Generate a perfect dataset for 8 fragments, and compute how many beams we observe relative to how many we could observe
'''
#Get how many beams we do observe here
with open(str(cwd) + '/Reference Files/MasterMNDict.json') as f:
    masterMNDict = json.load(f)

#Generate perfect dataset
deltasPerfect = [0] * 16
fragSubset = ['full','133','104','102','87','74High','74Low','61','56']

methioninePerfect = metInit.initializeMethionine(deltasPerfect, fragSubset, smallFrags = True, printHeavy = False)

predictedMeasurementPerfect, MNDictSmp, FFSmp = metInit.simulateMeasurement(methioninePerfect, overrideIsoDict = overrideIsoDict8,abundanceThreshold = 0, massThreshold = 4,disableProgress = False)

#Compare the number of beams observed
beamObs = {}
for MNKey in ['M1','M2','M3','M4']:
    beamObs[MNKey] = {'Obs':0,'Total':0}
    for fragKey, fragData in predictedMeasurementPerfect[MNKey].items():
        for subKey, subData in fragData.items():
            beamObs[MNKey]['Total'] += 1
            if fragKey in masterMNDict[MNKey]:
                if subKey in masterMNDict[MNKey][fragKey]:
                    beamObs[MNKey]['Obs'] += 1

#Construct figure
fig, axes = plt.subplots(nrows = 1, ncols = 4, figsize = (9,4), dpi = 150)
fig.set_facecolor('white')
blue3 = [0.7161860822760477,0.8332026143790849,0.916155324875048]
blue6 = [0.2909803921568628,0.5945098039215686,0.7890196078431373]
blue8 = [0.09019607843137256,0.39294117647058824,0.6705882352941177]
colors = [blue8, blue6]
for MNIdx, MNKey in enumerate(['M1','M2','M3','M4']):
    cAx = axes[MNIdx]
    nObs = beamObs[MNKey]['Obs']
    nTot =  beamObs[MNKey]['Total']
    sizes = [nObs, nTot - nObs]
    labels = ['# Ion Beams Observed', '# Ion Beams Total']

    def actualValues(val):
        a  = np.round(val/100.*nTot, 0)
        return int(a)

    patches, texts, other = cAx.pie(sizes, colors = colors, autopct=actualValues, startangle = 90)
    #patches, texts = cAx.pie(sizes, colors = colors, startangle = 90)
    #cAx.set_title('M+' + MNKey[-1])

plt.legend(patches, labels, loc="best", bbox_to_anchor= [1.00,0.5])

plt.tight_layout()

#Output figure
fig.savefig(str(cwd) + '/Fig5_Percentage_Complete/Fig5Percent_Completion_Panel2.eps')


import os
import time
from functools import partial
import random
from transformers import HfArgumentParser
from transformers.file_utils import ExplicitEnum

class BiasType(ExplicitEnum):
    LEXICAL = "lexical_bias"
    HYPOTHESIS = "hypothesis_only_bias"
    UNKNOWN = "unknown_bias"

def get_indices_arg(bias_type):
    evaluations_subsets = {
        BiasType.LEXICAL: [],
        BiasType.HYPOTHESIS: [
            # 'data/mnli_hypothesis_only_hard'
        ]
    }
    if bias_type == BiasType.UNKNOWN:
        evals = []
        # merge all evaluations sets and add unknown bias set
        for k in evaluations_subsets:
            evals += evaluations_subsets[k]
        # evals += ['data/tinybert_splits/5_folds_0.8_confidence/validation_matched']
        evaluations_subsets = evals
    else:
        evaluations_subsets = evaluations_subsets[bias_type]
    indices_arg = '--evaluate_on_hans' if bias_type in [BiasType.LEXICAL, BiasType.UNKNOWN] else ''
    if len(evaluations_subsets) > 0:
        indices_arg = indices_arg + f" --indices_dir  {' '.join(evaluations_subsets)}"

    if bias_type in [BiasType.UNKNOWN, BiasType.HYPOTHESIS]:
        indices_arg = f"{indices_arg} --mismatched_indices_dir data/hypothesis_bias_splits/validation_mismatched"
    return indices_arg

def sim_args(bias_type, biased_models, enforce_sim, unbiased_models):
    if enforce_sim:
        enforce_sim_arg = "--enforce_similarity"
        sim_name_arg = "_sim"
        weak_model_dir, fw_name = unbiased_models[bias_type]
    else:
        sim_name_arg = ""
        enforce_sim_arg = ""
        weak_model_dir, fw_name = biased_models[bias_type]
    return enforce_sim_arg, fw_name, sim_name_arg, weak_model_dir

def layer_to_arg(l, model_name='deberta'):
    layer_maper = {
        'P': 'pooler' if model_name == 'deberta' else 'bert.pooler',
        'e': f'{model_name}.embeddings'
    }
    return layer_maper[l] if l in layer_maper else f"{model_name}.encoder.layer.{l}"

def layers_args(regularized_layers_list, incraese_sim: bool):
    if type(regularized_layers_list) is tuple:
        main_model_layers = regularized_layers_list[0]
        weak_model_layers  = regularized_layers_list[1]
    else:
        main_model_layers = regularized_layers_list
        weak_model_layers = regularized_layers_list

    if type(main_model_layers) is not list:
        main_model_layers = [main_model_layers]
    if type(weak_model_layers) is not list:
        weak_model_layers = [weak_model_layers]

    weak_model_layers = list(map(str, weak_model_layers))
    main_model_layers = list(map(str, main_model_layers))

    layers_suffix = "-".join(weak_model_layers)
    layers_prefix = "-".join(map(lambda x: f"{x}M", main_model_layers))
    layers_arg = f"{layers_prefix}_{layers_suffix}"

    main_regularized_layers = " ".join(map(layer_to_arg, main_model_layers))
    if incraese_sim:
        weak_regularized_layers = " ".join(map(layer_to_arg, weak_model_layers))
    else:
        weak_regularized_layers = " ".join(map(partial(layer_to_arg, model_name='bert'), weak_model_layers))
    return layers_arg, main_regularized_layers, weak_regularized_layers

def only_bias_arg(reg_only_biased, bias_type: BiasType):
    bias_indices = {
        BiasType.LEXICAL: "data/lexical_bias_splits/train/confidence_based_0.65/train_biased_correct_indices.bin",
        BiasType.UNKNOWN: "data/tinybert_splits/5_folds_0.8_confidence/train/train_biased_correct_indices.bin",
        BiasType.HYPOTHESIS: "data/hypothesis_bias_splits/5_folds_0.6_confidence/train/train_biased_correct_indices.bin"
    }[bias_type]

    reg_only_biased_arg = ""
    reg_only_biased_identifier = ""
    if reg_only_biased:
        reg_only_biased_arg = f'--regularize_only_biased --bias_indices {bias_indices}'
        reg_only_biased_identifier = 'STOB_'
    return reg_only_biased_arg, reg_only_biased_identifier

def grads_arg(node_name, regularize_grads):
    grads_suffix = "_GReg" if regularize_grads else ""
    grads_arguments = "--regularize_grads" if regularize_grads else ""
    eval_bs = 64 if regularize_grads or node_name in ['plato1', 'plato2', 'nlp-2080-1', 'nlp-2080-2'] else 256
    return eval_bs, grads_arguments, grads_suffix

def main():
    parser = HfArgumentParser([])
    parser.add_argument('--regularize_only_grads', action='store_true', help='re-initialize weights of models randomly')
    parser.add_argument('--enforce_sim', action='store_true', help='re-initialize weights of models randomly')
    args = parser.parse_args()
    # main configs:
    regularize_only_grads = args.regularize_only_grads
    bias_type = BiasType.LEXICAL
    enforce_sim = args.enforce_sim

    # if regularize_only_grads:
    #     node_name = "nlp-a40-1"  # galileo2, newton2
    # else:
    #     node_name = "galileo2"  # plato1, plato2
    node_name = "nlp-a40-1"
    debertav3 = True
    if debertav3:
        unbiased_models = {
            BiasType.LEXICAL: ('runs/debertav3/extrensic_debiasing/lexical_bias/debertav3_64_lexical_extrensic_debiasing/checkpoint-11678', 'EDXE'),
            BiasType.UNKNOWN: ('runs/debertav3/extrensic_debiasing/unknown_bias/debertav3_64_unknown_extrensic_debiasing/checkpoint-9020', 'EDXE'),
            BiasType.HYPOTHESIS: ('runs/debertav3/extrensic_debiasing/hypohtesis_only_bias/debertav3_64_hypothesis_extrensic_debiasing/checkpoint-7044', 'EDXE')
        }
    else:
        unbiased_models = {
            BiasType.LEXICAL: ('runs/deberta/extrensic_debiasing/lexical_bias/deberta_64_lexical_extrensic_debiasing/checkpoint-11678', 'EDXE'),
            BiasType.UNKNOWN: ('runs/deberta/extrensic_debiasing/unknown_bias/deberta_64_unknown_extrensic_debiasing/checkpoint-9020', 'EDXE'),
            BiasType.HYPOTHESIS: ('runs/deberta/extrensic_debiasing/hypothesis_bias/deberta_64_hypothesis_extrensic_debiasing/checkpoint-7044', 'EDXE')
        }
    biased_models = {
        BiasType.LEXICAL: ('runs/bert_lexical_bias', 'LexicalModel'),
        BiasType.HYPOTHESIS: ([f'runs/5_fold_hypothesis_only/bert_{i}_hypothesis_only/checkpoint-14724' for i in range(4)], 'HOBert'),
        BiasType.UNKNOWN: ([f'runs/5_folds_unknown_bias/tinybert_{i}' for i in range(5)], 'TB')
    }
    # BiasType.LEXICAL: ('runs/deberta/extrensic_debiasing/lexical_bias/deberta_64_lexical_extrensic_debiasing', 'ED'),

    # BiasType.LEXICAL: ('runs/bert_lexical_bias', 'LexicalModel'),
    # BiasType.LEXICAL: ('runs/deberta/UBD/lexical_bias/UB_deberta', 'UBD'),

    batch_size = 64
    epochs = 10
    eval_bs, grads_arguments, grads_suffix = grads_arg(node_name, regularize_grads=regularize_only_grads)

    enforce_sim_arg, fw_name, sim_name_arg, weak_model_dir = sim_args(bias_type, biased_models, enforce_sim, unbiased_models)

    run_name_identifier = {
        BiasType.LEXICAL: 'lexicalDeBias',
        BiasType.HYPOTHESIS: 'HypoDeBias',
        BiasType.UNKNOWN: 'UNB'
    }[bias_type]

    main_task_lambda = 1
    indices_arg = get_indices_arg(bias_type)
    tags_arg = f'--tags {bias_type.value}'
    bias_dir = {
        BiasType.LEXICAL: 'lexical_bias_debiasing',
        BiasType.HYPOTHESIS: 'hypothesis_bias_debiasing',
        BiasType.UNKNOWN: 'unknown_bias/0.8_confidence'
    }[bias_type]
    bias_dir = bias_dir + ("/similarity" if enforce_sim else "/dissimilarity")

    reg_only_biased_arg, reg_only_biased_identifier = only_bias_arg(reg_only_biased=True, bias_type=bias_type)
    model_arg = 'microsoft/deberta-base' if not debertav3 else 'microsoft/deberta-v3-base'
    base_dir = 'deberta' if not debertav3 else 'debertav3'
    counter = 0
    for regularization_lambda in [100]:
        for regularization_method in ["linear_cka"]:
            # (['e', 10, 11], ['e', 1, 1]), ['e', 0, 6, 10, 11], ['e', 0, 6, 10, 11, 'P']
            for l in [[9, 10, 11]]:
                layers_arg, main_regularized_layers, weak_regularized_layers = layers_args(regularized_layers_list=l, incraese_sim=enforce_sim)
                reg_tokens_arg = "all" if 'P' not in l else " ".join(map(lambda x: "none" if x == 'P' else "all", l))
                model_name = f"{base_dir}_{fw_name}_{reg_only_biased_identifier}{run_name_identifier}_{regularization_method}_{layers_arg}_lambda_" \
                             f"{main_task_lambda}_{regularization_lambda}_{batch_size}{sim_name_arg}{grads_suffix}"
                seeds = [random.randint(0, 10000), random.randint(0, 1000)]
                for seed in seeds:
                    if type(weak_model_dir) is list:
                        current_seed_weak_model = weak_model_dir[random.randint(0, len(weak_model_dir) - 1)]
                    else:
                        current_seed_weak_model = weak_model_dir
                    output_dir = f"runs/{base_dir}/{bias_dir}/{regularization_method}/{model_name}_s{seed}"
                    tokenization_arg = '--separate_weak_tokenization' if bias_type in [BiasType.HYPOTHESIS, BiasType.UNKNOWN] and not enforce_sim else ''
                    padding_arg = '' #'--no_pad_to_max_length' if enforce_sim else ''
                    os.system(f"""
                        echo nlp_sbatch {node_name} run_glue.py --report_to wandb --model_name_or_path {model_arg} \
                            --output_dir {output_dir} --num_train_epochs {epochs} --learning_rate 2e-5 --per_device_eval_batch_size {eval_bs} \
                             --task_name mnli --do_train --do_eval --max_seq_length 128 --per_device_train_batch_size {batch_size} \
                             --weak_model_layer {weak_regularized_layers} --weak_model_path {current_seed_weak_model} --regularized_layers {main_regularized_layers} \
                             --regularization_method {regularization_method} --evaluation_strategy half_epoch --save_strategy no \
                             --regularization_lambda {regularization_lambda} --run_name {model_name} --weight_decay 0.1 --warmup_ratio 0.1 \
                             --logging_first_step --regularized_tokens {reg_tokens_arg} --token_aggregation_strategy mean {enforce_sim_arg} \
                             {grads_arguments} {indices_arg} {tags_arg} {reg_only_biased_arg} --main_task_lambda {main_task_lambda} \
                             --bias_sampling_strategy stochastic --seed {seed} --wandb_group {tokenization_arg} {padding_arg}
                    """)
                    print()
                    counter += 1

    print(f"dispatched {counter} jobs")

if __name__ == '__main__':
    main()


"""
"""
import numpy as np
from astropy.utils.misc import NumpyRNGContext

def sigmoid_downsampling_mask(mag, cut, k=5, seed=None):
    """ Function calculates a boolean array to mask out faint galaxies.

    Downsampling is done gradually as magnitude exceeds the input cut.

    For plots of the behavior, see
    https://gist.github.com/aphearin/d6d12fd0c759a931748c4435ebf4f84c

    Parameters
    ----------
    mag : ndarray
        ndarray of shape (ngals, ) storing apparent magnitude

    cut : float
        Parameter specifying the soft cut magnitude.

        Values of mag larger than the cut will be masked,
        with the soft transition implemented by a sigmoid function.

    k : int, optional
        Slope of the sigmoid function controlling the
        gradual selection probability. Default is 5.

    seed : int, optional
        Default is None, for stochastic results

    Returns
    -------
    mask : ndarray
        ndarray of shape (ngals, ) with boolean dtype
    """
    ntot = len(mag)
    with NumpyRNGContext(seed):
        uran = np.random.rand(ntot)
    p = sigmoid(mag, x0=cut, k=k)
    mask = p < uran
    return mask

def sigmoid(x, x0=0, k=1, ymin=0, ymax=1):
    height_diff = ymax-ymin
    return ymin + height_diff/(1 + np.exp(-k*(x-x0)))


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('measuring_stations', '0001_initial'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='measuringpoint',
            options={'ordering': ['name'], 'verbose_name': 'Messstelle', 'verbose_name_plural': 'Messstellen'},
        ),
        migrations.AddField(
            model_name='measuringpoint',
            name='slug',
            field=models.SlugField(default=''),
            preserve_default=False,
        ),
    ]


from django.urls import path
from . import views
#from .views import (requestSummary, requestConfirm, myRequest)

urlpatterns = [
  path('', views.home, name='home'),
  path('register', views.register, name='register'),
    path('sciclub', views.sciclub, name='sciclub'),
    path('acomclub', views.acomclub, name='acomclub'),
     path('asciclub', views.asciclub, name='asciclub'),
    path('comclub', views.comclub, name='comclub'),
      path('coordinator', views.coordinator, name='coordinator'),
      path('accept/<int:pk>/', views.accept, name='accept'),
]


# Trying to download YouTube video in audio format using Python
# First install pytube 3 from pip-- pip install pytube3
# Next, import YouTube module
from pytube import YouTube
# Welcome message
print("\n...Welcome to Denzel YouTube Downloader ðŸ¤—ðŸŽ¬ðŸŽ¶...")
# Allow the program to run until the user exits
rerun = "y"
while rerun.lower() == "y":
    # Get the user input and handle any errors-- video link:
    video_link = input("Enter the link of the video: ")
    save_location = "C:/Users/giton/OneDrive/Desktop"
    # User to select download format
    download_format = input("Which format do you want to download in? \n[Type video for video and audio for audio]: ")
    # Download audio
    if download_format.lower() == "audio":
        # To handle the connection errors:
        try:
            download_audio = YouTube(video_link)
            print("\nDownloading {}...\n...Please wait...\n...Almost there...".format(download_audio.title))
            audio = download_audio.streams.filter(only_audio=True, file_extension='mp4').first()
            audio.download(save_location)

            print("Download complete, check in your file ðŸ˜‰.\nEnjoy listening to {}".format(download_audio.title))

        except:
            print("...Connection Error...")
    # Download video
    elif download_format.lower() == "video":
        try:
            download_video = YouTube(video_link)
            print("\nDownloading {}...\n...Please wait...\n...Almost there...".format(download_video.title))
            video = download_video.streams.filter(only_audio=False, only_video=False, file_extension='mp4').first()
            video.download(save_location)

            print("Download complete, check in your file ðŸ˜‰.\nEnjoy watching {}".format(download_video.title))
        except:
            print("...Connection Error...")
    else:
        print("Something's not right...")

    # Ask if the user wishes to continue
    rerun = input("Would you like to download another one? \n[Type y to continue or x to exit] ")
# If the user exits
print("You're welcome ðŸ˜‰... \nFollow me on: \nTikTok: https://tinyurl.com/Denzel-TikTok \nYouTube: https://tinyurl.com/Denzel-YouTube \nInstagram: https://tinyurl.com/Denzel-Instagram \nTwitter: https://tinyurl.com/Denzel-Twitter \nLinkedIn: https://tinyurl.com/Denzel-LinkedIn")


import cv2
import numpy as np
import numpy.linalg as la
import os
from pathlib import Path
from os import walk

ABSOLUTE_PATH = Path.cwd() / 'test'
print(ABSOLUTE_PATH)

files = []
for (dirpath, dirnames, filenames) in walk(ABSOLUTE_PATH):
    files.extend(filenames)
    break

imgs= []

#read input
for f in files:
    if 'jpg' in f and 'background' not in f:
        imgs.append(cv2.imread(str(ABSOLUTE_PATH) + '/' + f))

#generate output
h, w = imgs[0].shape[:2]
img_out = np.zeros((h, w, 3), np.uint8)
num = len(imgs)

for i in range(h):
    for j in range(w):
        r = g = b = 0

        for img in imgs:
            r = r + img[i, j][0]
            g = g + img[i, j][1]
            b = b + img[i, j][2]
#            print "original: ", img[i, j]

        img_out[i, j] = [r/num, g/num, b/num]
#        print "background: ", img_out[i, j]

cv2.imwrite('backgroud_algo1.png', img_out)


import os
import pickle

import numpy as np
import pandas as pd
from tqdm import tqdm

from common.utils import read_sas7bdata_pd

sides = [None, 'R', 'L']

def build_single_img_based_multi_progression_meta(oai_src_dir, only_baseline=False, use_sas=False):
    # visits =     ['00', '12', '24', '36', '72', '96']
    # exam_codes = ['00', '01', '03', '05', '08', '10']

    visits = ['00', '12', '24', '36', '48', '72', '96']
    exam_codes = ['00', '01', '03', '05', '06', '08', '10']

    # visits =     ['48', '72', '96']
    # exam_codes = ['06', '08', '10']

    non_progressor_code = len(visits)

    grading_types = ['KL', 'OSTL', 'OSTM', 'OSFL', 'OSFM', 'JSL', 'JSM']
    max_grades = {'KL': 4, 'OSTL': 3, 'OSTM': 3, 'OSFL': 3, 'OSFM': 3, 'JSL': 3, 'JSM': 3}
    include_TKR = True
    grad_files = []
    if not os.path.isfile("oai_master_df.csv") or not os.path.isfile("oai_grad_files.pkl"):

        for i, visit in enumerate(visits):
            print(f'==> Reading OAI {visit} visit')

            if use_sas:
                meta = read_sas7bdata_pd(os.path.join(oai_src_dir,
                                                      'Semi-Quant Scoring_SAS',
                                                      f'kxr_sq_bu{exam_codes[i]}.sas7bdat'))
            #     data_clinical = read_sas7bdata_pd(os.path.join(oai_src_dir, f'allclinical{exam_codes[i]}.sas7bdat'))
            else:
                meta = pd.read_csv(os.path.join(oai_src_dir,
                                                'Semi-Quant Scoring_ASCII',
                                                f'kxr_sq_bu{exam_codes[i]}.txt'), sep='|')
            #     # data_clinical = pd.read_csv(os.path.join(oai_src_dir, 'AllClinical_ASCII', f'AllClinical{exam_codes[i]}.txt'), sep='|')
            data_clinical = build_clinical(oai_src_dir, visit_id=exam_codes[i], use_sas=True)

            meta['ID'] = meta['ID'].astype(str)
            meta.replace({'SIDE': {1: sides[1], 2: sides[2]}}, inplace=True)
            data_clinical['ID'] = data_clinical['ID'].astype(str)
            data_clinical.rename(columns={'Side': 'SIDE'}, inplace=True)

            meta = meta.merge(data_clinical, on=['ID', 'SIDE'], how='left')
            # Dropping the data from multiple projects
            meta.drop_duplicates(subset=['ID', 'SIDE'], inplace=True)
            meta.fillna(-1, inplace=True)
            for c in meta.columns:
                meta[c.upper()] = meta[c]
            # Removing the TKR and KL4 at the baseline
            if i == 0:
                meta = meta[meta[f'V{exam_codes[i]}XRKL'] != -1]
                # meta = meta[meta[f'V{exam_codes[i]}XRKL'] < 4]
            # meta = meta[meta[f'V{exam_codes[i]}XRKL'] <= max_grades['KL']]

            meta['KL'] = meta[f'V{exam_codes[i]}XRKL']
            meta['OSTL'] = meta[f'V{exam_codes[i]}XROSTL']
            meta['OSTM'] = meta[f'V{exam_codes[i]}XROSTM']

            meta['OSFL'] = meta[f'V{exam_codes[i]}XROSFL']
            meta['OSFM'] = meta[f'V{exam_codes[i]}XROSFM']

            meta['JSL'] = meta[f'V{exam_codes[i]}XRJSL']
            meta['JSM'] = meta[f'V{exam_codes[i]}XRJSM']

            # Merge levels 0 into 1 in every grading
            # Let TKR be the highest level
            for grading in grading_types:
                meta[grading] = meta[grading].round(decimals=0)
                if grading == "KL":
                    # Remove invalid records that are out of upper bound
                    # meta = meta[meta[grading] <= max_grades[grading]]
                    meta.loc[meta[grading] > max_grades[grading], grading] = None

                    # Set TKR the highest level
                    if include_TKR:
                        meta.loc[meta[grading] == -1, grading] = max_grades[grading] + 1
                        max_kl = max_grades[grading] + 1
                    else:
                        max_kl = max_grades[grading]

                    # Remove invalid records that are out of lower bound
                    # meta = meta[meta[grading] >= 0]
                    meta.loc[meta[grading] < 0, grading] = None

                    for v in range(max_kl):
                        meta = meta.replace({grading: {v + 1: v}})
                else:
                    meta.loc[meta[grading] < 0, grading] = None

            if i == 0:
                _pain_key = f'P01'
            else:
                _pain_key = f'V{exam_codes[i]}'

            # Add visit columns
            meta['visit'] = int(visit)
            meta['visit_id'] = int(exam_codes[i])

            grad_files.append(
                meta[['ID', 'SIDE', 'KL', 'visit', 'visit_id', 'OSTL', 'OSTM', 'OSFL', 'OSFM', 'JSL', 'JSM',
                      'AGE', 'SEX', 'BMI', 'INJ', 'SURG', 'WOMAC', 'V00SITE']])

        id_set_last_fu = set(grad_files[-1].ID.values.astype(int).tolist())  # Subjects present at all FU

        master_df = pd.concat(grad_files)

        with open("oai_grad_files.pkl", "wb") as f:
            pickle.dump(grad_files, f, protocol=4)
        master_df.to_csv("oai_master_df.csv", index=None)
    else:
        print(f'Loading oai_grad_files.pkl')
        with open("oai_grad_files.pkl", "rb") as f:
            grad_files = pickle.load(f)
        master_df = pd.read_csv("oai_master_df.csv")

    master_df['ID'] = master_df['ID'].astype(str)
    # Get baseline and follow-ups
    # for follow_up_id in range(0, len(KL_files)):
    #     KL_files[follow_up_id] = KL_files[follow_up_id].set_index(['ID', 'SIDE'])

    # master_df = master_df.set_index(['ID', 'SIDE'])

    # looking for progressors
    identified_prog = set()

    fus_df = []

    n_bs_knees = grad_files[0].shape[0]

    for bs_knee_id, knee in tqdm(grad_files[0].iterrows(), total=n_bs_knees, desc='Processing OAI:'):
        if int(knee.ID) in identified_prog:
            if identified_prog[int(knee.ID)] == sides[int(knee.SIDE)]:
                continue

        participant = master_df[(master_df['ID'] == knee.ID) & (master_df['SIDE'] == knee.SIDE)]
        participant.sort_values(by=['visit_id'])

        assert len(participant) > 0

        n_follow_ups = len(participant.index)

        for fu1_id in range(n_follow_ups - 1):
            valid_gradings1 = get_valid_gradings_1st_fu(participant.iloc[fu1_id], grading_types, max_grades)

            fu1 = participant.iloc[fu1_id]  # .add_suffix('1')

            # Condition if forced to use baseline only
            if only_baseline:
                fu1_cond = fu1['visit'] == 0
            else:
                fu1_cond = True

            if len(valid_gradings1) == 0:
                print(f'No valid gradings in FU1.')
                continue

            if not fu1_cond:
                print(f'Skip as the initial data is not a baseline.')
                continue

            start_visit = fu1['visit']

            fus = fu1

            valids = []
            for grade_id, grading in enumerate(grading_types):
                prev_grade = fu1[f'{grading}']

                reach_max = False
                valid_by_grading = True
                fus_by_grading = None
                for fu2_id in range(fu1_id + 1, n_follow_ups):
                    fu2 = participant.iloc[fu2_id].add_suffix('2')
                    dt12_y = int((fu2['visit2'] - start_visit) // 12)

                    if not check_valid_grading(fu2[f'{grading}2']):
                        continue

                    # Time points after a time point with max grade must be graded the max value
                    if reach_max or prev_grade == max_grades[grading]:
                        reach_max = True
                        fu2[f'{grading}2'] = max_grades[grading]

                    valid_bl = check_progression_validity(fu1[f'{grading}'], fu2[f'{grading}2'])
                    valid_prev = check_progression_validity(prev_grade, fu2[f'{grading}2'])
                    if not valid_bl or not valid_prev:
                        print(
                            f'[Invalid] {grading}, BL: {fu1[grading]}, prev: {prev_grade}, cur: {fu2[f"{grading}2"]}.')
                        valid_by_grading = False
                        break
                    if prev_grade == 1.0 and fu2[f"{grading}2"] == 0.0:
                        print(f'{grading}, BL: {fu1[grading]}, prev: {prev_grade}, cur: {fu2[f"{grading}2"]}.')

                    new_cols = {}
                    new_cols[f'DT_{dt12_y}y'] = (fu2['visit2'] - start_visit) / 12.0
                    new_cols[f'{grading}_{dt12_y}y'] = fu2[f'{grading}2']
                    new_cols = pd.Series(new_cols)

                    if fus_by_grading is None:
                        fus_by_grading = new_cols
                    else:
                        fus_by_grading = pd.concat([fus_by_grading, new_cols])

                    if check_valid_grading(fu2[f'{grading}2']):
                        prev_grade = fu2[f'{grading}2']

                if not valid_by_grading:
                    fus_by_grading = None

                # End fu2 loop
                if fus_by_grading is not None:
                    fus = pd.concat([fus, fus_by_grading])
                valids.append(valid_by_grading)

            # End grading loop
            if any(valids):
                fus_df.append(fus.to_dict())

    print(f'\nConverting to dataframes...')
    fus_df = pd.DataFrame(fus_df)

    # Remove redundant columns
    # Rename ID and Side
    fus_df = fus_df.rename(columns={'SIDE': 'Side'})
    if 'IDs' in fus_df:
        fus_df = fus_df.drop(columns=['ID2'])
    if 'SIDE2' in fus_df:
        fus_df = fus_df.drop(columns=['SIDE2'])

    fus_df = fus_df.astype({'ID': str})

    return fus_df

def check_valid_grading(v):
    return v is not None and not np.isnan(v)

def build_clinical(oai_src_dir, use_sas=False, visit_id='00'):
    if use_sas:
        data_enrollees = read_sas7bdata_pd(os.path.join(oai_src_dir, 'AllClinical_SAS', 'enrollees.sas7bdat'))
        data_clinical = read_sas7bdata_pd(
            os.path.join(oai_src_dir, 'AllClinical_SAS', f'allclinical{visit_id}.sas7bdat'))
    else:
        data_enrollees = pd.read_csv(os.path.join(oai_src_dir, 'General_ASCII', 'Enrollees.txt'), sep='|')
        data_clinical = pd.read_csv(os.path.join(oai_src_dir, 'AllClinical_ASCII', f'AllClinical{visit_id}.txt'),
                                    sep='|')

    clinical_data_oai = data_clinical.merge(data_enrollees, on='ID')

    clinical_data_oai = clinical_data_oai.replace({'0: No': 0, '1: Yes': 1})

    if visit_id == '00':
        AGE_col = 'V00AGE'
        BMI_col = 'P01BMI'
        HEIGHT_col = 'P01HEIGHT'
        WEIGHT_col = 'P01WEIGHT'
        INJL_col = 'P01INJL'
        INJR_col = 'P01INJR'
        SURGL_col = 'P01KSURGL'
        SURGR_col = 'P01KSURGR'
        WOMACL_col = 'V00WOMTSL'
        WOMACR_col = 'V00WOMTSR'
    else:
        AGE_col = f'V{visit_id}AGE'
        BMI_col = f'V{visit_id}BMI'
        HEIGHT_col = f'V{visit_id}HEIGHT'
        WEIGHT_col = f'V{visit_id}WEIGHT'
        INJL_col = f'V{visit_id}INJL12'
        INJR_col = f'V{visit_id}INJR12'
        SURGL_col = f'V{visit_id}KSRGL12'
        SURGR_col = f'V{visit_id}KSRGR12'
        WOMACL_col = f'V{visit_id}WOMTSL'
        WOMACR_col = f'V{visit_id}WOMTSR'

    # Age, Sex, BMI
    clinical_data_oai['SEX'] = clinical_data_oai['P02SEX']
    clinical_data_oai['AGE'] = clinical_data_oai[AGE_col]
    clinical_data_oai['BMI'] = clinical_data_oai[BMI_col]

    # clinical_data_oai['HEIGHT'] = clinical_data_oai[HEIGHT_col]
    # clinical_data_oai['WEIGHT'] = clinical_data_oai[WEIGHT_col]

    clinical_data_oai_left = clinical_data_oai.copy()
    clinical_data_oai_right = clinical_data_oai.copy()

    # Making side-wise metadata
    clinical_data_oai_left['Side'] = 'L'
    clinical_data_oai_right['Side'] = 'R'

    # Injury (ever had)
    clinical_data_oai_left['INJ'] = clinical_data_oai_left[INJL_col]
    clinical_data_oai_right['INJ'] = clinical_data_oai_right[INJR_col]

    # Surgery (ever had)
    clinical_data_oai_left['SURG'] = clinical_data_oai_left[SURGL_col]
    clinical_data_oai_right['SURG'] = clinical_data_oai_right[SURGR_col]

    # Total WOMAC score
    clinical_data_oai_left['WOMAC'] = clinical_data_oai_left[WOMACL_col]
    clinical_data_oai_right['WOMAC'] = clinical_data_oai_right[WOMACR_col]

    clinical_data_oai_left['V00SITE'] = clinical_data_oai['V00SITE']
    clinical_data_oai_right['V00SITE'] = clinical_data_oai['V00SITE']

    clinical_data_oai = pd.concat((clinical_data_oai_left, clinical_data_oai_right))
    clinical_data_oai.ID = clinical_data_oai.ID.values.astype(str)

    for col in ['BMI', 'INJ', 'SURG', 'WOMAC']:
        clinical_data_oai.loc[clinical_data_oai[col].isin(['.: Missing Form/Incomplete Workbook'])] = None
        clinical_data_oai.loc[clinical_data_oai[col] < 0, col] = None
    return clinical_data_oai[['ID', 'Side', 'AGE', 'SEX', 'BMI', 'INJ', 'SURG', 'WOMAC', 'V00SITE']]

def get_valid_gradings_1st_fu(r, grading_types, max_grades):
    valid_gradings = []
    for grading in grading_types:
        if r[grading] is None or np.isnan(r[grading]) or 0 <= r[grading] < max_grades[grading]:
            valid_gradings.append(grading)
        else:
            # print(f'Init input violation: grading {grading} = {r[grading]}.')
            pass
    return valid_gradings

def check_progression_validity(g1, g2):
    if g1 is None or g2 is None or np.isnan(g1) or np.isnan(g2):
        valid = True
    elif g1 <= g2:
        valid = True
    else:
        valid = False
    return valid


from flask import render_template,request,redirect,url_for,flash,abort
from . import main
from .. import db
from .forms import PostForm, CommentForm, SubscriptionForm
from ..models import User,Writer,Post,Comment,Subscription
from flask_login import login_required,current_user
from app import login_manager

@login_manager.user_loader
def load_user(user_id):
    '''
    @login_manager.user_loader Passes in a writer_id to this function
    Function queries the database and gets a user's id as a response
    '''
    return User.query.get(user_id)

#Views
@main.route('/')
def index():

    # if post.validate_on_submit():
    # title = post.title.data,
    # body = post.body.data

    posts=Post.query.all()

    # return redirect(url_for('main.index'))
    return render_template('index.html', posts=posts)

@main.route('/post', methods=['GET','POST'])
# def post():
#     '''
#     Post page
#     '''
#     post_form = PostForm()

#     if post_form.validate_on_submit():
#         title = post_form.title.data
#         body = post_form.body.data
#         # time_posted = post_form.time_posted.data
#         new_post = Post(title=title, body=body)
#         db.session.add(new_post)
#         db.session.commit()
#         return redirect(url_for('main.index'))

#     return render_template('post.html', post_form=post_form)

def new_post():
    '''
    New Post Page
    '''
    post_form = PostForm()

    if post_form.validate_on_submit():
        title = post_form.title.data
        body = post_form.body.data

        # updated review instance
        new_post = Post(title=title,body = body)

        #save review method
        new_post.save_post()
        return redirect(url_for('main.index'))

    title = f'{Post.title}'
    return render_template('post.html',title= title, post_form=post_form)

@main.route('/comments/<int:id>', methods=['GET', 'POST'])
@login_required
def comments(id):
    comment_form = CommentForm()
    comment = Comment.query.order_by('-id').all()

    if comment_form.validate_on_submit():
        comment = comment_form.comment.data()
        new_comment = Comment(comment=comment,users=current_user.username)
        new_comment.save_comment()
        return redirect(url_for('main.comments',id = post.id))
    return render_template('comments.html')

@main.route('/delete/<int:id>')
@login_required
def delete(id):
    del_comment = Comment.query.get(id)
    db.session.delete(del_comment)
    db.session.commit()
    return redirect(url_for('main.index'))

@main.route('/subscription',methods = ["GET","POST"])
def subscriber():

    form= SubscriptionForm()

    if form.validate_on_submit():
        email = form.email.data
        date = form.date.data

        # updated review instance
        new_subscriber = Subscription(email=email,date = date,user_id=current_user.id)

        #save review method
        new_subscriber.save_subscriber()
        return redirect(url_for('subscriber'))

    return render_template('index.html',email= email, subscribe_form=form )


import os
import sys
from urllib import request
from urllib.error import HTTPError

import pytest
import toml

import check_docker as module
from check_docker import check_swarm, check_docker

def test_versions_match():
    assert check_docker.__version__ == check_swarm.__version__

def test_module_version_matches():
    assert module.__version__ == check_docker.__version__

def test_project_version_matches():
    project_config = toml.load("pyproject.toml")
    project_version = project_config['tool']['poetry']['version']

    assert project_version == check_docker.__version__

@pytest.mark.skipif('isolated' in os.environ and os.environ['isolated'].lower != 'false',
                    reason="Can not reach Python packge index when isolated")
@pytest.mark.skipif(sys.version_info[0:2] != (3, 8), reason="Only check on python 3.8, not {}".
                    format(sys.version_info[0:2]))
def test_package_present():
    req = request.Request("https://pypi.org/project/check_docker/", method="HEAD")
    with request.urlopen(req) as resp:
        assert resp.getcode() == 200

@pytest.mark.xfail('TRAVIS_BRANCH' in os.environ and os.environ['TRAVIS_BRANCH'].lower != 'master',
                   reason="Ignore version check outside of master")
@pytest.mark.xfail('GITHUB_HEAD_REF' in os.environ and os.environ['GITHUB_HEAD_REF'].lower != 'master',
                   reason="Ignore version check outside of master")
@pytest.mark.skipif('isolated' in os.environ and os.environ['isolated'].lower != 'false',
                    reason="Can not reach Python package index when isolated")
@pytest.mark.skipif(sys.version_info[0:2] != (3, 8), reason="Only check on python 3.8")
def test_ensure_new_version():
    version = check_docker.__version__
    req = request.Request("https://pypi.org/project/check_docker/{version}/".
                          format(version=version), method="HEAD")

    try:
        with request.urlopen(req) as resp:
            http_code = resp.getcode()
    except HTTPError as e:
        http_code = e.code
    assert http_code == 404, "Version already exists. Ignore this if you are working on a PR"


import os
import os.path
import shutil
import re
import json

re_apikey = re.compile(r"apikey=([^&#]*)", re.IGNORECASE)
re_code = re.compile(r"code=([^&#]*)", re.IGNORECASE)
re_verifier = re.compile(r"verifier=([^&#]*)", re.IGNORECASE)
re_near_store_id = re.compile(r"nearStoreNumbers=([^&#]*)", re.IGNORECASE)
re_date_from = re.compile(r"from=(\d{4}-\d{1,2}-\d{1,2})", re.IGNORECASE)
re_date_to = re.compile(r"to=(\d{4}-\d{1,2}-\d{1,2})", re.IGNORECASE)

aadhi_prefix_regex = re.compile(r"aadhi.cma.r53.nordstrom.net(:443)?/", re.IGNORECASE)

headers_to_remove = ["connection", "content-length"]

NOT_FOUND_SKIP_RULES = [
    "\"message\": \"Not Found\",",
    "\"status\": \"404\""
]

def clear_headers(h):
    return {k: v for k, v in h.items() if k.lower() not in headers_to_remove}

def copy_stub(src, dst):
    os.makedirs(dst)
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if "resp_body.json" == item:
            shutil.copy2(s, d)
        elif "resp_header.json" == item:
            with open(s) as sfp, open(d, "w+") as dfp:
                h = json.load(sfp)
                h = clear_headers(h)
                json.dump(h, dfp, indent=2)

def read_content(src_path, id):
    body_path = os.path.join(src_path, id, "resp_body.json")
    if not os.path.isfile(body_path):
        return []
    with open(body_path, 'r') as f:
        content = f.readlines()
        return content

def check_skip_by_code(src_path, id, code):
    if code == "404":
        content = read_content(src_path, id)
        for line in content:
            line = line.strip()
            if line in NOT_FOUND_SKIP_RULES:
                return True
    return False

def clear_url(url):
    url =  aadhi_prefix_regex.sub("", url)

    url = re.sub(re_apikey, "apikey=*", url)
    url = re.sub(re_code, "code=*", url)
    url = re.sub(re_verifier, "verifier=*", url)
    url = re.sub(re_near_store_id, "nearStoreNumbers=*", url)
    url = re.sub(re_date_from, "from=*", url)
    url = re.sub(re_date_to, "to=*", url)

    return url


import os, re, json
import datetime as dt
from dateutil.relativedelta import relativedelta
import pandas as pd
import numpy as np
from unidecode import unidecode
from pdfminer.high_level import extract_text

def get_clinical_tests_available(file:str) -> dict:
    """
    Load available clinical tests from json file with each regular expression
    used to extract your results and personal information based on template
    file

    Args:
        file : filename
    Returns:
        dictionary with all clinical testes splited by regex used to extract
        results from text
    """

    # reading the data from the file
    with open(file) as f:
        data = f.read()

    # reading the file and close it
    js = json.loads(data)

    return js

def get_files(path:str, type:str) -> list:
    """
    List all files from specific type from target path

    Args:
        path: target path
        type: extension type without dot (.), ex: pdf
    Returns:
        list with all files
    """

    files = [os.path.join(path, f) for f in os.listdir(path)]
    files = [f for f in files if f[-len(type):] == type]

    return files

def extract_text_from_pdf(file:str) -> str:
    """
    Extract all text from pdf files

    """

    data = extract_text(file)
    data = data.lower()
    data = re.sub(" +", " ", data)
    data = unidecode(data)

    return data

def get_id(text, template = 'hospital_sao_paulo'):
    """Extract personal information from patient as name, sex, birthday etc"""

    name = re.findall(r"nome: (\w.+) sexo:", text)[0].strip()
    sex = re.findall(r"sexo: (\w)", text)[0].strip()
    birthday = re.findall(r"data de nascimento: (\w.+)", text)[0].strip()
    collection = re.findall(r"data de coleta: (\w.+)", text)[0].strip().split(" as ")
    result = {"nome" : name
              ,"sexo" : sex
              ,"dt_nascimento" : birthday
              ,"dt_coleta" : collection[0]
              ,"hora_coleta" : collection[1]
              }
    return result

def remove_header(text, template = 'hospital_sao_paulo'):
    """

    """

    start_header_list = []
    end_header_list = [0]
    new_text = ""

    for m in re.finditer(r"hospital sao paulo$", text, flags=re.MULTILINE):
        start_header_list.append(m.start())
    start_header_list.append(len(text))

    for m in re.finditer(r"^resultados$", text, flags=re.MULTILINE):
        end_header_list.append(m.end())

    for i, s,e in [*zip(list(range(len(start_header_list))),start_header_list, end_header_list)]:
        new_text = new_text + text[e:s]

    return new_text

def remove_trailer(text, template = 'hospital_sao_paulo'):
    """

    """

    start_trailer_list = []
    end_trailer_list = [0]
    new_text = ""

    for m in re.finditer(r"^a correta interpretacao ", text, flags=re.MULTILINE):
        start_trailer_list.append(m.start())
    start_trailer_list.append(len(text))

    for m in re.finditer(r"vila clementino, sao paulo - sp$", text, flags=re.MULTILINE):
        end_trailer_list.append(m.end())

    for i, s,e in [*zip(list(range(len(start_trailer_list))),start_trailer_list, end_trailer_list)]:
        new_text = new_text + text[e:s]

    return new_text

def remove_empty_lines(text):
    """

    """

    text = "\n".join([ll.rstrip() for ll in text.splitlines() if ll.strip()])

    return text

def remove_date_lines(text):
    """

    """

    text = "\n".join([line for line in text.splitlines() if line.startswith("data de coleta") == False])

    return text

def parse_results(text, clinical_tests):
    """

    """

    results = {}

    for test in clinical_tests['type_1']['tests']:
        for test_str in re.finditer(r"^" + test + "$", text, flags=re.MULTILINE):
            start_position = test_str.end()
            regex =  clinical_tests['type_1']['regex']
            try:
                resultado = re.findall(regex, (text[start_position:]))[0]
                resultado = float(re.sub(",", ".", resultado))
            except:
                resultado = None
            results[test] = resultado
            break #this break is for run the inner for loop just once

    for test in clinical_tests['type_2']['tests']:
        for test_str in re.finditer(r"^" + test, text, flags=re.MULTILINE):
            start_position = test_str.end()
            regex =  clinical_tests['type_2']['regex']
            try:
                resultado = re.findall(regex, (text[start_position:]))[0]
                resultado = float(re.sub(",", ".", resultado))
            except:
                resultado = None
            # print(f"teste:{test}, pos:{start_position}")
            # print(results)
            results[test] = resultado
            break #this break is for run the inner for loop just once

    for test in clinical_tests['type_3']['tests']:
        for test_str in re.finditer(r"^" + test + "$", text, flags=re.MULTILINE):
            start_position = test_str.end()
            regex =  clinical_tests['type_3']['regex']#r"resultado .+\n.*?(-?\d+\,?\d+)"
            try:
                resultado = re.findall(regex, (text[start_position:]))[0]
                resultado = float(re.sub(",", ".", resultado))
            except:
                resultado = None
            # print(f"teste:{test}, pos:{start_position}")
            results[test] = resultado
            break #this break is for run the inner for loop just once

    return results

def build_table_with_results(files: list, clinical_tests: dict) -> pd.DataFrame:

    df_main = pd.DataFrame()

    for file in files:
        text = extract_text_from_pdf(file)
        id = get_id(text)
        clean_text = remove_date_lines(remove_empty_lines(remove_trailer(remove_header(text))))
        results = parse_results(clean_text, clinical_tests)
        id.update({"arquivo" : file.split('/')[-1]})
        id.update(results)
        df_id = pd.DataFrame.from_dict(data = id, orient='index').T
        df_main = pd.concat([df_main, df_id], axis = 0)

    return df_main

def handle_dates(df, birthday_col, collection_col, format):

    for col in [birthday_col, collection_col]:
        df[col] = [dt.datetime.strptime(str(x), format).date() for x in df[col].values]

    df['idade'] = [np.around((y-x)/dt.timedelta(365),1) for (x,y) in df[[birthday_col, collection_col]].values]

    return df

def combine_records(df, coalesce_cols=['nome', 'sexo', 'dt_nascimento', 'dt_coleta'], combine_col='arquivo'):
    """

    """

    df = (df.sort_values(coalesce_cols)
            .reset_index()
            .drop(['index'], axis = 1)
    )

    df_colaesce = df.groupby(coalesce_cols, sort=True).first()
    df_join = (df.groupby(coalesce_cols, sort=True)
                        .agg({combine_col: '||'.join})
    )

    df = (df_colaesce.drop([combine_col], axis = 1)
            .join(df_join).reset_index()
    )

    return df

def clean_col_names(df: pd.DataFrame) -> pd.DataFrame:
    """

    """

    cols = df.columns.values
    df.columns = [re.sub(" ","_", re.sub("\s?-|:|\s\s+", "", str(x))) for x in cols]

    return df

def rebuild_table_by_time(df: pd.DataFrame, group_cols: list, time_col: str) -> pd.DataFrame:
    """
    """

    df['time'] = (df.groupby(group_cols, sort=time_col).cumcount())

    df = df.pivot(index = group_cols, columns='time')

    col_names = [str(x)+"_t"+str(y) for (x,y) in df.columns]

    df.columns = col_names

    return df


import json
from .commons import get_model, transform_image_yolo

idshoes_class = json.load(open('yolov5/idshoes_class.json'))

def get_prediction(image_bytes, model_ins, conf=None, iou=None):
    #try:
    model = model_ins
    image = transform_image_yolo(image_bytes=image_bytes)
    if conf:
        model.conf = float(conf)
    if iou:
        model.iou = float(iou)
    outputs = model(image)
    #except Exception:
    #    return 0, 'error'
    predicted_idx = None
    probability = None

    if outputs.xyxy[0].shape[0]!=0:
        max_conf_idx = int(outputs.xyxy[0][outputs.xyxy[0].max(0)[1][4].item()][5].item())
        predicted_idx = str(max_conf_idx)
        probability = outputs.xyxy[0].max(0)[0][4].item()
    else:
        probability = 0
        predicted_idx = str(25)

    return idshoes_class[predicted_idx], probability

def load_model(model_name="yolov5s"):
    return get_model(model_name)


import math
import warnings
import torch.distributed as dist
from torch.utils.data.distributed import DistributedSampler
from torch.utils.data.sampler import RandomSampler, Sampler
import numpy as np
import torch
from typing import Dict, Iterator, List, Optional, Union

class SequentialDistributedSampler(Sampler):
    """
    Distributed Sampler that subsamples indices sequentially, making it easier to collate all results at the end.
    Even though we only use this sampler for eval and predict (no training), which means that the model params won't
    have to be synced (i.e. will not hang for synchronization even if varied number of forward passes), we still add
    extra samples to the sampler to make it evenly divisible (like in `DistributedSampler`) to make it easy to `gather`
    or `reduce` resulting tensors at the end of the loop.
    """

    def __init__(self, dataset, num_replicas=None, rank=None, batch_size=None):
        warnings.warn(
            "SequentialDistributedSampler is deprecated and will be removed in v5 of Transformers.",
            FutureWarning,
        )
        if num_replicas is None:
            if not dist.is_available():
                raise RuntimeError("Requires distributed package to be available")
            num_replicas = dist.get_world_size()
        if rank is None:
            if not dist.is_available():
                raise RuntimeError("Requires distributed package to be available")
            rank = dist.get_rank()
        self.dataset = dataset
        self.num_replicas = num_replicas
        self.rank = rank
        num_samples = len(self.dataset)
        # Add extra samples to make num_samples a multiple of batch_size if passed
        if batch_size is not None:
            self.num_samples = int(math.ceil(num_samples / (batch_size * num_replicas))) * batch_size
        else:
            self.num_samples = int(math.ceil(num_samples / num_replicas))
        self.total_size = self.num_samples * self.num_replicas
        self.batch_size = batch_size

    def __iter__(self):
        indices = list(range(len(self.dataset)))

        # add extra samples to make it evenly divisible
        indices += indices[: (self.total_size - len(indices))]
        assert (
            len(indices) == self.total_size
        ), f"Indices length {len(indices)} and total size {self.total_size} mismatched"

        # subsample
        indices = indices[self.rank * self.num_samples : (self.rank + 1) * self.num_samples]
        assert (
            len(indices) == self.num_samples
        ), f"Indices length {len(indices)} and sample number {self.num_samples} mismatched"

        return iter(indices)

    def __len__(self):
        return self.num_samples

def distributed_concat(tensor: "torch.Tensor", num_total_examples: Optional[int] = None) -> torch.Tensor:
    try:
        if isinstance(tensor, (tuple, list)):
            return type(tensor)(distributed_concat(t, num_total_examples) for t in tensor)
        output_tensors = [tensor.clone() for _ in range(dist.get_world_size())]
        dist.all_gather(output_tensors, tensor)
        output_tensors = [t if len(t.shape) > 0 else t[None] for t in output_tensors]
        concat = torch.cat(output_tensors, dim=0)

        # truncate the dummy elements added by SequentialDistributedSampler
        if num_total_examples is not None:
            concat = concat[:num_total_examples]
        return concat
    except AssertionError:
        raise AssertionError("Not currently using distributed training")


import os
from datetime import datetime

import nylas
from dotenv import load_dotenv
from nylas.models.calendars import ListCalendersQueryParams
from nylas.models.events import ListEventQueryParams, CreateEventRequest, CreateEventQueryParams
from nylas.models.errors import NylasApiError

# Load environment variables from a .env file
load_dotenv()

def display_calendar_ids(client: nylas.Client) -> None:
    """
    Retrieve and display the Calendar IDs and names of connected calendars.

    This function authenticates with the Nylas account using the provided credentials and
    fetches the list of connected calendars. It then displays the Calendar IDs, names, and
    descriptions of the calendars.

    Note:
        Ensure that you have set up your Nylas application with the required credentials
        ('API_KEY' and 'GRANT_ID').

    Args:
        client (nylas.Client): An instance of the Nylas Client.

    Raises:
        nylas.models.errors.NylasApiError: If there is an error during the calendar retrieval process.
    """
    try:
        # Get a list of connected calendars
        query_params = ListCalendersQueryParams(limit=10)

        calendars, _, _ = client.calendars.list(
            identifier=os.environ.get("GRANT_ID"), query_params=query_params
        )

        if calendars:
            print("Connected Calendars:\n")
            for calendar in calendars:
                print(
                    f"Calendar ID: {calendar.id}, Name: {calendar.name}, Description: {calendar.description}"
                )
        else:
            print("No connected calendars found.")

    except NylasApiError as e:
        print(f"Nylas Error: {e}")

def read_calendar_events(
    client: nylas.Client, calendar_id: str, start_time: int, end_time: int
) -> list:
    """
    Read calendar events within a specified time range.

    Args:
        client (nylas.Client): An instance of the Nylas Client.
        calendar_id (str): The ID of the calendar to read events from.
        start_time (int): The start time of the range in Unix timestamp format.
        end_time (int): The end time of the range in Unix timestamp format.

    Returns:
        list: A list of calendar events within the specified time range.

    Raises:
        nylas.models.errors.NylasApiError: If there is an error while fetching calendar events.
    """
    try:
        # Get events from the specified calendar within the time range
        query_params = ListEventQueryParams(
            limit=20, calendar_id=calendar_id, start=start_time, end=end_time
        )

        events, _, _ = client.events.list(
            identifier=os.environ.get("GRANT_ID"), query_params=query_params
        )

        return list(events)
    except NylasApiError as e:
        print(f"Error while fetching calendar events: {e}")

def create_calendar_event(
    client,
    title,
    location,
    start_time,
    end_time,
    description=None,
    participants=None,
    calendar_id=None,
):
    """
    Create a new calendar event and save it inside a specific calendar.

    Args:
        client (nylas.APIClient): The initialized Nylas API client.
        title (str): The title or name of the event.
        location (str): The location where the event will take place.
        start_time (int): The start time of the range in Unix timestamp format.
        end_time (int): The end time of the range in Unix timestamp format.
        description (str, optional): A description or additional information about the event.
        participants (list, optional): List of participant dictionaries with an "email" field.
        calendar_id (str, optional): The ID of the calendar where the event should be saved.

    Returns:
        nylas.events.Event: The created event object.

    Raises:
        nylas.client.errors.NylasError: If there's an issue with the Nylas API.

    Examples:
        Create a simple event with participants and save it in a specific calendar:
        >>> participants = [{'email': 'oss1@wiseai.dev'}, {'email': 'oss2@wiseai.dev'}]
        >>> event = create_calendar_event(client, 'Meeting', 'Conference Room', 1729457999, 1729458000, participants, 'your_calendar_id' )

        Create an event with a description and participants and save it in a specific calendar:
        Create an event with a description and participants and save it in a specific calendar:
        >>> participants = [{'email': 'oss1@wiseai.dev'}, {'email': 'oss2@wiseai.dev'}]
        >>> event = create_calendar_event(client, 'Team Meeting', 'Virtual', 1729457999, 1729458000, 'Discuss project progress', participants, 'your_calendar_id')
    """
    try:
        # create an event for that calendar ID
        # request_body = CreateEventRequest(title=title, description=description, location=location, when=when, calendar_id=calendar_id)
        query_params = CreateEventQueryParams(
            calendar_id=calendar_id, notify_participants=True
        )

        event, _ = client.events.create(
            identifier=os.environ.get("GRANT_ID"),
            query_params=query_params,
            request_body={
                "title": title,
                "description": description,
                "location": location,
                "when": {"start_time": start_time, "end_time": end_time},
                "participants": participants,
            },
        )
        return event
    except NylasApiError as e:
        print(f"Error creating event: {e}")

def main() -> None:
    try:
        # Initialize the Nylas client with API_KEY environment variable
        client = nylas.Client(
            api_key=os.environ.get("API_KEY"),
        )
        # display all calendar ids
        display_calendar_ids(client)

        # Replace with your actual calendar ID, start, and end time
        calendar_id = "tfnid9uts44nb1bqku4bednt14@group.calendar.google.com"
        start_time = "2023-10-01T00:00:00Z"
        start_datetime_obj = datetime.strptime(start_time, "%Y-%m-%dT%H:%M:%SZ")
        start_unix_time = int(start_datetime_obj.timestamp())

        end_time = "2024-10-20T23:59:59Z"
        end_datetime_obj = datetime.strptime(end_time, "%Y-%m-%dT%H:%M:%SZ")
        end_unix_time = int(end_datetime_obj.timestamp())
        events = read_calendar_events(
            client, calendar_id, start_unix_time, end_unix_time
        )
        print("\nGetting events before event creation for Calendar 'test-update':\n")
        if events:
            for event in events:
                human_readable_start_time = datetime.fromtimestamp(event.when.start_time)
                formatted_start_datetime = human_readable_start_time.strftime('%Y-%m-%d %H:%M:%S')
                human_readable_end_time = datetime.fromtimestamp(event.when.end_time)
                formatted_end_datetime = human_readable_end_time.strftime('%Y-%m-%d %H:%M:%S')
                print(f"Event title: {event.title}, start time: {formatted_start_datetime}, end time: {formatted_end_datetime}")
        else:
            print("No events found in the specified calendar.")
        participants = [{"email": "oss1@wiseai.dev"}, {"email": "oss2@wiseai.dev"}]
        event = create_calendar_event(
            client,
            "Team Meeting",
            "Virtual",
            start_unix_time,
            end_unix_time,
            "Discuss project progress",
            participants,
            calendar_id,
        )
        print(f"\nEvent created: {event}\n")
        events = read_calendar_events(
            client, calendar_id, start_unix_time, end_unix_time
        )
        print("\nGetting events after event creation for Calendar 'test-update':\n")
        if events:
            for event in events:
                human_readable_start_time = datetime.fromtimestamp(event.when.start_time)
                formatted_start_datetime = human_readable_start_time.strftime('%Y-%m-%d %H:%M:%S')
                human_readable_end_time = datetime.fromtimestamp(event.when.end_time)
                formatted_end_datetime = human_readable_end_time.strftime('%Y-%m-%d %H:%M:%S')
                print(f"Event title: {event.title}, start time: {formatted_start_datetime}, end time: {formatted_end_datetime}")
        else:
            print("No events found in the specified calendar.")
    except Exception as e:
        print(f"An error occurred: {str(e)}")

if __name__ == "__main__":
    main()


import streamlit as st
import pickle
import requests
import pandas as pd

st.set_page_config(page_title=None, page_icon=None, layout="wide", initial_sidebar_state="auto", menu_items=None)

movies = pickle.load(open("model/movies_list.pkl",'rb'))
similarity = pickle.load(open("model/similarity.pkl",'rb'))

def fetch_poster(movie_id):
    url = "https://api.themoviedb.org/3/movie/{}?api_key=8735d1698444eb906584226ecb77307b&language=en-US".format(movie_id)
    data = requests.get(url)
    data = data.json()
    poster_path = data['poster_path']
    full_path = "https://image.tmdb.org/t/p/w500/" + poster_path
    return full_path

def recommend(movie):
    index = movies[movies['title'] == movie].index[0]
    distances = sorted(list(enumerate(similarity[index])), reverse=True, key=lambda x: x[1])
    recommended_movie_names = []
    recommended_movie_posters = []
    for i in distances[1:6]:
        # fetch the movie poster
        movie_id = movies.iloc[i[0]].movie_id
        recommended_movie_posters.append(fetch_poster(movie_id))
        recommended_movie_names.append(movies.iloc[i[0]].title)

    return recommended_movie_names,recommended_movie_posters

st.header('Movie Recommendation System (Content Based)')

st.text("A movie recommendation system, or a movie recommender system, is an ML-based approach to filtering or predicting the users' film preferences ")

movie_list = movies['title'].values
selected_movie = st.selectbox(
    "Type or select a movie from the dropdown",
    movie_list
)

if st.button('Show Recommendation'):
    recommended_movie_names,recommended_movie_posters = recommend(selected_movie)
    col1, col2, col3, col4, col5 = st.columns(5)
    with col1:
        st.text(recommended_movie_names[0])
        st.image(recommended_movie_posters[0])
    with col2:
        st.text(recommended_movie_names[1])
        st.image(recommended_movie_posters[1])

    with col3:
        st.text(recommended_movie_names[2])
        st.image(recommended_movie_posters[2])
    with col4:
        st.text(recommended_movie_names[3])
        st.image(recommended_movie_posters[3])
    with col5:
        st.text(recommended_movie_names[4])
        st.image(recommended_movie_posters[4])


import requests
from bs4 import BeautifulSoup
import pandas as pd
from pathlib import Path
from PIL import Image
from io import BytesIO
import json
OUTPUT_DIR="h:\\temp\\GB Imgs"
import re
import asyncio

def get_images(soup, out_dir,model_id):
    output_path = out_dir / "Images"
    output_path.mkdir(parents=True, exist_ok=True)
    #images=soup.find_all("div")#, class_= "slick-slide slick-current slick-active")
    imgprv=soup.find_all("div", {'data-image-url': True} )

    for image in imgprv:
        get_image(image, model_id, output_path)

def get_image(image, model_id, output_path):
    kkk = 0
    try:
        image_link = image['data-image-url'].replace("_xs", "")
        filename = image_link.split("/")[-1]
        spl = filename.split(".")
        # if spl[1]=='png' and "x" in spl[0]:
        if True:
            imfile = requests.get(image_link, stream=True)
            if imfile.ok:
                with open(output_path / f"{model_id} - {spl[0]}.{spl[1]}", "wb") as f:
                    f.write(imfile.content)
    except Exception:
        pass

def write_url(out_dir,title,URL):
    with open(out_dir/f"{title}.url","wt") as f:
        f.write("[InternetShortcut]\n")
        f.write(f"URL={URL}")

def write_info(out_dir, soup,row):
    info_file=out_dir / "INFO.txt"
    info_json=out_dir / "INFO.json"
    with open(info_file,"wt",encoding="utf-8") as f:

        author=soup.find_all("div", class_="dnamePrt")
        author2=author[0].find('a').get_text()
        f.write(f"Author: \n{author2.strip()}\n\n")
        f.write(f"Url: \n{row.link}\n\n")

        description = soup.find("div", {"id": "description-block"}).get_text()
        f.write("Description:")
        f.write(description)
        f.write("\nTags:\n")
        tags=soup.find("p",class_="cloudTags").get_text().split(',')
        for tag in tags:
            f.write(tag.strip()+"\n")
        f.write("\n")
    output_json={
        "name": re.sub('[^\w\-_\. ]', '_', row.title.split(" 3D")[0]),
        "author": author2,
        "distributor": "Gambody",
        "url": row.link,
        "description": description.strip(),
        "tags": [x.strip() for x in tags]
    }
    with open(info_json,"wt", encoding='utf-8') as f:
        json.dump(output_json, f, indent=4,ensure_ascii=False)

def process_row(row):
    model_id = f"{row.id:04d}"
    model_name = re.sub('[^\w\-_\. ]', '_', row.title.split(" 3D")[0])
    out_dir = path / f"{model_id} - {model_name.strip()}"
    infoname = out_dir / "INFO.txt"
    if infoname.is_file():
        print(f"{out_dir} existe")
    else:
        model_url = row.link

        page = requests.get(model_url)
        soup = BeautifulSoup(page.content, "html.parser")

        print(f"Processing {out_dir}")
        get_images(soup, out_dir, model_id)
        write_url(out_dir, model_name, model_url)
        write_info(out_dir, soup, row)

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    path = Path(OUTPUT_DIR)
    list_file=pd.read_csv("f:\\STL\\08 - Gambody\\products.txt",sep='\t')

    a=[process_row(list_file.iloc[row]) for row in range(len(list_file.index))]

    # for i,row in list_file.iterrows():
    #     process_row(row)
    #     # asyncio.run(process_row(row))
    #     kk=0


from collections import deque
import sys

n = int(sys.stdin.readline())
graph = [[] for _ in range(n + 1)]
visited = [False] * (n + 1)
parent = [[] for _ in range(n + 1)]

def bfs(x):
    queue = deque()
    queue.append(x)
    visited[x] = True
    while queue:
        a = queue.popleft()
        for i in graph[a]:
            if not visited[i]:
                visited[i] = True
                parent[i] = a
                queue.append(i)

for i in range(n-1):
    a, b = map(int, sys.stdin.readline().split())
    graph[a].append(b)
    graph[b].append(a)

for i in range(1, n+1):
    if not visited[i]:
        bfs(i)

for i in range(2, n+1):
    print(parent[i])


import argparse

import pytorch_lightning as pl
from omegaconf import OmegaConf

from Modules.data_lightning import DataModuleForSentenceClassification
# from Modules.MSP_lightning import ModelModule
from Modules.DMSP_lightning import ModelModule

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--max_len', type=int, default=64)
    parser.add_argument('--batch_size', type=int, default=64)
    parser.add_argument('--max_epoch', type=int, default=100)
    parser.add_argument('--ckpt', type=str, default=None)
    parser.add_argument('--is_train', type=str, default='Y')
    args = parser.parse_args()
    pl.seed_everything(1)
    """load config.yml"""
    config = OmegaConf.load("config.yml")
    dataset_conf, model_conf = config.dataset, config.model
    dm = DataModuleForSentenceClassification(file_path_dir=[dataset_conf.filename, dataset_conf.subset], max_len=args.max_len,
                                             batch_size=args.batch_size, label_map=dataset_conf.label_map)
    model = ModelModule(**model_conf, epsilon=0.5, patience=2)
    trainer = pl.Trainer(
        callbacks=[
            pl.callbacks.ModelCheckpoint(
                monitor="validation_step_accuracy",
                save_top_k=2,
                filename='{epoch}-{validation_step_loss:.4f}-{validation_step_accuracy:.4f}',
                mode='max'
            )
        ],
        accelerator="auto",
        devices='auto',
        max_epochs=args.max_epoch,
        check_val_every_n_epoch=1,
        log_every_n_steps=1
    )

    if args.is_train == "Y":
        dm.setup('fit')
        train_loader = dm.train_dataloader()
        val_loader = dm.val_dataloader()
        if args.ckpt:
            trainer.fit(model, train_loader, val_loader, ckpt_path=args.ckpt)
        else:
            trainer.fit(model, train_loader, val_loader)
    else:
        dm.setup('test')
        test_loader = dm.test_dataloader()
        trainer.test(model, test_loader, args.ckpt)


from django.urls import path

# importing views
from .views import (
    AllPizzaAPI,
    PizzaDetailAPI,
)

urlpatterns = [
    path('pizza', AllPizzaAPI.as_view(), name='list-pizza'),
    path('pizza/<pk>', PizzaDetailAPI.as_view(), name='pizza-detail'),
]


from collections import deque
import sys
sys.setrecursionlimit(10 ** 6)
input = sys.stdin.readline

n, m = map(int, input().split())
a = [input().rstrip() for _ in range(n)]
dr = (0, 1, 0, -1)
dc = (1, 0, -1, 0)
visited = [[0] * m for _ in range(n)]

def bfs():
    visited[0][0] = 1
    q = deque([(0, 0)])
    while q:
        r, c = q.popleft()
        if r == n - 1 and c == m - 1:
            return visited[n - 1][m - 1]
        for i in range(4):
            nr = r + dr[i]
            nc = c + dc[i]
            if 0 <= nr < n and 0 <= nc < m:
                if visited[nr][nc]: continue
                if a[nr][nc] == '0': continue
                visited[nr][nc] = visited[r][c] + 1
                q.append((nr, nc))
print(bfs())


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches

class DecisionBoundary2D():

	def __init__(self, a,b):
		self.a = a
		self.b = b

	def get_orthogonal_distance(self, x, y):
		# line orthogonal to boundary through x,y
		a2 = -1/self.a
		b2 = y - x*a2
		x2 = (b2 - self.b)/(self.a - a2)
		y2 = self.a * x2 + self.b
		#get euclidean distance
		vec, dist = np.array([x2-x,y2-y]), np.sqrt((x2 - x)**2 + (y-y2)**2)
		if y > self.a * x + self.b:
			vec = -1 * vec
			dist = -1 * dist
		return vec, dist

	def f(self, x):
		return self.a * x + self.b

def sigmoid(x):
	return 1 / (1 + np.clip(np.exp(-x), -10000, 10000))

def sigmoid_derivative(x):
	return sigmoid(x) * (1 - sigmoid(x))

def bce_derivative(x):
	return sigmoid(x)

def step(x,y, decision_boundaries, loss_function_derivative):
	gradient = np.zeros(2)

	for idx, boundary in enumerate(decision_boundaries):
		grad, distance = boundary.get_orthogonal_distance(x,y)
		grad = grad if np.sum(grad ** 2) == 0 else grad / np.sqrt(np.sum(grad ** 2))
		gradient += grad * loss_function_derivative(distance)
	gradient = gradient if np.sum(gradient ** 2) == 0 else gradient / np.sqrt(np.sum(gradient ** 2))

	return np.sign(gradient)

boundary1 = DecisionBoundary2D(1, 2)
boundary2 = DecisionBoundary2D(1, 5.0)
boundary3 = DecisionBoundary2D(-1, 13.0)
boundary4 = DecisionBoundary2D(-1, 16.0)
boundary5 = DecisionBoundary2D(1, 8.0)

xspace = np.linspace(-20, 40)
plt.plot(xspace, boundary1.f(xspace), color='black')
plt.plot(xspace, boundary2.f(xspace), color='black')
plt.plot(xspace, boundary3.f(xspace), color='black')
plt.plot(xspace, boundary4.f(xspace), color='black')
plt.plot(xspace, boundary5.f(xspace), color='black')

a = []
b = []
(x,y) = (8.0,0.0)
for i in range(14):
	gradient = step(x,y, [boundary1, boundary2, boundary3, boundary4, boundary5], bce_derivative)
	a.append(x)
	b.append(y)
	x += gradient[0]
	y += gradient[1]

plt.plot(a,b, color='blue', label='patient')

a = []
b = []
(x,y) = (8.0,0.0)
for i in range(14):
	gradient = step(x,y, [boundary1, boundary2, boundary3, boundary4, boundary5], sigmoid_derivative)
	a.append(x)
	b.append(y)
	x += gradient[0]
	y += gradient[1]

plt.plot(a,b, color='green', label='greedy')

# PLOT L2 BOUNDS
# r1 = 6
# r2 = 13
# def circle(r, x):
# 	return np.sqrt(r**2 - (x-8)**2)

# circle1x = np.linspace(8 - r1, 8 + r1, 100)
# circle2x = np.linspace(8 - r2, 8 + r2, 100)

# plt.plot(circle1x, circle(r1, circle1x), color='red')
# plt.plot(circle2x, circle(r2, circle2x), color='red')

# PLOT L-infinity BOUNDS
width = 9
plt.gca().add_patch(patches.Rectangle((8 - 0.5*width , 0 - 0.5*width), width, width, linewidth=1, linestyle='--', edgecolor='black',facecolor='none'))
width = 26
plt.gca().add_patch(patches.Rectangle((8 - 0.5*width , 0 - 0.5*width), width, width, linewidth=1, linestyle='--', edgecolor='black',facecolor='none'))

plt.axis([-15, 25, -5, 20])
plt.gca().set_aspect('equal')
plt.legend()

plt.show()


from hypothesis import strategies as st

from ssz.hashable_container import HashableContainer
from ssz.hashable_list import HashableList
from ssz.hashable_vector import HashableVector
from ssz.sedes import (
    BasicSedes,
    Bitlist,
    Bitvector,
    ByteVector,
    Container,
    List,
    UInt,
    Vector,
    boolean,
)
from ssz.sedes.serializable import Serializable

#
# Basic value strategies
#
@st.composite
def uint_value_st(draw, bits=None):
    if bits is None:
        bytes_ = 2 ** draw(st.integers(0, 5))
        bits = bytes_ * 8
    return draw(st.integers(0, bits))

def bool_value_st():
    return st.booleans()

def bitlist_value_st(max_size=None):
    return st.builds(tuple, st.lists(st.booleans(), max_size=max_size))

def bitvector_value_st(size=None):
    min_size = size or 1
    max_size = size
    return st.builds(
        tuple, st.lists(st.booleans(), min_size=min_size, max_size=max_size)
    )

def bytevector_value_st(size=None):
    min_size = size or 1
    max_size = size
    return st.binary(min_size=min_size, max_size=max_size)

#
# Composite value strategies
#
@st.composite
def vector_value_st(draw, elements, *, size=None):
    if size is None:
        size = draw(st.integers(0, 10))
    return st.builds(tuple, st.lists(elements, min_size=size, max_size=size))

@st.composite
def list_value_st(draw, elements, *, size=None):
    if size is None:
        size = draw(st.integers(0, 10))
    return st.lists(elements, min_size=size, max_size=size)

def container_value_st(element_sequence):
    return st.tuples(*element_sequence)

#
# Strategies for basic sedes with corresponding value strategy
#
def boolean_sedes_and_values_st():
    return st.just((boolean, bool_value_st()))

@st.composite
def uint_sedes_and_values_st(draw):
    bytes_ = 2 ** draw(st.integers(0, 5))
    bits = bytes_ * 8
    return UInt(bits), uint_value_st(bits)

@st.composite
def bitlist_sedes_and_values_st(draw):
    max_size = draw(
        st.one_of(
            st.integers(1, 10),
            st.just(300),  # choose at least one sample exceeding one chunk
        )
    )
    return Bitlist(max_size), bitlist_value_st(max_size)

@st.composite
def bitvector_sedes_and_values_st(draw):
    size = draw(
        st.one_of(
            st.integers(1, 10),
            st.just(300),  # choose at least one sample exceeding one chunk
        )
    )
    return Bitvector(size), bitvector_value_st(size)

@st.composite
def bytevector_sedes_and_values_st(draw):
    size = draw(st.integers(1, 65))
    return ByteVector(size), bytevector_value_st(size)

#
# Strategies for composite sedes objects with corresponding value strategy
#
@st.composite
def general_vector_sedes_and_values_st(draw, element_sedes_and_elements, size=None):
    element_sedes, elements = element_sedes_and_elements

    if size is None:
        size = draw(st.integers(min_value=1, max_value=10))

    sedes = Vector(length=size, element_sedes=element_sedes)
    values = st.builds(tuple, st.lists(elements, min_size=size, max_size=size))
    return sedes, values

@st.composite
def general_list_sedes_and_values_st(draw, element_sedes_and_elements, size=None):
    element_sedes, elements = element_sedes_and_elements

    if size is None:
        size = draw(st.integers(min_value=1, max_value=10))
    max_size = draw(st.integers(min_value=size, max_value=1024))

    sedes = List(element_sedes, max_length=max_size)
    values = st.lists(elements, min_size=size, max_size=size)
    return sedes, values

@st.composite
def general_container_sedes_and_values_st(draw, element_sedes_and_elements_sequence):
    element_sedes, elements = zip(*element_sedes_and_elements_sequence)
    sedes = Container(element_sedes)
    values = st.tuples(*elements)
    return sedes, values

#
# Strategies for depth-1 composite sedes objects with corresponding value strategy
#
@st.composite
def basic_vector_sedes_and_values_st(draw, size=None):
    sedes, values = draw(basic_sedes_and_values_st())
    return draw(general_vector_sedes_and_values_st((sedes, values), size=size))

@st.composite
def basic_list_sedes_and_values_st(draw, size=None):
    sedes, values = draw(basic_sedes_and_values_st())
    return draw(general_list_sedes_and_values_st((sedes, values), size=size))

@st.composite
def basic_container_sedes_and_values_st(draw, size=None):
    if size is None:
        size = draw(st.integers(min_value=1, max_value=4))

    element_sedes_and_elements_sequence = draw(
        st.lists(basic_sedes_and_values_st(), min_size=size, max_size=size)
    )
    return draw(
        general_container_sedes_and_values_st(element_sedes_and_elements_sequence)
    )

#
# Strategies for depth-2 composite sedes objects with corresponding value strategy
#
@st.composite
def nested_vector_sedes_and_values_st(draw, size=None):
    sedes, values = draw(basic_composite_sedes_and_values_st())
    return draw(general_vector_sedes_and_values_st((sedes, values), size=size))

@st.composite
def nested_list_sedes_and_values_st(draw, size=None):
    sedes, values = draw(basic_composite_sedes_and_values_st())
    return draw(general_list_sedes_and_values_st((sedes, values), size=size))

@st.composite
def nested_container_sedes_and_values_st(draw, size=None):
    if size is None:
        size = draw(st.integers(min_value=1, max_value=4))
    fields = st.one_of(
        basic_sedes_and_values_st(), basic_composite_sedes_and_values_st()
    )
    element_sedes_and_elements_sequence = draw(
        st.lists(fields, min_size=size, max_size=size)
    )
    return draw(
        general_container_sedes_and_values_st(element_sedes_and_elements_sequence)
    )

#
# Combinations
#
def basic_sedes_and_values_st():
    # bitlist, bitvector and bytevector are not technically basic, but we include them here as our
    # implementation does not allow them to have children
    return st.one_of(
        boolean_sedes_and_values_st(),
        uint_sedes_and_values_st(),
        bitlist_sedes_and_values_st(),
        bitvector_sedes_and_values_st(),
        bytevector_sedes_and_values_st(),
    )

def basic_composite_sedes_and_values_st():
    return st.one_of(
        basic_vector_sedes_and_values_st(),
        basic_list_sedes_and_values_st(),
        basic_container_sedes_and_values_st(),
    )

def nested_composite_sedes_and_values_st():
    return st.one_of(
        nested_vector_sedes_and_values_st(),
        nested_list_sedes_and_values_st(),
        nested_container_sedes_and_values_st(),
    )

def composite_sedes_and_values_st():
    return st.one_of(
        basic_composite_sedes_and_values_st(), nested_composite_sedes_and_values_st()
    )

def vector_sedes_and_values_st(size=None):
    return st.one_of(
        basic_vector_sedes_and_values_st(size=size),
        nested_vector_sedes_and_values_st(size=size),
    )

def list_sedes_and_values_st(size=None):
    return st.one_of(
        basic_list_sedes_and_values_st(size=size),
        nested_list_sedes_and_values_st(size=size),
    )

def container_sedes_and_values_st(size=None):
    return st.one_of(
        basic_container_sedes_and_values_st(size=size),
        nested_container_sedes_and_values_st(size=size),
    )

#
# Convert plain Python values generated by strategies to serializable or hashable datatypes
#
def to_serializable_value(value, sedes):
    if isinstance(sedes, (BasicSedes, Bitlist, Bitvector, ByteVector)):
        return value

    elif isinstance(sedes, List):
        elements = tuple(
            to_serializable_value(element, sedes.element_sedes) for element in value
        )
        return elements

    elif isinstance(sedes, Vector) and not isinstance(sedes, ByteVector):
        elements = tuple(
            to_serializable_value(element, sedes.element_sedes) for element in value
        )
        return elements

    elif isinstance(sedes, Container):
        field_names = [f"field{index}" for index in range(len(value))]

        class ValueClass(Serializable):
            fields = tuple(
                (field_name, field_sedes)
                for field_name, field_sedes in zip(field_names, sedes.field_sedes)
            )

        kwargs = {
            field_name: to_hashable_value(field_value, field_sedes)
            for field_name, field_value, field_sedes in zip(
                field_names, value, sedes.field_sedes
            )
        }

        return ValueClass(**kwargs)

    else:
        assert False

def to_hashable_value(value, sedes):
    if isinstance(sedes, (BasicSedes, Bitlist, Bitvector, ByteVector)):
        return value

    elif isinstance(sedes, List):
        elements = (
            to_hashable_value(element, sedes.element_sedes) for element in value
        )
        return HashableList.from_iterable(elements, sedes)

    elif isinstance(sedes, Vector) and not isinstance(sedes, ByteVector):
        elements = (
            to_hashable_value(element, sedes.element_sedes) for element in value
        )
        return HashableVector.from_iterable(value, sedes)

    elif isinstance(sedes, Container):
        field_names = [f"field{index}" for index in range(len(value))]

        class ValueClass(HashableContainer):
            fields = tuple(
                (field_name, field_sedes)
                for field_name, field_sedes in zip(field_names, sedes.field_sedes)
            )

        kwargs = {
            field_name: to_hashable_value(field_value, field_sedes)
            for field_name, field_value, field_sedes in zip(
                field_names, value, sedes.field_sedes
            )
        }

        return ValueClass.create(**kwargs)

    else:
        assert False

def to_plain_value(serializable_or_hashable_value, sedes):
    if isinstance(sedes, (BasicSedes, Bitlist, Bitvector, ByteVector)):
        return serializable_or_hashable_value

    elif isinstance(sedes, List):
        return [
            to_plain_value(element, sedes.element_sedes)
            for element in serializable_or_hashable_value
        ]

    elif isinstance(sedes, (Vector, ByteVector)):
        return tuple(
            to_plain_value(element, sedes.element_sedes)
            for element in serializable_or_hashable_value
        )

    elif isinstance(sedes, Container):
        return tuple(
            to_plain_value(element, field_sedes)
            for element, field_sedes in zip(
                serializable_or_hashable_value, sedes.field_sedes
            )
        )

    else:
        assert False

@st.composite
def transform_path_st(draw, hashable_value):
    if isinstance(hashable_value, HashableContainer):
        fields = hashable_value._meta.fields
        field_index = draw(st.integers(min_value=0, max_value=len(fields) - 1))
        field_name = fields[field_index][0]
        return (field_name,) + draw(transform_path_st(hashable_value[field_name]))
    elif isinstance(hashable_value, (HashableList, HashableVector)):
        if len(hashable_value) == 0:
            return ()
        else:
            index = draw(st.integers(min_value=0, max_value=len(hashable_value) - 1))
            return (index,) + draw(transform_path_st(hashable_value[index]))
    elif isinstance(hashable_value, (bool, int, tuple, bytes)):
        return ()
    else:
        assert False


import sys
from collections import deque

n = int(sys.stdin.readline().rstrip())

stack = deque()

def push(x):
    stack.append(x)

def size():
    return len(stack)

def empty():
    if size()==0:
        return 1
    else:
        return 0

def top():
    if empty()==1:
        return -1
    else:
        return stack[-1]

def pop():
    if empty()==1:
        return -1
    else:
        return stack.pop()

for _ in range(n):
    line = sys.stdin.readline().rstrip()
    if line.split()[0] == "push":
        push(line.split()[1])

    elif line == "size":
        print(size())

    elif line == "empty":
        print(empty())

    elif line == "top":
        print(top())

    elif line == "pop":
        print(pop())

    else:
        exit()