 # Helper functions for running CCLF QC pipeline
import pandas as pd
import numpy as np
import dalmatian as dm
from gsheets import Sheets
import sys
pathtoJK_parent = "../"
sys.path.append(pathtoJK_parent)
from JKBio import terra

sheets = Sheets.from_files('~/.client_secret.json', '~/.storage.json')

def getCohortAbbreviations(df, cohorts2id_url="https://docs.google.com/spreadsheets/d/1R97pgzoX0YClGDr5nmQYQwimnKXxDBGnGzg7YPlhZJU"):
    # mapping abbreviations to full names/descriptions
    cohorts = sheets.get(cohorts2id_url).sheets[0].to_frame()
    # match collection data and error out
    cohortlist = []
    no_cohort_match = set()
    for k, val in df['Collection'].iteritems():
        res = cohorts[cohorts['Name'] == val]
        if len(res) == 0:
            print(res)
            no_cohort_match.add(str(val))
            cohortlist.append('nan')
        else:
            cohortlist.append(res['ID'].values[0])
    print("we do not have a corresponding cohort abbreviation for these cohorts: [{0}]".format(", ".join(str(i) for i in no_cohort_match)))

    df['cohorts'] = cohortlist
    return df

def regenerate_variables(wm, samplesetnames, cohorts2id_url):
    # Run function when need to re-generate key variables: sample_info, all_pairsets, the cohorts_per_batch dictionary, cohort_pairsets
    # Inputs:
    # - wm: Dalmation workspace manager for the data processing workspace

    # wm = dm.WorkspaceManager(workspace)
    # will use "cohort_pairsets" to create cohort-specific SNV tsv and CN heat map
    cohorts = sheets.get(cohorts2id_url).sheets[0].to_frame()
    sample_info = wm.get_samples() # TODO: check whether to use this or not
    cohorts_per_batch = {} # will be dict of cohorts in each batch
    all_changed_cohorts = set()
    for i in range(len(samplesetnames)):
        # get appropriate subset of the samples for each batch
        batch_sample_info = sample_info[sample_info['batch'] == samplesetnames[i][:-4]]
        cohorts_in_batch = set()
        cohorts_with_pairs = [] # check: currently not used.
        # for each batch, make pairsets by cohort
        for val in cohorts['ID'].values:
            cohortsamples = batch_sample_info[batch_sample_info["cohorts"] == val].index.tolist()
            tumorsamplesincohort = batch_sample_info[batch_sample_info["cohorts"] == val][batch_sample_info['sample_type']=="Tumor"].index.tolist()
            if len(cohortsamples)>0:
                cohorts_in_batch.update([val])
        batch_name = samplesetnames[i]
        cohorts_per_batch[batch_name] = cohorts_in_batch
        all_changed_cohorts.update(cohorts_in_batch) # add all the new cohorts in this batch to the full list

    # list of the cohort pairsets affected by the new samples we added
    all_pairsets = wm.get_pair_sets().index.tolist()
    cohort_pairsets = set(all_changed_cohorts) - (set(all_changed_cohorts) - set(all_pairsets))

    return sample_info, all_pairsets, cohorts_per_batch, cohort_pairsets, all_changed_cohorts

def check_cohort_abbrev_availability(df, cohorts2id_url):
    _, no_cohort_match = get_list_of_cohort_abbrevs_in_df(df, cohorts2id_url)
    return no_cohort_match

def get_list_of_cohort_abbrevs_in_df(df, cohorts2id_url):
    # the df must contain the columns: `Collection` (this is found in the External ID sheet)

    # cohorts and cohort abbreviations
    cohorts = sheets.get(cohorts2id_url).sheets[0].to_frame()

    # match collection data and error out
    cohortlist = []
    no_cohort_match = set()
    for k, val in df['Collection'].iteritems():
        res = cohorts[cohorts['Name'] == val]
        if len(res) == 0:
            no_cohort_match.add(str(val))
            cohortlist.append('nan')
        else:
            cohortlist.append(res['ID'].values[0])

    assert len(no_cohort_match) == 0,\
        "We do not have a corresponding cohort abbreviation for these cohorts: [{0}]".format(", ".join(str(i) for i in no_cohort_match)) + \
        "\nStop running the pipeline and have the CCLF team fix the missing cohort information."
    return cohortlist, no_cohort_match

def create_preliminary_sample_and_metadata_tables(wto, wfrom, samplesetnames, external_sheets_url_list, cohorts2id_url, forcekeep=[]):
    # Inputs:
    # - wto: Dalmation workspace manager for the data processing workspace
    # - wfrom: Dalmation workspace manager for the data delivery workspace

    # we look at all the samples we already have in the CCLF QC processing workspace
    refsamples = wto.get_samples()
    refids = refsamples.index

    # get the External sheet data from google sheets
    gsheets = [sheets.get(url).sheets[0].to_frame() for url in external_sheets_url_list]

    # add a column with batch information (e.g. CCLF_TWIST1 vs CCLF_TWIST2)
    metadata = pd.concat(gsheets,sort=False, keys = samplesetnames)
    metadata = metadata.reset_index().rename(columns = {'level_0':'batch', "External ID":'external_id_validation'}).drop(['level_1'], axis = 'columns')

    print('Number of rows from the external ID gsheets: {}'.format(len(metadata)))

    # we use this gsheet package to get all the sheets into one dataframe
    # cohorts
    cohorts = sheets.get(cohorts2id_url).sheets[0].to_frame()

    # we look at all the samples we already have in Terra
    # we do some corrections just in case
    samples1 = wfrom.get_samples().replace(np.nan, '', regex=True)

    # creating sample_id (like in processing wm) for metadata and samples1
    newmetadata = metadata.dropna(0, subset=['Collaborator Sample ID','Sample Type','Exported DNA SM-ID'])
    # print("Dropped indices: "+str(set(metadata.index.tolist())-set(newmetadata.index.tolist())))
    print("Dropped samples: "+str(set(metadata['Exported DNA SM-ID'].tolist())-set(newmetadata['Exported DNA SM-ID'].tolist())))
    print("Note that we expect to drop one per batch, the technical sequencing control.")

    print('New length after dropping any samples with no value for the Collaborator Sample ID, Sample Type, or Exported DNA SM-ID: '+str(len(newmetadata)))
    metadata=newmetadata

    ttype = [i for i in metadata["Sample Type"]]
    metadata['sample_id'] = [str(val['Collaborator Sample ID'][:-1]) + '-' + str(val['Sample Type']) + '-' + str(val['Exported DNA SM-ID']) for i, val in metadata.iterrows()]

    samples1.index = [i.split('_')[2] for i, val in samples1.iterrows()]

    samples1['sample_id'] = [str(val["individual_alias"]) + '-' + str(val['sample_type']) + '-' + i for i, val in samples1.iterrows()]

    print("Number of samples already in Terra: ", samples1.index.isin(refids).tolist().count(False))

    metadata.index = metadata['Exported DNA SM-ID']
    # print("Number of samples in the input External ID metadata: ",len(metadata.index.tolist()))

    # filtering on what already exists in the processing wm (refids)
    newsamples = samples1[(~samples1.index.isin(refids)) | samples1.index.isin(forcekeep)]
    post_filter_len = len(metadata.index.tolist())
    print('Number of samples remaining after filtering out samples already in the processing wm: '+ str(post_filter_len))

    # keep the new samples that we have metadata for
    tokeep = set(metadata.index) & set(newsamples.index)
    if len(tokeep) != post_filter_len:
        print("\nThe data delivery workspace (which is used to create `newsamples`) doesn't contain all the samples specified in the External Sheet(s) (which are used to create `metadata`). \nThe",len(set(metadata.index) - set(newsamples.index)),"samples we couldn't find are:", set(metadata.index) - set(newsamples.index), "\nThis can occur if a line is blacklisted due to too few reads, for example.")

    # useful to merge the two df, sm-id is one of the only unique id here
    num_newsamples = len(newsamples[~newsamples.index.isin(tokeep)])
    if num_newsamples > 0:
        print('We could not add these ' + str(num_newsamples) + ' samples from the data delivery wm as we don\'t have metadata for them: ' + '\n'
              + str(newsamples[~newsamples.index.isin(tokeep)].index))
    newsamples = newsamples[newsamples.index.isin(tokeep)]
    newmetadata = metadata[metadata.index.isin(tokeep)].sort_index().drop_duplicates("Exported DNA SM-ID")
    print("Final dimensions of the newsamples table: ", newsamples.shape)
    print("Final dimensions of the newmetadata table: ", newmetadata.shape)

    return newsamples, newmetadata

def check_required_metadata_columns(df, required_metadata_cols, cohorts2id_url, drop=False):
    # check collection (cohort) data and raise warning if missing
    check_cohort_abbrev_availability(df, cohorts2id_url)

    # If samples in the df are lacking any of the required metadata columns, flag these samples.
    samplesMissingMetadata = df.iloc[[j for j,i in enumerate(df[required_metadata_cols].isna().values.sum(1)) if i !=0]].index.tolist()
    nMissing = len(samplesMissingMetadata)

    # Return if no samples are missing metadata
    if nMissing == 0:
        print("No samples are missing any required metadata. Continue running the pipeline.")
        return df

    print("Since they do not have full data, we will be dropping " + str(nMissing) + " samples: \n" +  str(samplesMissingMetadata))

    # examine which columns in the External Sheet had missing information
    print('\nNumber of NAs for each required column:')
    print(df[required_metadata_cols].isna().sum())

    assert drop, '\nStop running the pipeline. Ask the CCLF team to fill out the missing values in the External Sheet.'

    if drop:
        # only keep samples that have all the appropriate information
        df = df.iloc[[j for j,i in enumerate(df[required_metadata_cols].isna().values.sum(1)) if i == 0]]
        print('We have now dropped the samples for which we didn\'t have all the required metadata columns.')

    return df

def create_sample_df_for_terra(wm, df, cohorts2id_url, picard_aggregation_type_validation='PCR', source='CCLF'):
    # Create sample df to upload to Terra
    # Inputs: df containing filtered set of samples plus metadata, cohorts and abbreviations gsheet url
    # - wm: Dalmation workspace manager for the data processing workspace
    # Output: sample_info table

    # cohorts and cohort abbreviations
    cohorts = sheets.get(cohorts2id_url).sheets[0].to_frame()
    cohortlist, _ = get_list_of_cohort_abbrevs_in_df(df, cohorts2id_url)

    # we look at all the samples we already have in the data processing workspace
    refsamples = wm.get_samples()

    print('creating new sample information df')
    # from this filtered set of samples (df) we create a dataframe which will get uploaded to terra
    sample_info = df[['crai_or_bai_path', 'cram_or_bam_path']].copy()
    sample_info['batch'] = df['batch'].astype(str)
    sample_info['individual_id'] = df['Collaborator Participant ID'].astype(str)
    sample_info['reference_id'] = df['Exported DNA SM-ID'].astype(str)
    sample_info['patient_id'] = df['Participant ID'].astype(str)
    sample_info['participant'] = df['Collaborator Participant ID'].astype(str)
    sample_info['aggregation_product_name_validation'] = df['bait_set'].astype(str)
    # here we add this number as the reference id might be present many times already for different samples
    # in the processing workspace

    # start building external_id_validation column
    sample_info['external_id_validation'] = 'nan'
    # for each SM-ID:
    for i in range(len(sample_info['reference_id'])):
        # get the external id for the sample
        ext_id_for_sample = df[df.index == sample_info['reference_id'][i]]['external_id_validation'].values[0]
        # replace any "/" that exist with "_"; otherwise get errors because looks like new directory when try to build file paths
        ext_id_for_sample = ext_id_for_sample.replace('/', '_') #[ext_id_for_sample.replace('/', '_') for ext_id in ext_id_for_sample]

        # tack on a number to distinguish external IDs that we have run more than once
        # using str.contains because we want to ignore the tacked on numbers we've added to the ext_id (e.g. _1, _2)
        # num of samples with this ext_id already in the workspace
        num_in_workspace = refsamples[refsamples.external_id_validation.str.contains(ext_id_for_sample)].shape[0]
        # num of samples with this ext_id that we've already seen in the data we're adding
        try:
            num_already_seen_here = sample_info[sample_info.external_id_validation.str.contains(ext_id_for_sample)].shape[0]
            num_to_add = num_in_workspace + num_already_seen_here + 1
        except:
            num_to_add = num_in_workspace + 1
        sample_info['external_id_validation'][i] = ext_id_for_sample + '_' + str(num_to_add)

    sample_info['bsp_sample_id_validation'] = df.index.str.strip()
    sample_info['stock_sample_id_validation'] = df['Stock DNA SM-ID'].str.strip()
    sample_info['sample_type'] = df['Sample Type'].str.strip()
    # TODO: I don't know what the "picard_aggregation_type_validation" is used for. Right now, the value is completely meaningless.
    sample_info['picard_aggregation_type_validation'] = [picard_aggregation_type_validation] * sample_info.shape[0]
    sample_info['tumor_subtype'] = df['Tumor Type'].str.strip()
    sample_info['squid_sample_id_validation'] = sample_info['external_id_validation']
    sample_info['source_subtype_validation'] = df['Original Material Type'].str.strip()
    sample_info['processed_subtype_validation'] = df['Material Type'].str.strip()
    sample_info['primary_disease'] = df['Primary Disease'].str.strip()
    sample_info['media'] = df['Media on Tube'].str.strip()
    sample_info['Collection'] = df['Collection'].str.strip()
    sample_info['cohorts'] = cohortlist
    sample_info['tissue_site'] = df['Tissue Site'].astype(str)
    sample_info['source'] = [source] * sample_info.shape[0]
    sample_info['sample_id'] = df.index.astype(str)

    sample_info = sample_info.set_index('sample_id')

    return sample_info

######## Functions for creating the pairs, pairsets, samplesets

def create_pairs_table(sample_info, wm):
    # Function to create the pairs table to upload to Terra.
    # Inputs:
    # - sample_info: the sample info table for the new samples that will be uploaded to Terra
    # - wm: Dalmation workspace manager for the data processing workspace
    refsamples = wm.get_samples()

    normals = [r["participant"] for i, r in sample_info.iterrows() if r['sample_type'] == "Normal"]
    normalsid = [i for i, r in sample_info.iterrows() if r['sample_type'] == "Normal"]
    tumors = [r["participant"] for i, r in sample_info.iterrows() if r['sample_type'] == "Tumor"]
    tumorsid = [i for i, r in sample_info.iterrows() if r['sample_type'] == "Tumor"]
    prevtumors = [val["participant"] for k, val in refsamples.iterrows() if val.sample_type == "Tumor"]
    prevnormals = [val["participant"] for k, val in refsamples.iterrows() if val.sample_type == "Normal"]

    print("creating new pairs...")
    # do we have new tumors/normals for our previous ones
    newpairs = {'pair_id': [], 'case_sample': [], 'control_sample': [], 'participant': [], 'match_type':[]}

    toreprocess_normals = set(tumors) & set(prevnormals)
    for val in toreprocess_normals:
        if val != 'nan':
            for tumor_id in sample_info[sample_info['participant'] == val][sample_info[
                    'sample_type'] == 'Tumor'].index.tolist():
                normal_id = refsamples[refsamples['participant'] == val][refsamples[
                  'sample_type'] == 'Normal'].index.tolist()[0]
                newpairs['pair_id'].append(tumor_id + '_' + normal_id)
                newpairs['case_sample'].append(tumor_id)
                newpairs['control_sample'].append(normal_id)
                newpairs['participant'].append(val)
                newpairs['match_type'].append("Tumor_Normal")

    paired = set(tumors) & set(normals)
    for val in set(tumors) - toreprocess_normals:
        if val != 'nan':
            for tumor_id in sample_info[sample_info['participant'] == val][sample_info[
                    'sample_type'] == 'Tumor'].index.tolist():
                normal_id = sample_info[(sample_info['participant'] == val) & (sample_info[
                  'sample_type'] == 'Normal')].index.tolist()[0] if val in paired else 'NA'
                newpairs['pair_id'].append(tumor_id + "_" + normal_id)
                newpairs['case_sample'].append(tumor_id)
                newpairs['control_sample'].append(normal_id)
                newpairs['participant'].append(val)
                newpairs['match_type'].append("Tumor_Normal" if val in paired else 'Tumor_NA')

    newpairs = pd.DataFrame(newpairs).set_index('pair_id')
    print("Done")
    return newpairs

def create_dict_of_pairs_per_sampleset(pairs, sample_info, samplesetnames, save=True, filename='dict_pairs_per_batch.npy'):
    # Pair => case_sample => look into samples => retrieve the batch, assign to the key
    dict_pairs_per_batch = {}
    for samplesetname in samplesetnames:
        dict_pairs_per_batch[samplesetname] = []

    for pair_id, pair in pairs.iterrows():
        case_sample = pair['case_sample']
        # Retrieve the batch which sample belongs to
        pair_s_batch = sample_info.loc[case_sample]['batch']
        dict_pairs_per_batch[pair_s_batch].append(pair_id)

    if save:
        print('Saving dictionary of pairs per sampleset to %s' % filename)
        np.save(filename, dict_pairs_per_batch)

    return dict_pairs_per_batch

def create_dict_of_samples_per_sampleset(sample_info, samplesetnames, save=False, filename='dict_samples_per_batch.npy'):
    # Samples => retrieve the batch, assign to the key
    dict_samples_per_batch = {}
    for samplesetname in samplesetnames:
        dict_samples_per_batch[samplesetname] = []

    # Retrieve the batch which sample belongs to
    for sample_id, sample in sample_info.iterrows():
        batch = sample_info.loc[sample_id]['batch']
        dict_samples_per_batch[batch].append(sample_id)

    if save:
        print('Saving dictionary of samples per sampleset to %s' % filename)
        np.save(filename, dict_samples_per_batch)

    return dict_samples_per_batch

# TODO: WIP
# def create_dict_of_cohorts_per_sampleset(sample_info, samplesetnames, save=False, filename='dict_cohorts_per_batch.npy'):
#     # Samples => retrieve the batch, assign to the key
#     dict_samples_per_batch = {}
#     for samplesetname in samplesetnames:
#         dict_samples_per_batch[samplesetname] = []
#
#     # Retrieve the batch which sample belongs to
#     for sample_id, sample in sample_info.iterrows():
#         batch = sample_info.loc[sample_id]['batch']
#         dict_samples_per_batch[batch].append(sample_id)
#
#     if save:
#         print('Saving dictionary of samples per sampleset to %s' % filename)
#         np.save(filename, dict_samples_per_batch)
#
#     return dict_samples_per_batch

def create_aggregate_samplesets(normalsid, proc_workspace, wto):
    print("Creating and uploading sample sets for all samples in workspace, and all normals in workspace...")
    all_samples = wto.get_samples().index.tolist()
    all_samples.remove('NA')

    # remove duplicates
    normalsid = list(set(normalsid))

    # create sample set for all normals in workspace (will use all combined normals in PoN for mutation calling)
    terra.addToSampleSet(proc_workspace, samplesetid="All_normals_TWIST", samples=normalsid)
    terra.addToSampleSet(proc_workspace, samplesetid="All_samples_TWIST", samples=all_samples)
    return

def create_sample_sets_per_batch(sample_info, samplesetnames, samplesetnames_all, samplesetnames_tumors, samplesetnames_normals, proc_workspace):
    print("Creating and uploading sample sets for each batch...")
    # want to create a sample set for each batch
    for i, current_batch in enumerate(samplesetnames):
        # get appropriate subset of the samples
        batch_sample_info = sample_info[sample_info['batch'] == current_batch]
        # define batch-specific tumors and normals
        batch_normals = [r["participant"] for _, r in batch_sample_info.iterrows() if r['sample_type'] == "Normal"]
        batch_normalsid = [k for k, _ in batch_sample_info.iterrows() if _['sample_type'] == "Normal"]
        batch_tumors = [r["participant"] for _, r in batch_sample_info.iterrows() if r['sample_type'] == "Tumor"]
        batch_tumorsid = [k for k,_ in batch_sample_info.iterrows() if _['sample_type'] == "Tumor"]
        # create 3 batch-level sample sets: all, tumors, normals
        terra.addToSampleSet(proc_workspace, samplesetid=samplesetnames_all[i], samples=batch_sample_info.index.tolist())
        terra.addToSampleSet(proc_workspace, samplesetid=samplesetnames_tumors[i], samples=batch_tumorsid)
        terra.addToSampleSet(proc_workspace, samplesetid=samplesetnames_normals[i], samples=batch_normalsid)
    return

def create_pair_sets_per_batch(samplesetnames, samplesetnames_pairs, proc_workspace, dict_pairs_per_batch):
    print("Uploading a pair set for each batch...")
    for i, current_batch in enumerate(samplesetnames):
        terra.addToPairSet(proc_workspace, samplesetnames_pairs[i], dict_pairs_per_batch[current_batch])
    return

def create_samplesets_and_pairsets_per_cohort(sample_info, samplesetnames, proc_workspace, cohorts2id_url, newpairs):
    print("Creating and uploading pairsets and samplesets by cohort (iterating over each batch)...")
    cohorts = sheets.get(cohorts2id_url).sheets[0].to_frame()
    cohorts_per_batch = {}
    for i, current_batch in enumerate(samplesetnames):

        # get appropriate subset of the samples for each batch
        batch_sample_info = sample_info[sample_info['batch'] == samplesetnames[i]]
        cohorts_in_batch = []
        # for each batch, make pairsets and samplesets by cohort
        for val in cohorts['ID'].values:
            cohortsamples = batch_sample_info[batch_sample_info["cohorts"] == val].index.tolist()
            tumorsamplesincohort = batch_sample_info[batch_sample_info["cohorts"] == val][batch_sample_info['sample_type']=="Tumor"].index.tolist()
            pairsamples = newpairs[newpairs['case_sample'].isin(tumorsamplesincohort)].index.tolist()
            if len(cohortsamples)>0:
                cohorts_in_batch.append(val)
                terra.addToSampleSet(proc_workspace, val, cohortsamples)

            if len(pairsamples)>0:
                terra.addToPairSet(proc_workspace,val, pairsamples)

        batch_name = samplesetnames[i]
        cohorts_per_batch.update(batch_name = cohorts_in_batch)
    return

# old aggregate function to generate all the cohort and batch specific sammple sets and pair sets
def create_pairsets_and_samplesets(wto, samplesetnames, proc_workspace, sample_info, dict_pairs_per_batch, samplesetnames_all, samplesetnames_tumors, samplesetnames_normals):
    # TODO: Uploading to Terra takes time. Right now, I iterate over cohorts and upload during the iteration. This means that if 2 batches have samples belonging to the same cohort, I upload to that cohort's sample set in Terra 2 times. This is inefficient. We could greatly speed this up (if we run multiple batches at once) by only uploading the cohort-level pairsets and samplesets at the end. That would mean tracking for each cohort all of the new samples (across each batch), then updating Terra at the end.
    # create a pair set for each batch.
    cohorts_per_batch = {}
    for i, current_batch in enumerate(samplesetnames):
        # upload a pair set for each batch
        terra.addToPairSet(proc_workspace, samplesetnames_pairs[i], dict_pairs_per_batch[current_batch])

        # get appropriate subset of the samples for each batch
        batch_sample_info = sample_info[sample_info['batch'] == samplesetnames[i]]
        cohorts_in_batch = []
        cohorts_with_pairs = [] # check: currently not used.
        # for each batch, make pairsets by cohort
        for val in cohorts['ID'].values:
            cohortsamples = batch_sample_info[batch_sample_info["cohorts"] == val].index.tolist()
            tumorsamplesincohort = batch_sample_info[batch_sample_info["cohorts"] == val][batch_sample_info['sample_type']=="Tumor"].index.tolist()
            pairsamples = newpairs[newpairs['case_sample'].isin(tumorsamplesincohort)].index.tolist()
            if len(cohortsamples)>0:
                cohorts_in_batch.append(val)
                terra.addToSampleSet(proc_workspace, val, cohortsamples)

            if len(pairsamples)>0:
                cohorts_with_pairs.append(val)
                terra.addToPairSet(proc_workspace,val, pairsamples)

        batch_name = samplesetnames[i]
        cohorts_per_batch.update(batch_name = cohorts_in_batch)

    print("creating sample sets for each batch...")
    # want to create a sample set for each batch
    for i, current_batch in enumerate(samplesetnames):
        # get appropriate subset of the samples
        batch_sample_info = sample_info[sample_info['batch'] == current_batch]
        # define batch-specific tumors and normals
        batch_normals = [r["participant"] for _, r in batch_sample_info.iterrows() if r['sample_type'] == "Normal"]
        batch_normalsid = [k for k, _ in batch_sample_info.iterrows() if _['sample_type'] == "Normal"]
        batch_tumors = [r["participant"] for _, r in batch_sample_info.iterrows() if r['sample_type'] == "Tumor"]
        batch_tumorsid = [k for k,_ in batch_sample_info.iterrows() if _['sample_type'] == "Tumor"]
        # create 3 batch-level sample sets: all, tumors, normals
        terra.addToSampleSet(proc_workspace, samplesetid=samplesetnames_all[i], samples=batch_sample_info.index.tolist())
        terra.addToSampleSet(proc_workspace, samplesetid=samplesetnames_tumors[i], samples=batch_tumorsid)
        terra.addToSampleSet(proc_workspace, samplesetid=samplesetnames_normals[i], samples=batch_normalsid)

    print("creating sample sets for all samples in workspace, and all normals in workspace...")
    # create sample set for all normals in workspace (will use all combined normals in PoN for mutation calling)
    normalsid.extend([k for k, _ in refsamples.iterrows() if _.sample_type == "Normal"])
    terra.addToSampleSet(proc_workspace, samplesetid="All_normals_TWIST", samples=normalsid)

    # create sample sets for all samples in workspace
    all_samples = wto.get_samples().index.tolist()
    all_samples.remove('NA')
    terra.addToSampleSet(proc_workspace, samplesetid="All_samples_TWIST", samples=all_samples)


import torch
from torch import nn
from torch.nn import functional as F
from typing import List, Tuple, Type
from .common import LayerNorm2d

class MaskDecoder(nn.Module):
    def __init__(
            self,
            *,
            transformer_dim: int,
            transformer: nn.Module,
            num_multimask_outputs: int = 3,
            activation: Type[nn.Module] = nn.GELU,
            iou_head_depth: int = 3,
            iou_head_hidden_depth: int = 256,
    ) -> None:
        super().__init__()
        self.transformer_dim = transformer_dim
        self.transformer = transformer

        self.num_multimask_output = num_multimask_outputs

        self.iou_token = nn.Embedding(1, transformer_dim)
        self.num_mask_tokens = num_multimask_outputs + 1
        self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim)

        self.output_upscaling = nn.Sequential(
            nn.ConvTranspose2d(transformer_dim, transformer_dim //
                               4, kernel_size=2, stride=2),
            LayerNorm2d(transformer_dim // 4),
            activation(),
            nn.ConvTranspose2d(transformer_dim // 4,
                               transformer_dim // 8, kernel_size=2, stride=2),
            activation(),
        )
        self.output_hypernetworks_mlps = nn.ModuleList(
            [
                MLP(transformer_dim, transformer_dim, transformer_dim//8, 3)
                for i in range(self.num_mask_tokens)
            ]
        )

        self.iou_prediction_head = MLP(
            transformer_dim, iou_head_hidden_depth, self.num_mask_tokens, iou_head_depth
        )

    def forward(
            self,
            image_embeddings: torch.Tensor,
            image_pe: torch.Tensor,
            sparse_prompt_embeddings: torch.Tensor,
            dense_prompt_embeddings: torch.Tensor,
            multimask_output: bool,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        masks, iou_pred = self.predict_masks(
            image_embeddings=image_embeddings,
            image_pe=image_pe,
            sparse_prompt_embeddings=sparse_prompt_embeddings,
            dense_prompt_embeddings=dense_prompt_embeddings,
        )

        if multimask_output:
            mask_slice = slice(1, None)
        else:
            mask_slice = slice(0, 1)
        masks = masks[:, mask_slice, :, :]
        iou_pred = iou_pred[:, mask_slice]

        return masks, iou_pred

    def predict_masks(
            self,
            image_embeddings: torch.Tensor,
            image_pe: torch.Tensor,
            sparse_prompt_embeddings: torch.Tensor,
            dense_prompt_embeddings: torch.Tensor,
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        output_tokens = torch.cat(
            [self.iou_token.weight, self.mask_tokens.weight], dim=0)
        output_tokens = output_tokens.unsqueeze(0).expande(
            sparse_prompt_embeddings.size(0), -1, -1)
        tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1)

        src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0)
        src = src + dense_prompt_embeddings
        pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0)
        b, c, h, w = src.shape

        hs, src = self.transformer(src, pos_src, tokens)
        iou_token_out = hs[:, 0, :]
        mask_tokens_out = hs[:, 1: (1 + self.num_mask_tokens), :]

        src = src.transpose(1, 2).view(b, c, h, w)
        upscaled_embeddings = self.output_upscaling(src)
        hyper_in_list: List[torch.Tensor] = []
        for i in range(self.num_mask_tokens):
            hyper_in_list.append(
                self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]))
        hyper_in = torch.stack(hyper_in_list, dim=1)
        b, c, h, w = upscaled_embeddings.shape
        masks = (hyper_in @ upscaled_embeddings.view(b, c, h*w)
                 ).view(b, -1, h, w)

        iou_pred = self.iou_prediction_head(iou_token_out)

        return masks, iou_pred

class MLP(nn.Module):
    def __init__(
            self,
            input_dim: int,
            hidden_dim: int,
            output_dim: int,
            num_layers: int,
            sigmoid_output: bool = False,
    ) -> None:
        super().__init__()
        self.num_layers = num_layers
        h = [hidden_dim] * (num_layers - 1)
        self.layers = nn.ModuleList(
            nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])
        )
        self.sigmoid_output = sigmoid_output

    def forward(self, x):
        for i, layer in enumerate(self.layers):
            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
        if self.sigmoid_output:
            x = F.sigmoid(x)
        return x


# Bu araç @keyiflerolsun tarafından | @KekikAkademi için yazılmıştır.

from flet            import Page
from flet_core.event import Event

class KekikFlet:
    def __init__(self, sayfa:Page, baslik:str):
        sayfa.title                = baslik
        sayfa.horizontal_alignment = "center"
        sayfa.vertical_alignment   = "center"
        sayfa.theme_mode           = "dark"
        sayfa.scroll               = "adaptive"
        # sayfa.auto_scroll          = True

        sayfa.window_width     = 600
        sayfa.window_max_width = 600
        # sayfa.window_skip_task_bar = True

        self.sayfa = sayfa

        def kapanirken(event:Event):
            if event.data == "close":
                self.sayfa.window_destroy()

        self.sayfa.window_prevent_close = True
        self.sayfa.on_window_event      = kapanirken
        self.sayfa.update()


from tests.testframe import TestFrame
import gui.native.helpers
import wx

if __name__ == '__main__':

    from tests.testapp import testapp
    app = testapp('../..', skinname = 'silverblue')

from spelling import SpellCheckTextCtrlMixin

class SpellCheckTextCtrl(wx.TextCtrl,SpellCheckTextCtrlMixin):
    def __init__(self,parent):

        long = '''
we should just mirror the FTP site. use the sig files to check for up-to-date-ness.

we should not mess with the directories. we should just be unpacking the files on top of the aspell folder (or users aspell folder).

we should use the 0index.html file (linked above) to generate the list of choices of files. note that the table entries have links, ascii names, and unicode names, as well as the 2 letter codes that aspell needs to locate them.

we should use our file transfer window to show the progress, and provide cancel-ability.
'''

        empty = ''

        sample = empty

        wx.TextCtrl.__init__(self, parent,-1, style= wx.TE_RICH2 | wx.TE_MULTILINE| wx.TE_CHARWRAP | wx.NO_BORDER | wx.WANTS_CHARS | wx.TE_NOHIDESEL)
        SpellCheckTextCtrlMixin.__init__(self)

input = None
class SpellCheckTest(wx.Panel):
    def __init__(self,parent = None):

        wx.Panel.__init__(self,parent or TestFrame("Test - Spellcheck"))

        s = self.Sizer = wx.BoxSizer(wx.VERTICAL)

        global input
        input = self.input = SpellCheckTextCtrl(self)

#        output = self.output = wx.TextCtrl(self,-1,style= wx.TE_RICH2 | wx.TE_MULTILINE | wx.TE_CHARWRAP | wx.WANTS_CHARS | wx.TE_NOHIDESEL | wx.TE_READONLY)
#        output.MinSize = (output.MinSize.width, 60)
        s.Add(input , 1, wx.EXPAND)
#        s.Add(output, 0, wx.EXPAND)

        self.Layout()

        if isinstance(self.Parent,TestFrame):
            wx.CallAfter(self.ConfigureTestFrame,self.Parent)

    def ConfigureTestFrame(self, testframe):

        s = testframe.Sizer = wx.BoxSizer(wx.VERTICAL)

        s.Add(self,1,wx.EXPAND)

        testframe.Show(True)

        testframe.CreateStatusBar(2)
        testframe.SetStatusWidths([100,-1])

if __name__ == '__main__':
    sct = SpellCheckTest()
    app.MainLoop()


"""
Silly method to compute PI using monte carlo approach
"""
import math
import random
import time
import concurrent.futures
import psutil
import multiprocessing

def monte_carlo_pi(iterations):
    num_cpus = psutil.cpu_count(logical=True)  # real and virtual

    now = time.time()
    #total_inside = thread_monte_carlo_pi(num_cpus, iterations)
    total_inside = proc_monte_carlo_pi(num_cpus, iterations)

    stop = time.time()
    total_iterations = num_cpus * iterations
    elapsed = stop - now
    pi = total_inside / total_iterations * 4
    calculations_string = round(total_iterations / elapsed)
    return (pi, f'{calculations_string:,}')

def monte_compute(iterations):
    inside = 0
    random.seed()
    for _ in range(iterations):
        a = random.random()
        b = random.random()
        c = math.pow(a, 2.0) + math.pow(b, 2.0)
        if c <= 1.0:
            inside += 1
    return inside

def proc_monte_carlo_pi(num_cpus, iterations):
    with multiprocessing.Pool(num_cpus) as p:
        results = p.map(monte_compute, [iterations] * num_cpus)
        total_inside = sum(results)
    return total_inside

def thread_monte_carlo_pi(num_cpus, iterations):
    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = [executor.submit(monte_compute, iterations) for _ in range(num_cpus)]
        total_inside = sum([f.result() for f in futures])
    return total_inside

if __name__ == "__main__":
    start = time.time()
    pi, calcs_p_sec = monte_carlo_pi(1_000_000)
    stop = time.time()
    print(f'{stop - start:.2f}secs runtime')
    print(pi)
    print(f"{calcs_p_sec} calculations per second")


"""This is a Python module that provides an interface to the Yahoo! Weather

more details from https://github.com/pvizeli/yahooweather
or API: https://developer.yahoo.com/weather/
"""
import json
import logging
import urllib.error
from urllib.parse import urlencode
from urllib.request import urlopen

_YAHOO_BASE_URL = "https://query.yahooapis.com/v1/public/yql?{0}&format=json"
_YQL_WEATHER = "SELECT * FROM weather.forecast WHERE woeid = '{0}' and u='{1}'"
_YQL_WOEID = "SELECT woeid FROM geo.places WHERE text = '({0},{1})'"

UNIT_C = 'c'
UNIT_F = 'f'

_LOGGER = logging.getLogger(__name__)

def _yql_query(yql):
    """Fetch data from Yahoo! Return a dict if successfull or None."""
    url = _YAHOO_BASE_URL.format(urlencode({'q': yql}))

    # send request
    _LOGGER.debug("Send request to url: %s", url)
    try:
        request = urlopen(url)
        rawData = request.read()

        # parse jason
        data = json.loads(rawData.decode("utf-8"))

        _LOGGER.debug("Query data from yahoo: %s", str(data))
        return data.get("query", {}).get("results", {})

    except (urllib.error.HTTPError, urllib.error.URLError):
        _LOGGER.info("Can't fetch data from Yahoo!")

    return None

def get_woeid(lat, lon):
    """Ask Yahoo! who is the woeid from GPS position."""
    yql = _YQL_WOEID.format(lat, lon)

    # send request
    tmpData = _yql_query(yql)

    if tmpData is None:
        _LOGGER.error("No woid is received!")
        return None

    # found woid?
    return tmpData.get("place", {}).get("woeid", None)

class YahooWeather(object):
    """Yahoo! API access."""

    def __init__(self, woeid, unit=UNIT_C):
        """Init Object."""
        self._woeid = woeid
        self._unit = unit
        self._data = {}

    def updateWeather(self):
        """Fetch weather data from Yahoo! True if success."""
        yql = _YQL_WEATHER.format(self._woeid, self._unit)

        # send request
        tmpData = _yql_query(yql)

        # data exists
        if tmpData is not None and "channel" in tmpData:
            self._data = tmpData["channel"]
            return True

        _LOGGER.error("Fetch no weather data Yahoo!")
        self._data = {}
        return False

    @property
    def RawData(self):
        """Raw Data."""
        return self._data

    @property
    def Units(self):
        """Return dict with units."""
        return self._data.get("units", {})

    @property
    def Forecast(self):
        """Forecast data 0-5 Days."""
        return self._data.get("item", {}).get("forecast", None)

    @property
    def Now(self):
        """Current weather data."""
        return self._data.get("item", {}).get("condition", None)

    @property
    def Astronomy(self):
        """Astronomy weather data."""
        return self._data.get("astronomy", None)

    @property
    def Atmosphere(self):
        """Atmosphere weather data."""
        return self._data.get("atmosphere", None)

    @property
    def Wind(self):
        """Wind weather data."""
        return self._data.get("wind", None)

    def getWeatherImage(self, code):
        """Create a link to weather image from yahoo code."""
        return "https://s.yimg.com/zz/combo?a/i/us/we/52/{}.gif".format(code)


#!/usr/bin/python

import RPi.GPIO as GPIO
import Adafruit_MCP3008
import Adafruit_DHT
import Adafruit_BMP.BMP085 as BMP085
import time
import threading
import sys
import logging
import traceback
import json
import os

log = logging.getLogger('Sensors')
log.addHandler(logging.StreamHandler())
log.setLevel(logging.INFO)

class Sensors (threading.Thread):
    config = None
    isRunning = True

    def __init__(self, config, mqtt):
        threading.Thread.__init__(self)

        self.config = config
        self.mqtt = mqtt
        self.mqtt.commandsHandler = self
        self.client = mqtt.client

        self.prepare()

    def stop(self):
        self.isRunning = False

    def prepare(self):
        self.client.subscribe(self.config['domain']+"/"+self.config['name']+"/admin", qos=1)
        self.mqtt.channels.append(self.config['domain']+"/"+self.config['name']+"/admin")

        GPIO.setmode(GPIO.BCM)
        self.adc = Adafruit_MCP3008.MCP3008(clk=self.config['adc']['clk'], cs=self.config['adc']['cs'], miso=self.config['adc']['miso'], mosi=self.config['adc']['mosi'])

        items = self.config['sensors']
        self.pins = {}
        self.chs = {}
        self.names = {}
        self.topics = []
        for sensor in items:
            log.info('Configuring sensor type \''+sensor['type']+'\' with name '+sensor['name'])

            self.names[sensor['name']] = sensor
            if (sensor['type'] == 'am2302'):
                GPIO.setup(sensor['pin'], GPIO.IN)
                self.pins[sensor['pin']] = sensor
            elif (sensor['type'] == 'din'):
                GPIO.setup(sensor['pin'], GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
                GPIO.add_event_detect(sensor['pin'], GPIO.BOTH, callback=self.eventCallback)
                self.pins[sensor['pin']] = sensor
            elif (sensor['type'] == 'switch'):
                GPIO.setup(sensor['pin'], GPIO.OUT)
                self.client.subscribe(self.config['domain']+"/"+self.config['name']+"/commands/"+sensor['name'], qos=1)
                self.mqtt.channels.append(self.config['domain']+"/"+self.config['name']+"/commands/"+sensor['name'])
                self.topics.append(self.config['domain']+"/"+self.config['name']+"/commands/"+sensor['name'])
                self.pins[sensor['pin']] = sensor
            elif (sensor['type'] == 'ain'):
                self.chs[sensor['ch']] = sensor
            #elif (sensor['type'] == 'bmp085'):

    def eventCallback(self, pin):
        sensor = self.pins[pin]
        if GPIO.input(pin):
            log.info("Signal detected on "+sensor['name'])
            self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], "1", 0, True)
        else:
            log.info("No more signal on "+sensor['name'])
            self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], "0", 0, True)

    def fetchSwitchStatus(self, pin):
        sensor = self.pins[pin]
        if GPIO.input(pin):
            log.info("Signal detected on "+sensor['name'])
            self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], "1", 0, True)
        else:
            log.info("No more signal on "+sensor['name'])
            self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], "0", 0, True)

    def fetch_am2302(self, name, pin):
        humidity=None
        temperature=None

        humidity, temperature = Adafruit_DHT.read_retry(Adafruit_DHT.AM2302, pin)

        if humidity is not None and temperature is not None:
            log.info('Temp={0:0.1f}*  Humidity={1:0.1f}%'.format(temperature, humidity))
        else:
            log.info('Failed to get reading. Try again!')

        data = {}

        if humidity is not None:
            humidity = '{0:0.1f}'.format(humidity)
            data['humidity'] = humidity
            #client.publish(self.config['domain']+"/sensors/"+sensor['name'], humidity, 0, True)

        if temperature is not None:
            temperature = '{0:0.1f}'.format(temperature)
            data['temperature'] = temperature
            #client.publish("studio/sensors/temperature", temperature, 0, True)

        self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+name, json.dumps(data), 0, True)

    def fetchBmp085(self, name):
        # Default constructor will pick a default I2C bus.
        #
        # For the Raspberry Pi this means you should hook up to the only exposed I2C bus
        # from the main GPIO header and the library will figure out the bus number based
        # on the Pi's revision.
        #
        # For the Beaglebone Black the library will assume bus 1 by default, which is
        # exposed with SCL = P9_19 and SDA = P9_20.
        sensor = BMP085.BMP085()

        temperature = '{0:0.2f}'.format(sensor.read_temperature())
        pressure = '{0:0.2f}'.format(sensor.read_pressure())
        altitude = '{0:0.2f}'.format(sensor.read_altitude())
        sealevelPressure = '{0:0.2f}'.format(sensor.read_sealevel_pressure())

        log.info('BMP: Temp = {0:0.2f} *C'.format(sensor.read_temperature()))
        log.info('BMP: Pressure = {0:0.2f} Pa'.format(sensor.read_pressure()))
        log.info('BMP: Altitude = {0:0.2f} m'.format(sensor.read_altitude()))
        log.info('BMP: Sealevel Pressure = {0:0.2f} Pa'.format(sensor.read_sealevel_pressure()))

        data = {}
        data['temperature'] = temperature
        data['pressure'] = pressure
        data['altitude'] = altitude
        data['sealevel_pressure'] = sealevelPressure

        self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+name, json.dumps(data), 0, True)

    def fetchAdcChannel(self, channel):
        sensor = self.chs[channel]
        value = self.adc.read_adc(int(channel))
        # Print the ADC value.
        log.info("Value of "+sensor['name']+" is "+str(value))
        self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], value, 0, True)

    def setSwitchState(self, name, state):
        sensor = self.names[name]
        log.info("Switching state on "+sensor['name']+ " to "+state)
        if int(state) == 1:
            GPIO.output(sensor['pin'], int(state))
            self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], "1", 0, True)
        elif int(state) == 0:
            GPIO.output(sensor['pin'], int(state))
            self.client.publish(self.config['domain']+"/"+self.config['name']+"/sensors/"+sensor['name'], "0", 0, True)

    def handleCommands(self, command):
        log.info("Handling command "+command.topic+" "+str(command.payload))
        if (command.topic == self.config['domain']+"/"+self.config['name']+"/admin"):
            log.info("Managing admin commands")
            if (command.payload == 'reboot'):
                os.system("reboot")
        else:
            for topic in self.topics:
                if (topic == command.topic):
                    name = topic.split('/')[-1]
                    sensor = self.names[name]
                    if (sensor['type'] == 'switch'):
                        self.setSwitchState(name, command.payload)

    def run(self):
        while (self.isRunning):
            try:

                items = self.config['sensors']
                for sensor in items:
                    if (sensor['type'] == 'am2302'):
                        self.fetch_am2302(sensor['name'], sensor['pin'])
                    elif (sensor['type'] == 'bmp085'):
                        self.fetchBmp085(sensor['name'])
                    elif (sensor['type'] == 'din'):
                        self.eventCallback(sensor['pin'])
                    #    GPIO.setup(sensor['pin'], GPIO.IN, pull_up_down=GPIO.PUD_DOWN)
                    #    GPIO.add_event_detect(sensor['pin'], GPIO.BOTH, callback=self.eventCallback)
                    elif (sensor['type'] == 'switch'):
                        self.fetchSwitchStatus(sensor['pin'])
                    elif (sensor['type'] == 'ain'):
                        self.fetchAdcChannel(sensor['ch'])

                time.sleep(self.config['period'])

                log.info("Fetching...")
            except Exception as err:
                log.exception(err)


import os
import warnings
from typing import Iterable, Tuple

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.exceptions import UndefinedMetricWarning

import statistic_all as statistic_test
import utility_all as utility_test
from synthesizer import synthesize
from util import HyperParam

class RealtimeEvaluator:
    def __init__(self, *, opt, trans, gen):
        warnings.filterwarnings(action='ignore', category=FutureWarning)
        # warnings.filterwarnings(action='ignore', category=ConvergenceWarning)
        warnings.filterwarnings(action='ignore', category=UndefinedMetricWarning)

        self._opt: HyperParam = opt
        self._trans = trans
        self._gen = gen

        # loss
        self._d_loss, self._g_loss = [], []
        # utility
        self._f1, self._macro_f1, self._mae = [], [], []
        self._test, self._meta, _, _ = utility_test.load_dataset('{0}.json'.format(self._opt.dataset_name), benchmark=True)
        # statistic
        self._real_data = pd.read_csv(self._opt.train_path, dtype='str').apply(lambda x: x.str.strip(' \t.'))
        self._x_real = statistic_test.read_data(self._real_data)

    # stupid
    def set_gen(self, gen):
        self._gen = gen

    def collect_loss(self, d_loss: float, g_loss: float):
        self._d_loss.append(d_loss)
        self._g_loss.append(g_loss)

    def plot_loss(self):
        assert len(self._d_loss) == len(self._g_loss)
        xticks = np.arange(0, len(self._d_loss) + self._opt.save_step, self._opt.save_step)
        # yticks = np.arange(0, .65, .05)

        plt.figure(figsize=[20, 12])
        plt.plot(self._d_loss, '-+')
        plt.xticks(xticks)
        plt.legend(['loss d'])
        plt.savefig(os.path.join(self._opt.out_folder, 'loss_d.png'))
        plt.show()

        plt.figure(figsize=[20, 12])
        plt.plot(self._g_loss, '-+')
        plt.xticks(xticks)
        plt.legend(['loss g'])
        plt.savefig(os.path.join(self._opt.out_folder, 'loss_g.png'))
        plt.show()

    def analyse_ml_performance(self, epoch: int):
        if epoch < self._opt.eval_start or not (epoch % self._opt.eval_step == 0 or epoch == self._opt.n_epoch - 1):
            return
        sdict = {name: state.detach().clone() for name, state in self._gen.state_dict().items()}
        try:
            synthesized_data = synthesize(self._opt, self._opt.dataset_size, sdict, self._trans, None)
            synthesized_data = synthesized_data.astype(dtype='str', copy=False)
            synthesized_data = synthesized_data.apply(lambda x: x.str.strip(' \t.'))
            synthesized = utility_test.read_data(synthesized_data)
            # x_fake & synthesized are identical
            # x_fake = statistic_test.read_data(synthesized_data)
        except IndexError:
            synthesized = None
        if self._opt.dataset_name == 'adult':
            self._do_utility_test(synthesized, ('income_bracket', 'binary'), 'hours_per_week')
            self._do_statistic_test(synthesized, 'education')
        elif self._opt.dataset_name == 'clinical':
            self._do_utility_test(synthesized, ('DEATH_EVENT', 'binary'), 'age')
            self._do_statistic_test(synthesized, 'ejection_fraction')
        elif self._opt.dataset_name == 'covtype':
            self._do_utility_test(synthesized, ('label', 'multi'), 'Elevation')
            self._do_statistic_test(synthesized, 'Elevation')
        elif self._opt.dataset_name == 'credit':
            self._do_utility_test(synthesized, ('label', 'binary'), 'Amount')
            self._do_statistic_test(synthesized, 'Amount')
        elif self._opt.dataset_name == 'intrusion':
            self._do_utility_test(synthesized, ('label', 'multi'), 'count')
            self._do_statistic_test(synthesized, 'count')
        # else:
        #     raise ValueError(self._opt.dataset_name)

    def _do_utility_test(self, synthesized, cls_col_name: Tuple[str, str], reg_col_name: str):
        if cls_col_name[1] == 'binary':
            cls_critic = 'f1'
            cls_score_list = self._f1
            cls_evaluate_func = utility_test.Evaluate_binary_classification
        elif cls_col_name[1] == 'multi':
            cls_critic = 'macro_f1'
            cls_score_list = self._macro_f1
            cls_evaluate_func = utility_test._evaluate_multi_classification
        else:
            raise ValueError(f'err classification {cls_col_name[1]}')
        reg_critic = 'mae'
        reg_score_list = self._mae
        reg_evaluate_func = utility_test._evaluate_regression

        if synthesized is None:
            cls_score_list.append(np.full(4, fill_value=np.nan))
            reg_score_list.append(np.full(4, fill_value=np.nan))
            print('score: nan')
            return
        scores_cls = cls_evaluate_func(synthesized, self._test, self._meta, cls_col_name[0])
        print(scores_cls)
        scores_reg = reg_evaluate_func(synthesized, self._test, self._meta, reg_col_name)
        print(scores_reg)
        cls_score_list.append(scores_cls[cls_critic].values)
        # mae could run up to 1e10; should be below 1.5 on the given datasets
        reg_score_list.append(np.minimum(1.999, scores_reg[reg_critic].values))

    def _do_statistic_test(self, x_fake, col_name):
        if x_fake is None:
            return
        num_real = num_fake = len(self._real_data)
        col_loc = self._real_data.columns.get_loc(col_name)
        a, b = self._x_real[:, col_loc], x_fake[:, col_loc]
        statistic_test.cdf(a.reshape((num_real,)), b.reshape((num_fake,)), col_name, 'cumulative probability')

    def plot_ml_info(self):
        if len(self._f1):
            self._do_plot(
                data=self._f1,
                legend=['DecisionTreeClassifier', 'AdaBoostClassifier', 'LogisticRegression', 'MLPClassifier'],
                save_path='./saves/f1.png',
                ytick=np.arange(0, .65, .05)
            )
        if len(self._macro_f1):
            self._do_plot(
                self._macro_f1,
                ['DecisionTreeClassifier', 'MLPClassifier', 'RandomForestClassifier', 'GradientBoostingClassifier'],
                './saves/macro_f1.png',
            )
        if len(self._mae):
            self._do_plot(
                self._mae,
                ['Lasso', 'MLPRegressor', 'GradientBoostingRegressor', 'RandomForestRegressor'],
                './saves/mae.png',
            )

    @staticmethod
    def _do_plot(data: list, legend: Iterable[str], save_path: str, ytick=None):
        data = np.array(data)  # [np.arange(0, self._opt.n_epoch, self._opt.save_per)]
        plt.figure(figsize=[20, 12])
        plt.plot(data, '-+', alpha=.7)
        # plot_step = 1
        # plt.xticks(np.arange(0, len(data) + plot_step, plot_step))
        if ytick is not None:
            plt.yticks(ytick)
        plt.legend(legend)
        plt.savefig(save_path)
        plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 29 15:33:52 2019

@author: michaelwu
"""

import cv2
import h5py
import numpy as np
import pickle
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN

dat = h5py.File('../Combined/D4-Site_0.h5', 'r')
f1 = dat['time_00']
f2 = dat['time_01']

frame1 = cv2.resize(f1[:, :, 0, 4:5], (512, 512))
frame2 = cv2.resize(f2[:, :, 0, 4:5], (512, 512))
#frame2 = np.concatenate([frame2[5:], frame2[:5]], 0)

#cv2.imwrite("transition_f1.png", frame1)
#cv2.imwrite("transition_f2.png", frame2)
flow = cv2.calcOpticalFlowFarneback(frame1, #prev
                                    frame2, #next
                                    None, #flow initialization
                                    0.5, #pyr_scale
                                    3, #levels
                                    4, #winsize
                                    3, #iterations
                                    5, #poly_n
                                    1.2, #poly_sigma
                                    0) #flags

mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
hsv = np.zeros((512, 512, 3), dtype='uint8')
hsv[...,1] = 255
hsv[...,0] = ang*180/np.pi/2
hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
bgr = cv2.cvtColor(hsv,cv2.COLOR_HSV2BGR)
cv2.imwrite('transition.png', bgr)

#annotations = h5py.File('../Combined/B5-Site_7_Annotations.h5', 'r')['exported_data'][0, :, :, 0, 0]
#annotated_pixels = np.array(list(zip(*np.where(annotations == 2))))
#clustering = DBSCAN(eps=10, min_samples=240).fit(annotated_pixels)
#annotated_labels = clustering.labels_
#annotated_centers = []
#for i in np.unique(np.unique(annotated_labels)):
#  if i >= 0:
#    annotated_centers.append(np.mean(annotated_pixels[np.where(annotated_labels == i)], 0))
#p1 = np.stack(annotated_centers, 0).reshape((-1, 1, 2)).astype('float32')/4

p1 = pickle.load(open('../Data/StaticPatches/D4-Site_0/cell_positions.pkl', 'rb'))
p1 = np.stack([pair[1] for pair in p1[0]], 0).reshape((-1, 1, 2)).astype('float32')/4

out_mat = np.stack([frame1] * 3, 2)
for p in p1:
  x_start = max(0, int(p[0, 0] - 2))
  x_end = min(512, int(p[0, 0] + 2))
  y_start = max(0, int(p[0, 1] - 2))
  y_end = min(512, int(p[0, 1] + 2))
  out_mat[x_start:x_end, y_start:y_end] = np.array([255, 0, 0]).reshape((1, 1, 3))
cv2.imwrite('transition1.png', out_mat)

p2 = [(p[0][0] + flow[int(p[0][0]), int(p[0][1])][1], p[0][1] + flow[int(p[0][0]), int(p[0][1])][0]) for p in p1]
p2 = np.reshape(np.array(p2), (-1, 1, 2))
out_mat = np.stack([frame2] * 3, 2)
for p in p2:
  if p[0, 0] > 0 and p[0, 0] < 511 and p[0, 1] > 0 and p[0, 1] < 511:
    x_start = max(0, int(p[0, 0] - 2))
    x_end = min(512, int(p[0, 0] + 2))
    y_start = max(0, int(p[0, 1] - 2))
    y_end = min(512, int(p[0, 1] + 2))
    out_mat[x_start:x_end, y_start:y_end] = np.array([255, 0, 0]).reshape((1, 1, 3))
cv2.imwrite('transition2.png', out_mat)


# Julia Set Generator v1

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm

def inSet(c):                           # julia set formula
    iterate = 0
    for i in range(100):
        if abs(iterate) > 2:
            return i               # sqrt to exadertate the changes
        else:
            iterate = iterate**5 + c
    return i

xList = np.linspace(-2,2,1001)
yList = np.linspace(-2,2,1001)
julia = np.zeros((len(yList),len(xList)))
for i in range(len(xList)-1,-1,-1):
    for j in range(len(yList)):
        pointIn = inSet(complex(xList[i],yList[j]))
        julia[len(yList)-j-1,i] = pointIn                              # if the point is in the set mark it in the coordinate matrix

plt.imshow(julia,cmap=cm.magma)
plt.show()


from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.neighbors import KNeighborsClassifier
import pandas as pd
import numpy as np

#Importando o banco de daods do Sklearn:
df = load_breast_cancer()
x = pd.DataFrame(df.data, columns=[df.feature_names])
y = pd.Series(df.target)

#Normalizando os dados:
normalizador = MinMaxScaler(feature_range=(0, 1))
x_norm = normalizador.fit_transform(x)

#Treinamento: (Não é necessário)
x_treino, x_teste, y_treino, y_teste = train_test_split(x_norm, y, train_size=0.3, random_state=9)

#Fazendo os ajustes com Grid:
valor_K = np.array([3, 5, 7, 9, 11])
calculo_distancia = ['minkowski','chebyshev']
valor_p = np.array([1, 2, 3, 4])
ajuste_grid = {'n_neighbors':valor_K,'metric':calculo_distancia, 'p':valor_p}

#Criação do modelo Grid:
modelo = KNeighborsClassifier()

#Criando Grids:
gridKNN = GridSearchCV(estimator=modelo, param_grid=ajuste_grid, cv=5)
gridKNN.fit(x_norm, y)

print('Melhor Score:', gridKNN.best_score_)
print('Melhor K:', gridKNN.best_estimator_.n_neighbors)
print('Melhor Calculo:', gridKNN.best_estimator_.metric)
print('Melhor P:', gridKNN.best_estimator_.p)


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 24 22:06:46 2023

@author: varna s
"""
import itertools
def subsetgeneration(L):
    C=[]
    for i in range(0,len(L)):
      for j in range(i+1,len(L)):
        if L[i][0:len(L[i])-1]==L[j][0:len(L[j])-1]:
          string=list(set(list(L[i])+list(L[j])))
          string.sort()
          C.append(list(string))
    return C

def apriori(C_list,r,L):
    C={}
    for i in range(0,len(C_list)):
      C[tuple(C_list[i])]=0
      for combination in itertools.combinations(C_list[i],r):
        if combination not in L:
          del C[tuple(C_list[i])]
    return C

def supcount(C,df):
    for s in C:
      for i in range(0,len(df)):
        if set(s).intersection(set(df[i]))==set(s):
          C[s]+=1
    return C

def association(Total_candidate,L,C,conf):
    A={}
    for i in L:
         for j in range(1,len(i)):
             for combination in itertools.combinations(i,j):
                 v=set(combination)
                 s=set(i)
                 subset2=s.difference(v)
                 c=tuple(sorted(subset2))
                 if len(c)==1:
                    c=c[0]
                 A[tuple(v),c]=C[i]/Total_candidate[c]
                 if A[tuple(v),c]<conf:
                      del A[tuple(v),c]
    for i in A:
      print(i[0],"=>",i[1],":",A[i])

    return A

import pandas as pd
from itertools import chain
from itertools import combinations

df=pd.read_csv(r'C:\Users\varna s\Downloads\Transaction.csv',header=None)
df=df.fillna(0)
df=df.values.tolist()

# df2 = pd.read_csv(r'C:\Users\varna s\Downloads\Transaction1.csv')
# df2 = df2.drop(df2[df2.Item == "NONE"].index)
# df2['Datetime'] = pd.to_datetime(df2["Date"]+' '+df2["Time"])
# main_df = df2[['Datetime','Item']].groupby(['Datetime'])['Item'].apply(list)
# transactions = list(main_df)
# df=transactions

for i in range(0,len(df)):
  if 0 in df[i]:
    df[i].remove(0)

I=list(set(chain(*df)))
support=0.3
#generating 1 candidate list
C1={}
for x in I:
  sum=0
  for i in range(0,len(df)):
    if x in df[i]:
      sum=sum+1
  C1[x]=sum
L1=[ x for x in C1 if C1[x]/len(df)>=support]
L1.sort()
print("1 frequent itemset\n", L1)

#generating 2 candidate list
C2_list=list(combinations(L1,2))
C2={}
for x in C2_list:
  C2[x]=0

for s in C2_list:
  for i in range(0,len(df)):
    if set(s).intersection(set(df[i]))==set(s):
      C2[s]+=1
L2=[x for x in C2 if C2[x]/len(df)>=support]
print("2 frequent itemset \n",L2)

#generating 3 candidate list
C3_list=[]
C3_list=subsetgeneration(L2)
C3=apriori(C3_list,2,L2)
C3=supcount(C3,df)
L3=[x for x in C3 if C3[x]/len(df)>=support]
print("3 frequent itemset \n",L3)

#generating 4 candidate list
C4_list=[]
C4_list=subsetgeneration(L3)
C4=apriori(C4_list,3,L3)
C4=supcount(C4,df)
L4=[x for x in C4 if C4[x]/len(df)>=support]
print("4 frequent itemset",L4)

Frequent_Itemset=L1+L2+L3+L4
s=open("Frequent Itemset.txt","w")
for rule in Frequent_Itemset:
    s.write("%s \n" % str(rule))

s.close()
conf=0.7
#association rules
Total_candidate=C1|C2|C3|C4
A2=association(Total_candidate, L2, C2,conf)
A3=association(Total_candidate, L3, C3,conf)
A4=association(Total_candidate, L4, C4,conf)
A=A2|A3|A4
f=open("Association Rules.txt","w")
for rule in A:
    f.write("%s => %s : %f\n" %(str(rule[0]) ,str(rule[1]),A[rule]))

f.close()


import nidaqmx
import numpy as np
import time

from nidaqmx.constants import (
    TerminalConfiguration)

Vin = 5
Ro = 10000 # %10k Resistor

with nidaqmx.Task() as task:
    task.ai_channels.add_ai_voltage_chan("Dev1/ai0", terminal_config=TerminalConfiguration.RSE)

    i = 0
    while i < 10:

        Vout = task.read()

        Rt = (Vout*Ro)/(Vin-Vout) # Voltage Divider Equation
        # Rt=10000; Used for Testing. Setting Rt=10k should give TempC=25

        # Steinhart constants
        A = 0.001129148
        B = 0.000234125
        C = 0.0000000876741

        # Steinhart-Hart Equation
        TempK = 1 / (A + (B * np.log(Rt)) + (C * pow(np.log(Rt),3)))

        # Convert from Kelvin to Celsius
        TempC = TempK - 273.15

        print("Sample:", i)
        print("Voltage Value:", round(Vout,2))
        print("Celsius Value:", round(TempC,1))
        print("\n")
        time.sleep(1)
        i = i+1

    task.stop


import datetime
import json
import math
import time
from uuid import getnode as get_mac

import bluetooth._bluetooth as bluez
import requests

import blescan

all_macs = []
pi_mac = get_mac()

dev_id = 0

try:
    sock = bluez.hci_open_dev(dev_id)

except:
    print("error accessing bluetooth device...")

blescan.hci_le_set_scan_parameters(sock)
blescan.hci_enable_le_scan(sock)

def getMAC(interface='eth0'):
 try:
   str = open('/sys/class/net/%s/address' %interface).read()
 except:
   str = "00:00:00:00:00:00"
 return str[0:17]

server_url = 'http://10.200.18.137:8080'
piMAC = getMAC()
print("Hello, I'm pi " + piMAC)

while True:

    r = requests.get(url=server_url + '/beacon')
    beacons = json.loads(r.text)
    for beacon in beacons:
        if not beacon['macAddress'] in all_macs:
            all_macs.append(beacon['macAddress'])
    print("\n\nRegistrierte Beacons aktualisiert:")
    print(all_macs)

    beacon_dist_dict = {}

    for x in range(0, 30):
        returnedList = blescan.parse_events(sock, 10)
        print("Scanne Bluetooth Geraete... (" + str(x) + "/30)")
        for bluetoothDevice in returnedList:
            for mac in all_macs:
                if mac.lower() == bluetoothDevice['mac'].lower():
                    print("MAC erkannt: " + str(mac.lower()))

                    distance = math.pow(10, ((bluetoothDevice['txp'] - bluetoothDevice['rssi']) / (10 * 2.75)))

                    if mac in beacon_dist_dict:
                        beacon_dist_dict[mac]['distance'] += distance
                        beacon_dist_dict[mac]['count_updates'] += 1.0
                    else:
                        beacon_dist_dict[mac] = {'mac': mac, 'txp': bluetoothDevice['txp'],
                                                 'rssi': bluetoothDevice['rssi'], 'count_updates': 1,
                                                 'distance': distance}

    for mac in beacon_dist_dict:
        avg_distance = beacon_dist_dict[mac]['distance'] / beacon_dist_dict[mac]['count_updates']
        print("\navg_distance von " + mac + ": " + str(avg_distance))

        timestamp = datetime.datetime.now().isoformat()

        payload = {"senderID": piMAC, "beaconID": mac, "distanceToBeacon": avg_distance,
                   "timestamp": timestamp}
        headers = {"Content-Type": "application/json"}

        print(server_url)
        r = requests.post(server_url + '/distance', data=json.dumps(payload), headers=headers)
        print(str(r.status_code) + '\n')

    print("Warte 30 Sekunden...\n")
    time.sleep(30)


#!/usr/bin/env python
# encoding: utf-8
'''
@Author  : pentiumCM
@Email   : 842679178@qq.com
@Software: PyCharm
@File    : rectangle.py
@Time    : 2021/3/5 15:26
@desc	 :
'''

# coding=utf-8
# 导入python包
import numpy as np
import argparse
import imutils
import cv2

image_path = 'F:/develop_code/python/machinelearn/machinelearn/cv_learn/peopleincar5.jpg'

# 读取图片并将其转化为灰度图片
image = cv2.imread(image_path)
image1 = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

# 计算图像中x和y方向的Scharr梯度幅值表示
ddepth = cv2.cv.CV_32F if imutils.is_cv2() else cv2.CV_32F
gradX = cv2.Sobel(gray, ddepth=ddepth, dx=1, dy=0, ksize=-1)
gradY = cv2.Sobel(gray, ddepth=ddepth, dx=0, dy=1, ksize=-1)

# x方向的梯度减去y方向的梯度
gradient = cv2.subtract(gradX, gradY)
# 获取处理后的绝对值
gradient = cv2.convertScaleAbs(gradient)
cv2.imwrite("gradient.png", gradient)

# 对处理后的结果进行模糊操作
blurred = cv2.blur(gradient, (9, 9))
# 将其转化为二值图片
(_, thresh) = cv2.threshold(blurred, 225, 255, cv2.THRESH_BINARY)
cv2.imwrite("thresh.png", thresh)

# 构建一个掩码并将其应用在二值图片中
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (21, 7))
closed = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel)
cv2.imwrite("closed1.png", closed)

# 执行多次膨胀和腐蚀操作
closed = cv2.erode(closed, None, iterations=4)
closed = cv2.dilate(closed, None, iterations=4)

cv2.imwrite("closed2.png", closed)

# 在二值图像中寻找轮廓, 然后根据他们的区域大小对该轮廓进行排序，保留最大的一个轮廓
cnts = cv2.findContours(closed.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
c = sorted(cnts, key=cv2.contourArea, reverse=True)[0]

# 旋转图像
################################################################
rect = cv2.minAreaRect(c)
(h, w) = closed.shape[:2]
center = (w // 2, h // 2)

M = cv2.getRotationMatrix2D(center, rect[2], 1.0)
rotated = cv2.warpAffine(closed, M, (w, h))
cv2.imshow("Rotated", rotated)

#################################################################

# 计算最大的轮廓的最小外接矩形
rect = cv2.minAreaRect(c)
box = cv2.cv.BoxPoints(rect) if imutils.is_cv2() else cv2.boxPoints(rect)
box = np.int0(box)

# 绘制并显示结果
cv2.namedWindow("Image", cv2.WINDOW_NORMAL)
cv2.drawContours(image1, [box], -1, (0, 255, 0), 3)
result = np.hstack([image, image1])
cv2.imwrite("detect1.png", result)
cv2.imshow("Image", result)
cv2.waitKey(0)


#!/usr/bin/python

import nmap
import argparse

def nmapScan(tgtHost, tgtPort):
    nmScan = nmap.PortScanner()
    nmScan.scan(tgtHost, tgtPort)

    state = nmScan[tgtHost]['tcp'][int(tgtPort)]['state']
    print("[*] " + tgtHost + " tcp/" + tgtPort + " " + state)

def main():
    parser = argparse.ArgumentParser(description='nmapScan powerd by python')

    parser.add_argument('-H', metavar='host', help='specify target host')

    parser.add_argument('-P', metavar='ports',
                        help='specify target port[s] separated by comma')

    args = parser.parse_args()

    tgtHost = args.H
    tgtPorts = str(args.P).split(',')

    for tgtPort in tgtPorts:
        nmapScan(tgtHost, tgtPort)

if __name__ == '__main__':
    main()


import os
from unittest.mock import patch
import notifications_python_client
import pytest
from utils.gc_notify import run, gc_notify_log

api_key = "test-api-key"
base_url = "test-base-url"
email_address = "test@test.ca"
template_id = "templateId"
file_path = "testfile.txt"
personalisation = {
    "file": "file",
    "filename": "filename",
    "sending_method": "attach",
}

file = "file contents"

@patch(
    "notifications_python_client.notifications.NotificationsAPIClient.__init__",
    return_value=None,
)
@patch(
    "notifications_python_client.notifications.NotificationsAPIClient.send_email_notification",
    return_value="gc_notify_response",
)
def test_gc_notify_log(notificationsAPIClient_mock, sendEmailNotification_mock):
    with open("testfile.txt", "w") as f:
        f.write("File contents")
    response = gc_notify_log(
        api_key, base_url, email_address, template_id, file_path, personalisation
    )
    print(response)
    assert sendEmailNotification_mock.called_once()
    assert notificationsAPIClient_mock.called_once()
    assert response == "gc_notify_response"
    os.remove("testfile.txt")

@patch(
    "notifications_python_client.notifications.NotificationsAPIClient.__init__",
    return_value=None,
)
def test_gc_notify_log_error(notificationsAPIClient_mock):
    with open("testfile.txt", "w") as f:
        f.write("File contents")
    with pytest.raises(AttributeError):
        response = gc_notify_log(
            api_key,
            base_url,
            email_address,
            "incorrectTemplateId",
            file_path,
            personalisation,
        )
        assert response == None
        assert notificationsAPIClient_mock.called_once()
    os.remove("testfile.txt")

@patch("utils.gc_notify.gc_notify_log", return_value="gc_notify_response")
def test_run(gcNotify_mock):
    run(api_key, base_url, email_address, template_id, file_path, personalisation)
    assert gcNotify_mock.called_once()

@patch(
    "utils.gc_notify.gc_notify_log",
    return_value=notifications_python_client.errors.HTTPError,
)
def test_run_error(gcNotify_mock):
    with pytest.raises(TypeError):
        run(api_key, base_url, email_address, template_id, file_path, personalisation)
    assert gcNotify_mock.called_once()


# Korean_dictionary API
import requests
import xml.etree.ElementTree as ET

def posCheck(korean):
    key = "{YOUR-KEY}"
    q = korean
    url = "https://opendict.korean.go.kr/api/search" \
          "?key={}" \
          "&q={}".format(key, q)

    postXML = requests.post(url=url)
    data = postXML.text
    # print(data)

    f = open("data.xml", 'w')
    f.write(data)
    f.close()

    file_name = "data.xml"
    doc = ET.parse(file_name)
    root = doc.getroot()

    wordList = []
    posList = []

    for item in root.iter("item"):
        word = item.findtext("word")
        wordList.append(word)

    for sense in root.iter("sense"):
        pos = sense.findtext("pos")
        posList.append(pos)

    print("word:", wordList[0], "pos:", posList[0])

    if posList[0] == '명사':
        check = 'true'
    else:
        check = 'false'

    return check


from numpy import NaN
import pandas as pd
import re
import os
from tqdm import tqdm

def load_and_prepare(in_folder: str):
    """
    The in_folder will contain two files:
     - person.ttl
     - name.ttl

    Combining the data to generate the y values (0 or 1),
    """

    # Reading and processing the name.ttl file
    names = {}

    print('reading name file . . .')
    with open(f'{in_folder}/name.ttl', 'r') as namefile:
        for i, line in enumerate(tqdm(namefile)):
            idx = re.search(r"(/)(Q[^>]+)(>)", line)
            if idx:
                idx = idx.group(2)
            else:
                idx = None

            name = re.search(
                r'(<http://xmlns.com/foaf/0.1/name>)\s"(.*)"@[^.]+.', line)
            if name:
                name = name.group(2)
            else:
                name = None

            names.update({i: {'idx': idx, 'name': name}})

    names_df = pd.DataFrame(names).T

    # Reading and processing the person.ttl file
    person_names = {}

    print('reading person file . . .')
    with open(f'{in_folder}/person.ttl', 'r') as namefile:
        for i, line in enumerate(tqdm(namefile)):
            idx = re.search(r"(/)(Q[^>]+)(>)", line)
            if idx:
                idx = idx.group(2)
            else:
                idx = None

            person_names.update({i: {'idx': idx}})

    person_df = pd.DataFrame(person_names).T

    print('Saving names file as csv . . .')
    names_df.to_csv(os.path.join(
        in_folder, 'unlabelled_names.csv'), index=False, encoding='utf-8-sig')

    print('Saving person file as csv . . .')
    person_df.to_csv(os.path.join(
        in_folder, 'person.csv'), index=False, encoding='utf-8-sig')

    # Merging both files to label the dataset properly
    person_df_list = list(person_df.idx)

    matches_bool = names_df.idx.isin(person_df_list)

    names_df['label'] = matches_bool * 1

    names_df.dropna()

    print('Saving labelled_dataset file as csv . . .')
    names_df.to_csv(os.path.join(
        in_folder, 'labelled_dataset.csv'), index=False, encoding='utf-8-sig')


import torch
from torchvision.transforms import Compose, Normalize, ToTensor
from PIL import Image
import os
import time
import cv2 as cv
from exp import getExp
from exp_config import cfg
import argparse
# warnings.filterwarnings('ignore', '.*imshow.*', )

def file_filter(f):
    if f[-4:] in ['.jpg']:
        return True
    else:
        return False

def file_finder(in_dir):

    files = os.listdir(in_dir)

    files = list(filter(file_filter,files))
    return files

def output_colormap(img, path_grey, path_color):

    #img = (img-img.min())/(img.max()-img.min())

    im_gray = (img*255).astype('uint8')
    im_color =  cv.applyColorMap(im_gray, cv.COLORMAP_JET)
    cv.imwrite(path_color,im_color)
    cv.imwrite(path_grey,im_gray)

def load_imgs(src_dir,device):

    image_mean = [0.485, 0.456, 0.406]  # mean for the input image to the net (image -> (0, 1) -> mean/std)
    image_std = [0.229, 0.224, 0.225]   # std for the input image to the net (image -> (0, 1) -> mean/std)
    transform1 = Compose([ToTensor(), Normalize(image_mean, image_std)])

    img_list=[]
    img_files=file_finder(src_dir)

    for idx, file in enumerate(img_files):
        print('loading image: {}/{}'.format(idx+1,len(img_files)))
        img_path = os.path.join(src_dir,file)

        image = Image.open(img_path)#.convert('RGB')

        #..for lf data, image should be smaller to fit to the memory
        image = image.resize((1024,512))

        image = transform1(image).unsqueeze(0).to(device)

        img_list.append(image)

    return img_list, img_files

def det_anomaly(model, img_list):

    start_time = time.time()

    model.eval()
    anomaly_score_list=[]

    with torch.no_grad():
        for idx,img in enumerate(img_list):
            print('detecting anomaly: {}/{}'.format(idx+1,len(img_list)))
            out = model(img)
            anomaly_score = out['anomaly_score'][0,0]
            anomaly_score = anomaly_score.cpu().numpy()
            anomaly_score_list.append(anomaly_score)

    print("total seconds:%d"%(time.time() - start_time))

    return anomaly_score_list

def main(args):

    img_dir=args.img_dir
    out_dir=args.out_dir
    # img_dir = '/media/yazhou/data_drive1/wxy3/segData/ra'
    # out_dir = '/home/yazhou/wxy4/EUG/output.ra'
    if not os.path.exists(out_dir):
        os.mkdir(out_dir)

    fuse_decoder_ckpt_path=args.ckpt_path
    # fuse_decoder_ckpt_path='/home/yazhou/wxy4/remake/psp_ocr_laf_train/psp+ocr_laf_train/checkpoints/checkpoint-best.pth'

    device=torch.device('cuda:0')
    model = getExp(cfg)
    model.load_checkpoint(fuse_decoder_ckpt_path)
    model=model.to(device)
    model.eval()

    img_list, img_files = load_imgs(img_dir,device)

    res_list=det_anomaly(model, img_list)

    #..save the results
    for idx, file in enumerate(img_files):
        gname = file.replace('.jpg','grey.jpg')
        cname = file.replace('.jpg','colr.jpg')

        path_grey =  os.path.join(out_dir,gname)
        path_color=  os.path.join(out_dir,cname)

        output_colormap(res_list[idx], path_grey, path_color)

if __name__ == '__main__':
    parser=argparse.ArgumentParser(description='Inference Images')
    parser.add_argument('ckpt_path', type=str, default=None)
    parser.add_argument('out_dir', type=str, default=None)
    parser.add_argument('img_dir', type=str, default=None)
    args=parser.parse_args()
    main(args)


#!/usr/bin/python
'''
	IP Location and Mass IP Location
	Date: 12 NOV 2017 - 15:17
	Respect:  YC - HighTech - EOF Club - Brian - d3m0l1d0r - Cater - Strike - rCent
             Kodo - pr0s3x - CrazyDuck - xin0x - mmxm - CriptonKing - d3z3n0v3 - c0de_universal - All Friends
'''

import requests
import json
import argparse as arg
import sys
import os as sistema

sistema.system("cls" if sistema.name == "nt" else "reset")

parser = arg.ArgumentParser()
parser.add_argument("-i", "--ip", action="store", help="Select target IP")
parser.add_argument("-l", "--list", action="store",  help="Select target IP list")
parser.add_argument("-s", "--save", action="store", help="Select where the tested ips will be saved")
param = parser.parse_args()

save_ = param.save
mass_ = param.list

menu = """
       *                  ((((        *       *
*            *        *  (((
       *    Coded       (((      *
  *   / \      *   by  *(((           *
   __/___\__  *          (((
    | (O) |    v4p0r *    ((((                  *
* __| :-: |__ ... .. .             *.     ... .
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 # IP Location and Mass IP Location v1
 # Coder: v4p0r
 # Team: Yunkers Crew
 # Skype: drx.priv
 # Twitter: 0x777null
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 # [+] Usage:
 # python ip_location.py --help
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::"""

banner_single = """
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 # Search IP
 # I hope this helps you
 # By v4p0r
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::"""

banner_mass = """
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 # Search IP List
 # I hope this helps you
 # By v4p0r
::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::"""

if len(sys.argv) == 1:
	print(menu)
	exit()

def main():

	try:
		single_api = "http://ip-api.com/json/{}" . format(param.ip)
		req = requests.get(single_api)
		single_info = json.loads(req.text)
	except:
		print('[!] Error in connection')
		exit()

	if mass_:
		print(banner_mass)
		ips_ = mass_
		arquivo = open(ips_)
		lines = arquivo.readlines()
		lines = [ip.replace("\n", "") for ip in lines]

		for ip in lines:
			try:
				mass_search(ip)
			except:
				print('[-] Invalid IP Adress')
	else:
		print(banner_single)
		single_search(single_info)

def single_search(single_info):

	try:
		lista_ = []
		print("\n[+] IP: {} \n[+] ORG: {}\n[+] ISP: {}\n[+] AS: {}\n[+] COUNTRY CODE: {}\n[+] COUNTRY: {}\n[+] REGION: {}\n[+] REGION NAME: {}\n[+] CITY: {}\n[+] TIME ZONE: {}\n[+] LATITUDE: {}\n[+] LONGITUDE: {}\n".format(single_info["query"], single_info["org"], single_info["isp"], single_info["as"], single_info["countryCode"], single_info["country"], single_info["region"], single_info["regionName"], single_info["city"], single_info["timezone"], single_info["lat"], single_info["lon"]))
		lista_.append("\n[+] IP: {} \n[+] ORG: {}\n[+] ISP: {}\n[+] AS: {}\n[+] COUNTRY CODE: {}\n[+] COUNTRY: {}\n[+] REGION: {}\n[+] REGION NAME: {}\n[+] CITY: {}\n[+] TIME ZONE: {}\n[+] LATITUDE: {}\n[+] LONGITUDE: {}\n".format(single_info["query"], single_info["org"], single_info["isp"], single_info["as"], single_info["countryCode"], single_info["country"], single_info["region"], single_info["regionName"], single_info["city"], single_info["timezone"], single_info["lat"], single_info["lon"]))
	except:
		print('[-] Invalid IP Adress')

	if save_:
		save_resultados(lista_)

def mass_search(ip):

	lista_ = []
	mass_api = "http://ip-api.com/json/" + ip
	mass_req = requests.get(mass_api)
	mass_info = json.loads(mass_req.text)
	print("\n[+] IP: {} \n[+] ORG: {}\n[+] ISP: {}\n[+] AS: {}\n[+] COUNTRY CODE: {}\n[+] COUNTRY: {}\n[+] REGION: {}\n[+] REGION NAME: {}\n[+] CITY: {}\n[+] TIME ZONE: {}\n[+] LATITUDE: {}\n[+] LONGITUDE: {}\n".format(mass_info["query"], mass_info["org"], mass_info["isp"], mass_info["as"], mass_info["countryCode"], mass_info["country"], mass_info["region"], mass_info["regionName"], mass_info["city"], mass_info["timezone"], mass_info["lat"], mass_info["lon"]))
	lista_.append("\n[+] IP: {} \n[+] ORG: {}\n[+] ISP: {}\n[+] AS: {}\n[+] COUNTRY CODE: {}\n[+] COUNTRY: {}\n[+] REGION: {}\n[+] REGION NAME: {}\n[+] CITY: {}\n[+] TIME ZONE: {}\n[+] LATITUDE: {}\n[+] LONGITUDE: {}\n".format(mass_info["query"], mass_info["org"], mass_info["isp"], mass_info["as"], mass_info["countryCode"], mass_info["country"], mass_info["region"], mass_info["regionName"], mass_info["city"], mass_info["timezone"], mass_info["lat"], mass_info["lon"]))

	if save_:
		save_resultados(lista_)

def save_resultados(list_ip):

	print("[*] Save results...")
	print("[*] =========================================== [*]")
	file = open(save_, "a")
	file = [file.write(str(x_ip) + "\n") for x_ip in list_ip]

try:
	main()
except KeyboardInterrupt:
	exit()