# -*- coding: utf-8 -*-
"""
Created on Sun May  5 19:20:59 2019

@author: Sanchayni
"""

from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('ml-bank').getOrCreate()
spark.conf.set("spark.sql.execution.arrow.enabled", "true")

RDD_01 = spark.read.csv('New_cs_data.csv', header = True, inferSchema = True)
RDD_01.printSchema()

df = RDD_01.select("*").toPandas()
df = df.drop(['_c0'],axis=1)
df = df.drop(['Unnamed: 0'],axis=1)

print(df.columns)
import numpy as np
import pygal
import pandas as pd

df = df.apply(lambda x: x.replace('NA',np.nan))
df = df.apply(lambda x: x.astype(float))

print(df.isna().sum())

# Plotting

# --------------- UNIVARIATE plots ---------------

# Age
age_chart = pygal.Bar(x_label_rotation=-90)
age_chart.title = 'Histogram for variable Age'
age_chart.x_labels = map(str,pd.DataFrame(df.groupby('age').age.count()).index)
age_chart.add('age' ,  pd.DataFrame(df.groupby('age').age.count()).age)
age_chart.render_to_file('EDA_svgs/age_chart.svg')

# Dept Ratio
DebtRatio_chart = pygal.Bar(x_label_rotation=45)
DebtRatio_chart.title = 'Histogram for variable Debt Ratio'
DebtRatio_chart.x_labels = map(str, pd.DataFrame(pd.cut(df['DebtRatio'],20).value_counts().sort_index()).index)
DebtRatio_chart.add('DeptRatio' ,  pd.DataFrame(pd.cut(df['DebtRatio'],20).value_counts().sort_index()).DebtRatio)
DebtRatio_chart.render_to_file('EDA_svgs/DebtRatio_chart.svg')

# Monthly Income
Income_chart = pygal.Bar(x_label_rotation=45)
Income_chart.title = 'Histogram for variable Monthly Income'
Income_chart.x_labels = map(str, pd.DataFrame(pd.cut(df['MonthlyIncome'],20).value_counts().sort_index()).index)
Income_chart.add('MonthlyIncome' ,  pd.DataFrame(pd.cut(df['MonthlyIncome'],20).value_counts().sort_index()).MonthlyIncome)
Income_chart.render_to_file('EDA_svgs/Income_chart.svg')

# NumberOfDependents
NOD_chart = pygal.Bar(x_label_rotation=-90)
NOD_chart.title = 'Histogram for variable Number of Dependents'
NOD_chart.x_labels = map(str,pd.DataFrame(df.groupby('NumberOfDependents').NumberOfDependents.count()).index)
NOD_chart.add('NumberOfDependents',  pd.DataFrame(df.groupby('NumberOfDependents').NumberOfDependents.count()).NumberOfDependents)
NOD_chart.render_to_file('EDA_svgs/NOD_chart.svg')

# SeriousDlqin2yrs
SDy_chart = pygal.Bar(x_label_rotation=-90)
SDy_chart.title = 'Histogram for variable SeriousDlqin2yrs'
SDy_chart.x_labels = map(str,pd.DataFrame(df.groupby('SeriousDlqin2yrs').SeriousDlqin2yrs.count()).index)
SDy_chart.add('SeriousDlqin2yrs',  pd.DataFrame(df.groupby('SeriousDlqin2yrs').SeriousDlqin2yrs.count()).SeriousDlqin2yrs)
SDy_chart.render_to_file('EDA_svgs/SDy_chart.svg')

# RevolvingUtilizationOfUnsecuredLines
RUOUL_chart = pygal.Bar(x_label_rotation=-90)
RUOUL_chart.title = 'Histogram for variable RevolvingUtilization Of Unsecured Lines'

RUOUL_chart.x_labels = map(str,pd.DataFrame(pd.cut(df['RevolvingUtilizationOfUnsecuredLines'],20).value_counts().sort_index()).index)
RUOUL_chart.add('RevolvingUtilizationOfUnsecuredLines',  pd.DataFrame(pd.cut(df['RevolvingUtilizationOfUnsecuredLines'],20).value_counts().sort_index()).RevolvingUtilizationOfUnsecuredLines)
RUOUL_chart.render_to_file('EDA_svgs/RUOUL_chart.svg')

# NumberRealEstateLoansOrLines
NRELL_chart = pygal.Bar(x_label_rotation=45)
NRELL_chart.title = 'Histogram for NumberRealEstateLoansOrLines'
NRELL_chart.x_labels = map(str,pd.DataFrame(df.groupby('NumberRealEstateLoansOrLines').NumberRealEstateLoansOrLines.count()).index)
NRELL_chart.add('NumberRealEstateLoansOrLines',  pd.DataFrame(df.groupby('NumberRealEstateLoansOrLines').NumberRealEstateLoansOrLines.count()).NumberRealEstateLoansOrLines)
NRELL_chart.render_to_file('EDA_svgs/NRELL_chart.svg')

# NumberOfOpenCreditLinesAndLoans
NOCLL_chart = pygal.Bar(x_label_rotation=45)
NOCLL_chart.title = 'Histogram for NumberOfOpenCreditLinesAndLoans'
NOCLL_chart.x_labels = map(str,pd.DataFrame(df.groupby('NumberOfOpenCreditLinesAndLoans'). NumberOfOpenCreditLinesAndLoans.count()).index)
NOCLL_chart.add('NumberOfOpenCreditLinesAndLoans',  pd.DataFrame(df.groupby('NumberOfOpenCreditLinesAndLoans'). NumberOfOpenCreditLinesAndLoans.count()).NumberOfOpenCreditLinesAndLoans)
NOCLL_chart.render_to_file('EDA_svgs/NOCLL_chart.svg')

# NumberOfTime30-59DaysPastDueNotWorse
NT30_59PD_chart = pygal.Bar(x_label_rotation=45)
NT30_59PD_chart.title = 'Histogram for variable NumberOfTime30-59DaysPastDueNotWorse'
NT30_59PD_chart.x_labels = map(str,pd.DataFrame(df['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index()).index)
NT30_59PD_chart.add('NumberOfTime30-59DaysPastDueNotWorse',  pd.DataFrame(df['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index())['NumberOfTime30-59DaysPastDueNotWorse'])
NT30_59PD_chart.render_to_file('EDA_svgs/NT30_59PD_chart.svg')

# NumberOfTime60-89DaysPastDueNotWorse
NT60_89PD_chart = pygal.Bar(x_label_rotation=45)
NT60_89PD_chart.title = 'Histogram for variable NumberOfTime60-89DaysPastDueNotWorse'
NT60_89PD_chart.x_labels = map(str,pd.DataFrame(df['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index()).index)
NT60_89PD_chart.add('NumberOfTime60-89DaysPastDueNotWorse',  pd.DataFrame(df['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index())['NumberOfTime60-89DaysPastDueNotWorse'])
NT60_89PD_chart.render_to_file('EDA_svgs/NT60_89PD_chart.svg')

# NumberOfTimes90DaysLate
NT90DL_chart = pygal.Bar(x_label_rotation=45)
NT90DL_chart.title = 'Histogram for variable NumberOfTimes90DaysLate'
NT90DL_chart.x_labels = map(str,pd.DataFrame(df['NumberOfTimes90DaysLate'].value_counts().sort_index()).index)
NT90DL_chart.add('NumberOfTimes90DaysLate',  pd.DataFrame(df['NumberOfTimes90DaysLate'].value_counts().sort_index()).NumberOfTimes90DaysLate)
NT90DL_chart.render_to_file('EDA_svgs/NT90DL_chart.svg')

#-------------- BOXPLOTS --------------

# Age
BP_age_default = pygal.Box(box_mode="1.5IQR")
BP_age_default.title = 'Age variation across Default'
BP_age_default.add('Defaulter', df[df['SeriousDlqin2yrs'] == 1.0].age)
BP_age_default.add('Non Defaulter', df[df['SeriousDlqin2yrs'] == 0.0].age)
BP_age_default.render_to_file('EDA_svgs/BP_age_default.svg')

# DebtRatio
BP_DR_default = pygal.Box(box_mode="tukey")
BP_DR_default.title = 'Dept Ratio variation across Default'
BP_DR_default.add('Defaulter', df[df['SeriousDlqin2yrs'] == 1.0].DebtRatio)
BP_DR_default.add('Non Defaulter', df[df['SeriousDlqin2yrs'] == 0.0].DebtRatio)
BP_DR_default.render_to_file('EDA_svgs/BP_DR_default.svg')

# MonthlyIncome
# BP_MI_default = pygal.Box(box_mode="1.5IQR", range = (min(df['MonthlyIncome']),max(df['MonthlyIncome']) ) )

BP_MI_default = pygal.Box(box_mode="1.5IQR", range = (-10000,40000 ))

BP_MI_default.title = 'Monthly Income variation across Default'
BP_MI_default.add('Defaulter', df[(df['SeriousDlqin2yrs'] == 1.0) & (df['MonthlyIncome'].notna())].MonthlyIncome)
BP_MI_default.add('Non Defaulter', df[(df['SeriousDlqin2yrs'] == 0.0) & (df['MonthlyIncome'].notna())].MonthlyIncome)
BP_MI_default.render_to_file('EDA_svgs/BP_MI_default.svg')

# NumberOfDependents
BP_NOD_default = pygal.Box(box_mode="tukey", range = (-3,10))
BP_NOD_default.title = 'NumberOfDependents variation across Default'
BP_NOD_default.add('Defaulter', df[ (df['SeriousDlqin2yrs'] == 1.0) & (df['NumberOfDependents'].notna())].NumberOfDependents)
BP_NOD_default.add('Non Defaulter', df[ (df['SeriousDlqin2yrs'] == 0.0) & (df['NumberOfDependents'].notna())].NumberOfDependents)
BP_NOD_default.render_to_file('EDA_svgs/BP_NOD_default.svg')

# RevolvingUtilizationOfUnsecuredLines
BP_RUOUL_default = pygal.Box(box_mode="tukey")
BP_RUOUL_default.title = ' RevolvingUtilizationOfUnsecuredLines variation across Default'
BP_RUOUL_default.add('Defaulter', df[df['SeriousDlqin2yrs'] == 1.0].RevolvingUtilizationOfUnsecuredLines)
BP_RUOUL_default.add('Non Defaulter', df[df['SeriousDlqin2yrs'] == 0.0].RevolvingUtilizationOfUnsecuredLines)
BP_RUOUL_default.render_to_file('EDA_svgs/BP_RUOUL_default.svg')

# NumberOfTimes90DaysLate
Default_90ds = pygal.Bar(x_label_rotation=45)
Default_90ds.title = ' Default across Number of times 90 days late'
Default_90ds.x_labels = map(str, df['NumberOfTimes90DaysLate'].value_counts().sort_index().index)
Default_90ds.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ].NumberOfTimes90DaysLate.value_counts().sort_index())
Default_90ds.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ].NumberOfTimes90DaysLate.value_counts().sort_index())

Default_90ds.render_to_file('EDA_svgs/Default_90ds.svg')

# NumberOfTime60-89DaysPastDueNotWorse
Default_60_89ds = pygal.Bar(x_label_rotation=45)
Default_60_89ds.title = ' Default across NumberOfTime60-89DaysPastDueNotWorse'
Default_60_89ds.x_labels = map(str, df['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index().index)
Default_60_89ds.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ]['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index())
Default_60_89ds.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ]['NumberOfTime60-89DaysPastDueNotWorse'].value_counts().sort_index())

Default_60_89ds.render_to_file('EDA_svgs/Default_60_89ds.svg')

# NumberOfTime30-59DaysPastDueNotWorse
Default_30_59ds = pygal.Bar(x_label_rotation=45)
Default_30_59ds.title = ' Default across NumberOfTime30-59DaysPastDueNotWorse'
Default_30_59ds.x_labels = map(str, df['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index().index)
Default_30_59ds.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ]['NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index())
Default_30_59ds.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ][ 'NumberOfTime30-59DaysPastDueNotWorse'].value_counts().sort_index())

Default_30_59ds.render_to_file('EDA_svgs/Default_30_59ds.svg')

# NumberOfOpenCreditLinesAndLoans
Default_OpenCredit = pygal.Bar(x_label_rotation=45)
Default_OpenCredit.title = ' Default across NumberOfOpenCreditLinesAndLoans'
Default_OpenCredit.x_labels = map(str, df['NumberOfOpenCreditLinesAndLoans'].value_counts().sort_index().index)
Default_OpenCredit.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ].NumberOfOpenCreditLinesAndLoans.value_counts().sort_index())
Default_OpenCredit.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ].NumberOfOpenCreditLinesAndLoans.value_counts().sort_index())

Default_OpenCredit.render_to_file('EDA_svgs/Default_OpenCredit.svg')

# NumberRealEstateLoansOrLines
Default_RealEstate= pygal.Bar(x_label_rotation=45)
Default_RealEstate.title = ' Default across NumberRealEstateLoansOrLines '
Default_RealEstate.x_labels = map(str, df['NumberRealEstateLoansOrLines'].value_counts().sort_index().index)
Default_RealEstate.add('Non Defaulter' ,  df[df['SeriousDlqin2yrs'] == 0.0 ].NumberRealEstateLoansOrLines.value_counts().sort_index())
Default_RealEstate.add('Defaulter',  df[df['SeriousDlqin2yrs'] == 1.0 ].NumberRealEstateLoansOrLines.value_counts().sort_index())

Default_RealEstate.render_to_file('EDA_svgs/Default_RealEstate.svg')

print('My work here is done')

# execfile('/home/maria_dev/EDA_CS.py')


import datetime
from dateutil.relativedelta import relativedelta

def now() -> datetime.datetime.strftime:
    today = datetime.datetime.now()
    return today.strftime("%H-%M-%S-%f %d-%m-%Y")

def changed_time(time_lst: list):
    date_and_time = datetime.datetime.now()
    time_change = datetime.timedelta(hours=time_lst[0], minutes=time_lst[1], seconds=time_lst[2])
    new_time = date_and_time + time_change
    return new_time

def time_difference(time_now: str):
    try:
        start = datetime.datetime.strptime(time_now, "%H-%M-%S-%f %d-%m-%Y")
    except ValueError:
        start = datetime.datetime.strptime(now(), "%H-%M-%S-%f %d-%m-%Y")
    ends = datetime.datetime.strptime(now(), "%H-%M-%S-%f %d-%m-%Y")
    difference = relativedelta(ends, start)
    return [difference.hours, difference.minutes, difference.seconds]


import torch
from util.fairness_utils import ImageFolderWithProtectedAttributes, ImageFolderWithProtectedAttributesModify
import torchvision.transforms as transforms
from torch.utils.data import Subset
import random
import os
import errno
import shutil
import json

def load_dict_as_str(filename):
    with open(filename) as f:
        dict = json.loads(f.read())
        for k in dict.keys():
            for i in range(len(dict[k])):
                dict[k][i] = str(dict[k][i])
    return dict

def create_folder(path):
    try:
        os.mkdir(path)
    except OSError as exc:
        if exc.errno != errno.EEXIST:
            raise
        pass

def del_folder(path):
    try:
        shutil.rmtree(path)
    except OSError as exc:
        pass

def print_per_gender_count(dataset):

    labels_gender = {}
    proportion_set = [0] * len(dataset.class_to_idx.keys())

    for idx in range(0, len(dataset)):
        label = dataset.imgs[idx][1]
        gender = dataset[idx][3]

        if label not in labels_gender.keys():
            labels_gender[label] = 0

        labels_gender[label] += gender
        proportion_set[label] += 1

    gender_labels = {}
    gender_labels['male'] = []
    gender_labels['female'] = []

    for label in labels_gender.keys():
        if labels_gender[label] > 0:
            gender_labels['male'].append(label)
        else:
            gender_labels['female'].append(label)

    print(len(gender_labels['male']))
    print(len(gender_labels['female']))

    return;

def get_less_data(dataset, men_proportion, women_proportion):
    labels_gender = {}
    proportion_set = [0] * len(dataset.class_to_idx.keys())

    for idx in range(0, len(dataset)):
        label = dataset.imgs[idx][1]
        gender = dataset[idx][3]

        if label not in labels_gender.keys():
            labels_gender[label] = 0

        labels_gender[label] += gender
        proportion_set[label] += 1

    gender_labels = {}
    gender_labels['male'] = []
    gender_labels['female'] = []

    for label in labels_gender.keys():
        if labels_gender[label] > 0:
            gender_labels['male'].append(label)
        else:
            gender_labels['female'].append(label)

    for label in gender_labels['male']:
        #proportion_set[label] = proportion_set[label] * men_proportion
        #new
        l = proportion_set[label] * men_proportion
        if men_proportion != 0:
            if l < 2:
                l = 2
        proportion_set[label] = l

    for label in gender_labels['female']:
        #proportion_set[label] = proportion_set[label] * women_proportion
        l = proportion_set[label] * women_proportion
        if women_proportion != 0:
            if l < 2:
                l = 2
        proportion_set[label] = l

    return proportion_set

def get_desired_women_labels(dataset, proportion):
    labels_gender = {}

    for idx in range(0, len(dataset)):
        label = dataset.imgs[idx][1]
        gender = dataset[idx][3]

        if label not in labels_gender.keys():
            labels_gender[label] = 0

        labels_gender[label] += gender

    gender_labels = {}
    gender_labels['male'] = []
    gender_labels['female'] = []

    for label in labels_gender.keys():
        if labels_gender[label] > 0:
            gender_labels['male'].append(label)
        else:
            gender_labels['female'].append(label)

    male_count = len(gender_labels['male'])
    female_count = len(gender_labels['female'])

    print(male_count, female_count)
    gender_labels['female'] = gender_labels['female'][0:int(female_count * proportion)]

    male_count = len(gender_labels['male'])
    female_count = len(gender_labels['female'])
    print(male_count, female_count)

    return gender_labels

def get_desired_men_labels(dataset, proportion):
    labels_gender = {}

    for idx in range(0, len(dataset)):
        label = dataset.imgs[idx][1]
        gender = dataset[idx][3]

        if label not in labels_gender.keys():
            labels_gender[label] = 0

        labels_gender[label] += gender

    gender_labels = {}
    gender_labels['male'] = []
    gender_labels['female'] = []

    for label in labels_gender.keys():
        if labels_gender[label] > 0:
            gender_labels['male'].append(label)
        else:
            gender_labels['female'].append(label)

    male_count = len(gender_labels['male'])
    female_count = len(gender_labels['female'])

    print(male_count, female_count)
    gender_labels['male'] = gender_labels['male'][0:int(male_count * proportion)]

    male_count = len(gender_labels['male'])
    female_count = len(gender_labels['female'])
    print(male_count, female_count)

    return gender_labels

def balanced_weights(images, nclasses, attr=1):
    '''
        Make a vector of weights for each image in the dataset, based
        on class frequency. The returned vector of weights can be used
        to create a WeightedRandomSampler for a DataLoader to have
        class balancing when sampling for a training batch.
            images - torchvisionDataset.imgs
            nclasses - len(torchvisionDataset.classes)
        https://discuss.pytorch.org/t/balanced-sampling-between-classes-with-torchvision-dataloader/2703/3
    '''
    count = [0] * nclasses
    for item in images:
        # print(item[attr])
        count[item[attr]] += 1  # item is (img-data, label-id)
    weight_per_class = [0.] * nclasses
    N = float(sum(count))  # total number of images
    for i in range(nclasses):
        if float(count[i]) != 0:
            weight_per_class[i] = N / float(count[i])
        else:
            weight_per_class[i] = 0
    weight = [0] * len(images)
    for idx, val in enumerate(images):
        weight[idx] = weight_per_class[val[attr]]

    return weight

def get_label_count(dataset):
    All_labels = {}

    for idx in range(0, len(dataset)):
        label = dataset.imgs[idx][1]
        if label not in All_labels:
            All_labels[label] = {}

        All_labels[label][dataset.imgs[idx][0]] = 1

    return All_labels

class WomenSampler(torch.utils.data.sampler.Sampler):
    def __init__(self, dataset, labels=None):
        self.labels = labels
        self.dataset = dict()
        self.orig_dataset = dict()
        self.balanced_max = 0

        # Save all the indices for all the classes
        for idx in range(0, len(dataset)):
            label = self._get_label(dataset, idx)
            gender = dataset[idx][3]

            if gender == -1:  # women

                if label not in self.dataset:
                    self.dataset[label] = list()

                self.dataset[label].append(idx)
                self.balanced_max = len(self.dataset[label]) \
                    if len(self.dataset[label]) > self.balanced_max else self.balanced_max

        # Oversample the classes with fewer elements than the max
        for label in self.dataset:
            j = 0
            while len(self.dataset[label]) < self.balanced_max:
                self.dataset[label].append(random.choice(self.dataset[label]))

        self.keys = list(self.dataset.keys())
        self.currentkey = 0
        self.indices = [-1] * len(self.keys)

    def __iter__(self):
        while self.indices[self.currentkey] < self.balanced_max - 1:
            self.indices[self.currentkey] += 1

            yield self.dataset[self.keys[self.currentkey]][self.indices[self.currentkey]]
            self.currentkey = (self.currentkey + 1) % len(self.keys)

        self.indices = [-1] * len(self.keys)

    def _get_label(self, dataset, idx, labels=None):
        # print(dataset[idx])
        return dataset.imgs[idx][1]

    def __len__(self):
        return len(self.dataset.keys())

def prepare_data(args):
    # function prepares data: loads images and prepares dataloaders
    train_transform = transforms.Compose([
        transforms.Resize([int(128 * args.input_size[0] / 112), int(128 * args.input_size[0] / 112)]),
        transforms.RandomCrop([args.input_size[0], args.input_size[1]]),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(mean=args.mean,
                             std=args.std),
    ])
    test_transform = transforms.Compose([
        transforms.Resize([int(128 * args.input_size[0] / 112), int(128 * args.input_size[0] / 112)]),
        transforms.CenterCrop([args.input_size[0], args.input_size[1]]),
        transforms.ToTensor(),
        transforms.Normalize(mean=args.mean,
                             std=args.std)])

    ###################################################################################################################
    # ======= data, loss, network =======#
    demographic_to_classes = load_dict_as_str(args.demographics)
    classes_to_demographic = {}
    for k in demographic_to_classes.keys():
        for cl in demographic_to_classes[k]:
            if k == 'male':
                classes_to_demographic[cl] = 1
            if k == 'female':
                classes_to_demographic[cl] = -1

    datasets = {}
    datasets['train'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform, classes_to_demographic = classes_to_demographic)
    datasets['test'] = ImageFolderWithProtectedAttributesModify(args.data_test_root, transform=test_transform, classes_to_demographic = classes_to_demographic)

    print('Old')
    print(len(datasets['train']))
    print(len(datasets['train'].classes))
    print_per_gender_count(datasets['train'])

    if args.modify_identities:

        if args.modify_gender == 'men':
            gender_labels = get_desired_men_labels(datasets['train'], args.proportion)
        else:
            gender_labels = get_desired_women_labels(datasets['train'], args.proportion)
        datasets['train'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform,
                                                                     gender_labels=gender_labels, classes_to_demographic = classes_to_demographic)

        # change the proportion to 0 for men/ women on new dataset
        # now when I call the function for dataset creation, these proportions are already on new labels

        women = get_less_data(datasets['train'], 0, 1)
        men = get_less_data(datasets['train'], 1, 0)

        if args.proportion != 0 and args.modify_gender == 'men':
            datasets['women'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform,
                                                                         gender_labels=gender_labels, proportions=women, classes_to_demographic = classes_to_demographic)

        if args.proportion != 0 and args.modify_gender == 'women':
            datasets['men'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform,
                                                                       gender_labels=gender_labels, proportions=men, classes_to_demographic = classes_to_demographic)

    if args.modify_data:
        if args.modify_gender == 'men':
            proportions = get_less_data(datasets['train'], args.proportion, 1)
            women = get_less_data(datasets['train'], 0, 1)
            men = get_less_data(datasets['train'], args.proportion, 0)
        else:
            proportions = get_less_data(datasets['train'], 1, args.proportion)
            women = get_less_data(datasets['train'], 0, args.proportion)
            men = get_less_data(datasets['train'], 1, 0)

        datasets['train'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform,
                                                                     proportions=proportions, classes_to_demographic = classes_to_demographic)

        datasets['women'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform,
                                                                     proportions=women, classes_to_demographic = classes_to_demographic)

        datasets['men'] = ImageFolderWithProtectedAttributesModify(args.data_train_root, transform=train_transform,
                                                                   proportions=men, classes_to_demographic = classes_to_demographic)

        tlc = get_label_count(datasets['train'])
        mlc = get_label_count(datasets['men'])
        wlc = get_label_count(datasets['women'])

        print(len(tlc))
        print(len(mlc))
        print(len(wlc))

        for l in wlc.keys():
            for d in wlc[l].keys(): # the data d in the label l
                if d not in tlc[l].keys(): # check if this data d is in the tlc as well
                    print(l,d)

        for l in mlc.keys():
            for d in mlc[l].keys():  # the data d in the label l
                if d not in tlc[l].keys():  # check if this data d is in the tlc as well
                    print(l, d)

    print('New')
    print(len(datasets['train']))
    print(len(datasets['train'].classes))

    print_per_gender_count(datasets['train'])

    all_sample = datasets['train'].samples
    women_sample = datasets['women'].samples
    men_sample = datasets['men'].samples

    print('Check')
    print(len(all_sample))
    print(len(women_sample))
    print(len(men_sample))

    check = {}
    for inst in all_sample:
        path = inst[0]
        check[path] = 1

    print(len(check.keys()))

    k = 0
    for inst in women_sample:
        path = inst[0]
        if k == 0:
            print('Just check one', path)
        k+=1
        if path not in check.keys():
            print(inst)

    print(k)

    k = 0
    for inst in men_sample:
        path = inst[0]
        if k == 0:
            print('Just check one', path)
        k+=1
        if path not in check.keys():
            print(inst)

    print(k)

    dataloaders = {}
    train_imgs = datasets['train'].imgs
    weights_train = torch.DoubleTensor(balanced_weights(train_imgs, nclasses=len(datasets['train'].classes)))
    train_sampler = torch.utils.data.sampler.WeightedRandomSampler(weights_train, len(weights_train))
    num_class = len(datasets['train'].classes)
    dataloaders['train'] = torch.utils.data.DataLoader(datasets['train'], batch_size=args.batch_size,
                                                       sampler=train_sampler, num_workers=args.num_workers,
                                                       drop_last=True)

    dataloaders['naive_train'] = torch.utils.data.DataLoader(datasets['train'], batch_size=args.batch_size,
                                                             shuffle=True, num_workers=args.num_workers, drop_last=True)

    dataloaders['test'] = torch.utils.data.DataLoader(datasets['test'], batch_size=args.batch_size, shuffle=False,
                                                      num_workers=args.num_workers)

    if args.outer_loader == 'Women':
        print(args.outer_loader)
        dataloaders['outer'] = torch.utils.data.DataLoader(datasets['train'], batch_size=args.outer_batch_size,
                                                           sampler=WomenSampler(datasets['train']),
                                                           num_workers=args.num_workers, drop_last=True)
    elif args.outer_loader == 'Simple':
        dataloaders['outer'] = torch.utils.data.DataLoader(datasets['train'], batch_size=args.outer_batch_size,
                                                           num_workers=args.num_workers, drop_last=True)

    elif args.outer_loader == 'Both':
        print('Both')

        # orig_dataset = ImageFolderWithProtectedAttributes(args.data_train_root, train_transform)

        dataloaders['women'] = torch.utils.data.DataLoader(datasets['women'], batch_size=int(args.outer_batch_size / 2),
                                                           num_workers=args.num_workers, shuffle=True, drop_last=True)
        dataloaders['men'] = torch.utils.data.DataLoader(datasets['men'], batch_size=int(args.outer_batch_size / 2),
                                                         num_workers=args.num_workers, shuffle=True, drop_last=True)

        dataloaders['women_inner'] = torch.utils.data.DataLoader(datasets['women'], batch_size=int(args.outer_batch_size / 2),
                                                           num_workers=args.num_workers, shuffle=True, drop_last=True)
        dataloaders['men_inner'] = torch.utils.data.DataLoader(datasets['men'], batch_size=int(args.outer_batch_size / 2),
                                                         num_workers=args.num_workers, shuffle=True, drop_last=True)

    for k in dataloaders.keys():
        print('Len of {} dataloader is {}'.format(k, len(dataloaders[k])))

    return dataloaders, num_class


import cv2
import imutils
import numpy as np

# Ham sap xep contour tu trai sang phai

def get_contour_precedence(contour, cols):
    tolerance_factor = 10
    origin = cv2.boundingRect(contour)
    return ((origin[1] // tolerance_factor) * tolerance_factor) * cols + origin[0]

def sort_contours(cnts):
    cnts.sort(key=lambda x:get_contour_precedence(x, img.shape[1]))
    return cnts

# Ham fine tune bien so, loai bo cac ki tu khong hop ly
def fine_tune(lp):
    newString = ""
    for i in range(len(lp)):
        if lp[i] in char_list:
            newString += lp[i]
    return newString

# Dinh nghia cac ky tu tren bien so
char_list =  '0123456789ABCDEFGHKLMNPRSTUVXYZ'

# Param
max_size = 8000
min_size = 200

# Load image
img = cv2.imread('plate/0006_06797_b.jpg', cv2.IMREAD_COLOR)

# Cau hinh tham so cho model SVM
digit_w = 30 # Kich thuoc ki tu
digit_h = 60 # Kich thuoc ki tu

model_svm = cv2.ml.SVM_load('svm.xml')

# Resize image
img = cv2.resize(img, (470 , 300))

# Edge detection
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)  # convert to grey scale
# gray = cv2.GaussianBlur(gray,(5,5),0)
# gray = cv2.bilateralFilter(gray, 11, 17, 17)  # Blur to reduce noise
edged = cv2.Canny(gray, 30, 200)  # Perform Edge detection
# find contours in the edged image, keep only the largest
# ones, and initialize our screen contour
cv2.imshow("Gray image",edged )
cnts = cv2.findContours(edged.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)
screenCnt = None

# loop over our contours
for c in cnts:
    (x, y, w, h) = cv2.boundingRect(c)

    # approximate the contour
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.05 * peri, True)

    # if our approximated contour has four points, then
    # we can assume that we have found our screen

    if len(approx) == 4 and max_size > cv2.contourArea(c) > min_size:
        screenCnt = approx
        break
if screenCnt is None:
    detected = 0
    print ("No plate detected")
    cv2.waitKey(0)
    cv2.destroyAllWindows()
else:
    detected = 1

if detected == 1:
    cv2.drawContours(img, [screenCnt], -1, (0, 255, 0), 3)

    # Masking the part other than the number plate
    mask = np.zeros(gray.shape, np.uint8)
    new_image = cv2.drawContours(mask, [screenCnt], 0, 255, -1, )
    new_image = cv2.bitwise_and(img, img, mask=mask)

    # Now crop
    (x, y) = np.where(mask == 255)
    (topx, topy) = (np.min(x)+1, np.min(y)+1)
    (bottomx, bottomy) = (np.max(x)-1, np.max(y)-1)
    Cropped = gray[topx:bottomx + 1, topy:bottomy + 1]

    # Display image
    cv2.imshow('Input image', img)
    cv2.imshow('License plate', Cropped)

    copyCropped = Cropped.copy()
    # gray = cv2.GaussianBlur(copyCropped,(5,5),0)
    # gray = cv2.bilateralFilter(gray, 11, 17, 17)  # Blur to reduce noise
    copyCropped = cv2.GaussianBlur(copyCropped,(5,5),1)
    edged = cv2.Canny(copyCropped, 30, 200,apertureSize=3)
    lines = cv2.HoughLines(edged,1,np.pi/180,200)

    # for line in lines:
    #     for rho,theta in line:
    #         a = np.cos(theta)
    #         b = np.sin(theta)
    #         x0 = a*rho
    #         y0 = b*rho
    #         x1 = int(x0 + 1000*(-b))
    #         y1 = int(y0 + 1000*(a))
    #         x2 = int(x0 - 1000*(-b))
    #         y2 = int(y0 - 1000*(a))

    #         cv2.line(copyCropped,(x1,y1),(x2,y2),(0,0,255),2)
    print(lines)
    cv2.imshow('Line', edged)

    roi = Cropped

    # Ap dung threshold de phan tach so va nen
    binary = cv2.threshold(Cropped, 190, 255,
                         cv2.THRESH_BINARY_INV)[1]

    cv2.imshow("Anh bien so sau threshold", binary)
    # Xoay ảnh
    # cv2.houghLines(binary)

    # Segment kí tự
    # kernel3 = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
    # thre_mor = cv2.morphologyEx(binary, cv2.MORPH_DILATE, kernel3)
    # binary = cv2.dilate(binary,kernel3,iterations = 1)
    cont, _  = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    # cv2.imshow("Anh bien so sau bien doi ", thre_mor)
    plate_info = ""
    num = 0
    for c in sort_contours(cont):
        (x, y, w, h) = cv2.boundingRect(c)
        ratio = h/w
        if 1.5<=ratio<=5: # Chon cac contour dam bao ve ratio w/h
            if h/roi.shape[0]>=0.32 and h/roi.shape[0] <0.6 : # Chon cac contour cao tu 60% bien so tro len
                # Ve khung chu nhat quanh so

                cv2.rectangle(roi, (x, y), (x + w, y + h), (0, 255, 0), 2)

                # Tach so va predict
                curr_num = binary[y:y+h,x:x+w]
                curr_num = cv2.copyMakeBorder(curr_num,1,1,1,1,cv2.BORDER_CONSTANT,value=[0,0,0])
                curr_num = cv2.resize(curr_num, dsize=(digit_w, digit_h))
                num  = num +1
                cv2.imshow("Contour num " + str(num), curr_num)
                _, curr_num = cv2.threshold(curr_num, 30, 255, cv2.THRESH_BINARY)
                curr_num = np.array(curr_num,dtype=np.float32)
                curr_num = curr_num.reshape(-1, digit_w * digit_h)

                # Dua vao model SVM
                result = model_svm.predict(curr_num)[1]
                result = int(result[0, 0])

                if result<=9: # Neu la so thi hien thi luon
                    result = str(result)
                else: #Neu la chu thi chuyen bang ASCII
                    result = chr(result)

                plate_info +=result

    cv2.imshow("Cac contour tim duoc", roi)

    # Viet bien so len anh
    cv2.putText(img,fine_tune(plate_info),(50, 50), cv2.FONT_HERSHEY_PLAIN, 3.0, (0, 0, 255), lineType=cv2.LINE_AA)
    # Hien thi anh
    print("Bien so=", plate_info)
    cv2.imshow("Hinh anh output",img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()


import numpy as np
import os

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt

with open(os.path.join(os.getcwd(), 'clean', 'clean-antivirus-features.txt')) as f:
    all_features = f.readlines()

cosine_sim_scores = []

tfidf_vectorizer = TfidfVectorizer()
with open(os.path.join(os.getcwd(), 'requirement','antivirus-requirements.txt')) as f:
    for generated_feature in f.readlines()[:500]:
        appended_features = [generated_feature] + all_features
        tfidf_matrix = tfidf_vectorizer.fit_transform(appended_features)
        cosine_sim_scores.append(np.mean(cosine_similarity(tfidf_matrix[0:1], tfidf_matrix)))

X = np.arange(len(cosine_sim_scores))

# fig,ax = plt.subplots(1)

plt.figure(figsize=(8,4))
plt.bar(X, cosine_sim_scores, width=0.3, color='#4286f4')
plt.xlabel("generated requirement id")
plt.ylabel("cosine similarity td-idf score")
# plt.gca().axes.get_xaxis().set_visible(False)

cosine_sim_scores.sort()

plt.axhline(y=cosine_sim_scores[400], color='r', linestyle='-', linewidth=0.5)
plt.tight_layout()
plt.show()

print('Finished!')


#!/usr/bin/python3
# -*- coding: UTF-8 -*-
import codecs
import hashlib
import io
import shutil
import tempfile

from .libs.process import Process

try:
    from .config import Configuration
except (ValueError, ImportError) as e:
    raise Exception('You may need to run filecrawler from the root directory (which includes README.md)', e)

import sys, datetime, os
from filecrawler.libs.color import Color
from filecrawler.libs.logger import Logger
from .util.tools import Tools

class FileCrawler(object):

    def main(self):
        ''' Either performs action based on arguments, or starts attack scanning '''

        self.dependency_check()

        Configuration.initialize()

        if not Configuration.initialized:
            return

        self.run()

    def dependency_check(self):
        ''' Check that required programs are installed '''
        required_apps = []
        optional_apps = []
        missing_required = False
        missing_optional = False

        for app in required_apps:
            if not Process.exists(app):
                missing_required = True
                Color.pl('{!} {R}error: required app {O}%s{R} was not found' % app)

        for app in optional_apps:
            if not Process.exists(app):
                missing_optional = True
                Color.pl('{!} {O}warning: recommended app {R}%s{O} was not found' % app)

        if missing_required:
            Color.pl('{!} {R}required app(s) were not found, exiting.{W}')
            sys.exit(-1)

        if missing_optional:
            Color.pl('{!} {O}recommended app(s) were not found')
            Color.pl('{!} {O}filecrawler may not work as expected{W}')

    def run(self):

        try:

            timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            Logger.pl('{+} {C}start time {O}%s{W}' % timestamp)

            FileCrawler.load_tika()

            # Execute the specific actions
            Configuration.module.run()

        except Exception as e:
            Color.pl("\n{!} {R}Error: {O}%s" % str(e))
            if Configuration.verbose > 0 or True:
                Color.pl('\n{!} {O}Full stack trace below')
                from traceback import format_exc
                Color.p('\n{!}    ')
                err = format_exc().strip()
                err = err.replace('\n', '\n{W}{!} {W}   ')
                err = err.replace('  File', '{W}{D}File')
                err = err.replace('  Exception: ', '{R}Exception: {O}')
                Color.pl(err)
        except KeyboardInterrupt as e:
            raise e

        timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        Logger.pl('{+} {C}End time {O}%s{W}' % timestamp)
        Logger.pl(' ')

    def print_banner(self):
        """ Displays ASCII art of the highest caliber.  """
        Color.pl(Configuration.get_banner())

    @staticmethod
    def load_tika():
        from .libs.process import Process
        import logging

        if not Configuration.ocr_enabled:
            return

        try:
            tika_proc = Process.find_process('tika-server.jar')
            if tika_proc is not None:
                if Configuration.verbose >= 1:
                    Color.pl('{?} Killing tika server with PID: {G}%s{W}' % tika_proc[0])
                Process.kill(pid=tika_proc[0])

            if Configuration.verbose >= 1:
                Color.pl('{?} Loading tika: {GR}')

            p = tempfile.gettempdir()
            os.environ["TIKA_PATH"] = p
            os.environ["TIKA_LOG_PATH"] = p
            os.environ["TIKA_SERVER_JAR"] = 'tika-server.jar'
            jar_file = os.path.join(p, 'tika-server.jar')
            shutil.copytree(
                os.path.join(Configuration.lib_path, 'bin'),
                p,
                dirs_exist_ok=True)

            # create md5 hash file
            m = hashlib.md5()
            with open(jar_file, 'rb') as f:
                binContents = f.read()
                m.update(binContents)
                with open(jar_file + ".md5", "w") as em:
                    em.write(m.hexdigest())

            file = os.path.join(Configuration.lib_path, 'bin', 'loader.pdf')

            import tika
            from tika import parser

            #Change log level
            if Configuration.verbose == 0:
                log = logging.getLogger('tika.tika')
                log.setLevel(logging.ERROR)

            headers = {
                "X-Tika-OCRLanguage": f"eng+{Configuration.ocr_language}",
                "X-Tika-PDFocrStrategy": Configuration.ocr_pdf_strategy
            }
            parser.from_file(file, headers=headers)
            Color.pl('{+} Tika lib loaded {W}')
        finally:
            Color.p('{W}')

    # Used to supress libmagic error 'lhs/off overflow 4294967295 0'
    # https://bugs.astron.com/view.php?id=426
    #  This code will suppress any child process stderr output
    @staticmethod
    def redirect_stderr():
        new = os.dup(2)  # Create a copy of stderr (new)
        sys.stderr = io.TextIOWrapper(os.fdopen(new, 'wb'))
        _file = tempfile.TemporaryFile(mode='w+t')
        os.dup2(_file.fileno(), 2)  # Redirect stdout into tmp

def run():
    # Explicitly changing the stdout encoding format
    if sys.stdout.encoding is None:
        # Output is redirected to a file
        sys.stdout = codecs.getwriter('latin-1')(sys.stdout)

    FileCrawler.redirect_stderr()

    o = FileCrawler()
    o.print_banner()

    try:
        o.main()

    except Exception as e:
        Color.pl('\n{!} {R}Error:{O} %s{W}' % str(e))

        if Configuration.verbose > 0 or True:
            Color.pl('\n{!} {O}Full stack trace below')
            from traceback import format_exc
            Color.p('\n{!}    ')
            err = format_exc().strip()
            err = err.replace('\n', '\n{W}{!} {W}   ')
            err = err.replace('  File', '{W}{D}File')
            err = err.replace('  Exception: ', '{R}Exception: {O}')
            Color.pl(err)

        Color.pl('\n{!} {R}Exiting{W}\n')
        Tools.exit_gracefully(1)

    except KeyboardInterrupt:
        Color.pl('\n{!} {O}interrupted, shutting down...{W}')
        Tools.exit_gracefully(2)

    Tools.exit_gracefully(0)

{'format_exc': 'traceback.format_exc', 'Process': 'libs.process.Process', 'logging': 'logging', 'tika': 'tika', 'parser': 'tika.parser'}.redirect_stderr


# -*- coding: utf-8 -*-
"""
Created on Wed Apr  6 11:42:29 2022

@author: Fede
"""
from sklearn import tree, metrics
from sklearn.metrics import accuracy_score, precision_score, confusion_matrix
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sn
import joblib

# Cargo datos
df = pd.read_excel(r"C:\Users\fede_\OneDrive\Escritorio\clientes robo advisor2.xlsx")
df_datos = df.drop(["Resultado del perfil"], axis=1)
df_target = df["Resultado del perfil"]

# Invoco a la librería train_test_split para partir el dataset
X_train, X_test, Y_train, Y_test = train_test_split(df_datos, df_target,test_size=0.3, random_state=0)
print(X_train)

# Selecciono el algoritmo para clasificar los datos
clf = tree.DecisionTreeClassifier(max_depth=9, random_state=42)

# Procedo a entrenarlo
clf.fit(X_train, Y_train)

# Obtengo las predicciones
y_pred = clf.predict(X_test)

# Grafico
plt.figure(figsize=(15,10))
tree.plot_tree(clf)
plt.title("Arbol de decisión")
plt.savefig("Arbol de decisión.png")
plt.show()

# Matriz de confusion
matriz = confusion_matrix(Y_test, y_pred)
print('\n- Matriz de Confusión -')
sn.heatmap(matriz, annot=True)

# Metricas
print("Acuracy:", accuracy_score(Y_test, y_pred))
print("Precision:", precision_score(Y_test, y_pred,average='weighted'))
print("Recall" , metrics.recall_score(Y_test,y_pred,average='weighted'))
print(metrics.classification_report(Y_test, y_pred))
plt.show()

#Guardo el modelo
joblib.dump(clf, "arbol_roboAdvisor.pkl")


import os, torch, misc

def parse_arguments():
    import argparse
    parser = argparse.ArgumentParser(description='DeepSpeech training')
    # wav16000
    parser.add_argument('--raw_data_dir', default='raw_data', help='Path to read raw dataset')
    parser.add_argument('--data_16000_dir', default='wav_16000', help='Path to save dataset')
    parser.add_argument('--data_dir', default='data', help='Path to save dataset')
    parser.add_argument('--sample_rate', default=16000, type=int, help='Sample rate')
    # manifest
    parser.add_argument('--manifest_dir', default='manifest', help='Path to manifest')
    parser.add_argument('--eval_size', type=int, default=500, help='Size of held-out evaluation set')
    parser.add_argument('--test_size', type=int, default=500, help='Size of held-out test set')
    # data loader
    parser.add_argument('--batch_size', default=20, type=int, help='Batch size for training')
    parser.add_argument('--num_workers', default=4, type=int, help='Number of workers used in data-loading')
    parser.add_argument('--window_size', default=.02, type=float, help='Window size for spectrogram in seconds')
    parser.add_argument('--window_stride', default=.01, type=float, help='Window stride for spectrogram in seconds')
    # parser.add_argument('--crop_audio_length', default=20, type=float, help='Crop audios exceeding this length in seconds')
    # audio model
    parser.add_argument('--hidden_size', default=800, type=int, help='Hidden size of RNNs')
    parser.add_argument('--hidden_layers', default=5, type=int, help='Number of RNN layers')
    parser.add_argument('--num_classes', default=10, type=int, help='Number of audio classes')
    parser.add_argument('--model_path', default='models/best.pth', help='Location to save best validation model')
    # train
    parser.add_argument('--gpu', default=-1, type=int, help='The single device used in training, -1 means using CPU or ALL GPUs if avaiable.')
    parser.add_argument('--train_manifest', default='train.csv', help='path to train manifest csv')
    parser.add_argument('--eval_manifest', default='eval.csv', help='path to eval manifest csv')
    parser.add_argument('--test_manifest', default='test.csv', help='path to test manifest csv')
    parser.add_argument('--epochs', default=100, type=int, help='Number of training epochs')
    parser.add_argument('--lr', default=1e-2, type=float, help='initial learning rate')
    parser.add_argument('--momentum', default=0.9, type=float, help='momentum')
    parser.add_argument('--max_norm', default=400, type=int, help='Norm cutoff to prevent explosion of gradients')
    parser.add_argument('--learning_anneal', default=1.05, type=float, help='Annealing applied to learning rate every epoch')
    parser.add_argument('--logdir', default='log', type=str, help='Log folder')
    parser.add_argument('--continue_from', type=str, help='Continue from last training epoch')
    parser.add_argument('--plot', action='store_true', help='Plot training progress in terms of accuracies')
    # parse
    return parser.parse_args()

class Logger(object):

    def __init__(self, args, init_train_info={}, sub_dir=None):
        self.args = args
        misc.ensure_dir(args.logdir)
        sub_dir = args.continue_from or sub_dir or misc.datetimestr()
        self.logdir = os.path.join(args.logdir, sub_dir)
        misc.ensure_dir(self.logdir)
        self._setup_log_file()
        self._create_train_info(args, init_train_info)

    def _setup_log_file(self):
        import logging
        log_file = os.path.join(self.logdir, 'log.txt')
        self._logger = logging.getLogger()
        self._logger.setLevel('INFO')
        fh = logging.FileHandler(log_file)
        ch = logging.StreamHandler()
        self._logger.addHandler(fh)
        self._logger.addHandler(ch)

    def info(self, str_info):
        self._logger.info(str_info)

    # train_info

    def _create_train_info(self, args, init_train_info):
        import json
        train_info_path = os.path.join(self.logdir, 'train_info.txt')
        if os.path.isfile(train_info_path):
            self.train_info = json.loads(misc.load_file(train_info_path))
        else:
            self.train_info = init_train_info

    def save_train_info(self):
        import json
        train_info_path = os.path.join(self.logdir, 'train_info.txt')
        misc.save_file(train_info_path, json.dumps(self.train_info))

    def plot_progress(self):
        import plot
        pdf_plot_path = os.path.join(self.logdir, 'progress.pdf')
        x_data = torch.Tensor(range(1, 1 + len(self.train_info['train_acc'])))
        y_data = torch.Tensor([self.train_info['train_acc'], self.train_info['eval_acc']])
        # y_data, y_err = plot.smooth2d(y_data, step=3)
        plot.plot(x_data, y_data, y_err=None, legends=['Train', 'Eval'],
                title=None, xlabel='Epoch', ylabel='Acccuracy', filename=pdf_plot_path)

    # model

    def save_model(self, model, model_path=None):
        if isinstance(model, torch.nn.DataParallel):
            model = model.module
        if model_path is None:
            if 'model_path' in self.train_info and os.path.exists(os.join(self.logdir, self.train_info['model_path'])):
                print("Removing old model {}".format(self.train_info['model_path']))
                os.remove(os.join(self.logdir, self.train_info['model_path']))
            model_path = misc.datetimestr() + '.model.pth'
            self.train_info['model_path'] = model_path
        print("Saving model to {}".format(model_path))
        model.save(os.path.join(self.logdir, model_path))

    def create_model(self):
        import model, rnn
        if 'model_path' in self.train_info and os.path.exists(self.train_info['model_path']):
            model = model.AudioModel(rnn.BatchRNNLayers, config=None)
            model.load(self.train_info['model_path'])
        else:
            model = model.AudioModel(rnn.BatchRNNLayers, config=self.args.__dict__)
        if torch.cuda.is_available():
            if self.args.gpu == -1:
                model = torch.nn.DataParallel(model).cuda()
            else:
                model = model.cuda()
        return model

def run_epoch(model, data_loader, is_cuda, optimizer=None, max_norm=None):
    import time
    from tqdm import tqdm
    from torch.autograd import Variable

    is_train = optimizer is not None
    if is_train:
        model.train()
    else:
        model.eval()

    avg_loss = misc.Average()
    avg_acc = misc.Average()

    for i, (input, target) in tqdm(enumerate(data_loader), total=len(data_loader)):
        if is_train:
            input = Variable(input, requires_grad=False)
        else:
            input = Variable(input, volatile=True)
        if is_cuda:
            input = input.cuda()

        output = model(input)
        _, pred = output.data.cpu().max(1)
        batch_size = input.size(0)
        acc = (pred.view(-1) == target).sum() / float(batch_size)
        avg_acc.update(acc, batch_size)

        if is_train:
            target = Variable(target, requires_grad=False)
            if is_cuda:
                target = target.cuda()

            loss = torch.nn.functional.cross_entropy(output, target) / batch_size
            avg_loss.update(loss.data[0], batch_size)
            optimizer.zero_grad()
            loss.backward()
            if max_norm is not None:
                torch.nn.utils.clip_grad_norm(model.parameters(), max_norm)
            optimizer.step()

        if is_cuda:
            torch.cuda.synchronize()

    return avg_acc.avg, avg_loss.avg

def init_train_info():
    train_info = {}
    train_info['epoch'] = 0
    train_info['train_loss'] = []
    train_info['train_acc'] = []
    train_info['eval_acc'] = []
    train_info['test_acc'] = []
    train_info['best_eval_acc'] = 0
    train_info['best_test_acc'] = 0
    return train_info

def anneal_lr(epoch, args, optimizer, logger):
    if epoch > 1:
        optim_state = optimizer.state_dict()
        optim_state['param_groups'][0]['lr'] = optim_state['param_groups'][0]['lr'] / args.learning_anneal
        optimizer.load_state_dict(optim_state)
        logger.info('Learning rate annealed to: {lr:.6f}'.format(lr=optim_state['param_groups'][0]['lr']))

def train(args):
    is_cuda = torch.cuda.is_available()
    logger = Logger(args, init_train_info=init_train_info())

    model = logger.create_model()
    parameters = model.parameters()
    optimizer = torch.optim.SGD(parameters, lr=args.lr, momentum=args.momentum, nesterov=True)
    logger.info("Number of parameters: %d" % misc.get_param_size(model))

    import loader
    train_loader, _, _ = loader.get(args, os.path.join(args.manifest_dir, args.train_manifest))
    eval_loader, _, _ = loader.get(args, os.path.join(args.manifest_dir, args.eval_manifest))
    test_loader, _, _ = loader.get(args, os.path.join(args.manifest_dir, args.test_manifest))

    for epoch in range(args.epochs):
        anneal_lr(epoch + 1, args, optimizer, logger)
        if epoch < logger.train_info['epoch']: continue
        logger.train_info['epoch'] += 1

        model.train()
        acc, loss = run_epoch(model, train_loader, is_cuda, optimizer, args.max_norm)
        logger.info('Epoch: {}, train_acc: {}, train_loss: {}'.format(epoch + 1, acc, loss))
        logger.train_info['train_loss'].append(loss)
        logger.train_info['train_acc'].append(acc)

        model.eval()
        acc, _ = run_epoch(model, eval_loader, is_cuda)
        logger.info('Epoch: {}, eval_acc: {}'.format(epoch + 1, acc))
        logger.train_info['eval_acc'].append(acc)

        if acc > logger.train_info['best_eval_acc']:
            logger.train_info['best_eval_acc'] = acc
            acc, _ = run_epoch(model, test_loader, is_cuda)
            logger.info('Epoch: {}, test_acc: {}'.format(epoch + 1, acc))
            logger.train_info['test_acc'].append(acc)

            if acc > logger.train_info['best_test_acc']:
                print("Found better model (acc=%.5f) with test dataset." % acc)
                logger.train_info['best_test_acc'] = acc
                logger.save_model(model, 'best.model.pth')

        logger.save_train_info()
        logger.save_model(model)
        if args.plot:
            logger.plot_progress()

if __name__ == '__main__':
    args = parse_arguments()
    if args.gpu >= 0:
        misc.prepare_single_device(args)
    train(args)


import json
import os
from pathlib import Path

import pandas as pd
from typing import Dict
import matplotlib
import matplotlib.pyplot as plt

from drl.experiment.config import Config

class Analyzer:

    def __init__(self, config: Config, session_id):
        self.__config = config
        self.__session_id = session_id
        pass

    def play_analysis(self, path_to_experiment) -> Dict:

        # path = self.__config.get_app_experiments_path(train_mode=False)
        path = os.path.join(path_to_experiment, 'log.csv')

        df = pd.read_csv(path)

        analysis = {}
        analysis['number_of_episodes'] = df['episode'].max() + 1
        analysis['average_number_of_steps'] = df.groupby('episode').count()['step'].mean()
        analysis['average_reward_per_step'] = df.groupby('episode').mean()['reward'].mean()
        analysis['average_reward'] = df.groupby('episode').max()['reward_total'].sum()

        """

        :return:
            number_of_episodes
            average_number_of_steps
            average_reward
            average_number_of_actions
        """
        return analysis

    def compare_train_config(self, path_to_experiments):

        for path_to_experiment in path_to_experiments:
            path = os.path.join(path_to_experiment, 'config.json')

            with open(path, 'r') as fp:
                cfg = json.load(fp)

            print(path_to_experiment)
            print(json.dumps(cfg, indent=4, sort_keys=True))

        pass

    def compare_train_epoch_cols(self, path_to_experiments, compare_col, plot=False):
        max_x = 0

        for path_to_experiment in path_to_experiments:
            path = os.path.join(path_to_experiment, 'epoch-log.csv')

            df = pd.read_csv(path, index_col=0)
            if df.shape[0] > max_x:
                max_x = df.shape[0]

        df_main = pd.DataFrame(list(range(max_x)))

        for path_to_experiment in path_to_experiments:
            path = os.path.join(path_to_experiment, 'epoch-log.csv')

            df_tmp = pd.read_csv(path, index_col=0)

            df_tmp = df_tmp[[compare_col]]

            df_tmp.columns = [(path_to_experiment.rsplit('/', 1)[1])]

            df_main = df_main.join(df_tmp)

        df_main = df_main[df_main.columns.difference([0])]

        ax = df_main.plot()

        ax.set_xlabel('epoch')
        ax.set_ylabel(compare_col)

        from matplotlib import pyplot
        fig = ax.get_figure()

        path = os.path.join(self.__config.get_app_analysis_path(), self.__session_id)
        Path(path).mkdir(parents=True, exist_ok=True)

        path = os.path.join(path, compare_col +'.png' )

        if plot:
            plt.show()
            return None
        else:
            matplotlib.use('Agg')
            fig.savefig(path)
            return path

    def compare_train_epoch_score(self, path_to_experiment, plot=False):

        path = os.path.join(path_to_experiment, 'epoch-log.csv')

        df_tmp = pd.read_csv(path, index_col=0)

        df_tmp = df_tmp[['avg_score', 'avg_val_score']]

        ax = df_tmp.plot()

        ax.set_xlabel('epoch')
        ax.set_ylabel('score')

        from matplotlib import pyplot
        fig = ax.get_figure()

        path = os.path.join(self.__config.get_app_analysis_path(), self.__session_id)
        Path(path).mkdir(parents=True, exist_ok=True)

        path = os.path.join(path, 'score.png' )

        if plot:
            plt.show()
            return None
        else:
            matplotlib.use('Agg')
            fig.savefig(path)
            return path

    def log_analysis(self, analysis: Dict):
        for key in analysis.keys():
            print("{}: {}".format(key, analysis[key]))


from django.db import models
from django.core.cache import cache
from django.utils import timezone

from django_redis import get_redis_connection

import random
import requests

import json
import string

class Leaderboards(object):
    """
    Get page leaderboards
    """
    @staticmethod
    def get_all_time():
        pages = cache.get("alltime_leaderboards")

        if pages is None:
            pages = Page.objects.filter(saved=True).order_by(
                "-score").only("title", "char_id", "score", "datetime")[0:10].values()
            cache.set("alltime_leaderboards", pages, 300)

        return pages

    @staticmethod
    def get_last_month():
        pages = cache.get("last_month_leaderboards")

        if pages is None:
            month_ago = timezone.now() - timezone.timedelta(days=30)
            pages = Page.objects.filter(saved=True,
                                        datetime__gte=month_ago).order_by("-score").only("title", "char_id", "score", "datetime")[0:10].values()
            cache.set("last_month_leaderboards", pages, 300)

        return pages

    @staticmethod
    def get_last_week():
        pages = cache.get("last_week_leaderboards")

        if pages is None:
            week_ago = timezone.now() - timezone.timedelta(days=7)
            pages = Page.objects.filter(saved=True,
                                        datetime__gte=week_ago).order_by("-score").only("title", "char_id", "score", "datetime")[0:10].values()
            cache.set("last_week_leaderboards", pages, 300)

        return pages

    @staticmethod
    def get_last_day():
        pages = cache.get("last_day_leaderboards")

        if pages is None:
            day_ago = timezone.now() - timezone.timedelta(hours=24)
            pages = Page.objects.filter(saved=True,
                                        datetime__gte=day_ago).order_by("-score").only("title", "char_id", "score", "datetime")[0:10].values()
            cache.set("last_day_leaderboards", pages, 60)

        return pages

class ComicImage(object):
    """
    Get a random image from the comic
    """
    @staticmethod
    def get_random_image():
        con = get_redis_connection("persistent")

        image_no = con.srandmember("panel_images")

        if image_no is None:
            raise Exception("No image found! Are you sure you've configured Redis correctly and run 'update_images' command?")

        image_file_no = "%d" % int(image_no)

        chars_to_add = 5 - len(image_file_no)

        for i in range(0, chars_to_add):
            image_file_no = "0%s" % image_file_no

        return image_file_no

# Character names corresponding to Markov chains that exist for each character
CHARACTER_NAMES = [
    'SOLLUX', 'PAA', 'PTA', 'PAG', 'CCG', 'CCC', 'NEPETA', 'FGA', 'TEREZI',
    'GT', 'GG', 'GC', 'GA', 'FAC', 'DIRK', 'ARADIABOT', 'ROSE', 'ARQUIUSPRITE',
    'FAG', 'uu', 'EQUIUS', 'ARADIASPRITE', 'ROXY', 'AC', 'PCG3', 'MEENAH',
    'JAKE', 'GAMZEE', 'PCA', 'PCG', 'CEB', 'JASPROSESPRITE^2', 'FCG2', 'PTC',
    'VRISKA', 'KANAYA?', 'EB', 'KARKAT', 'FTC', 'UU', 'KANAYA',
    'DAVEPETASPRITE^2', 'ARADIA', '?CG', 'JADE', 'DAVE', 'CTG', 'CTA',
    'ARANEA', 'FCT', 'CGC', 'FCA', 'FCG', 'FGC', 'JOHN', 'TAVROS', 'CC',
    'CA', 'CG', 'ROSESPRITE', 'CT', 'PCG8', 'PCG6', 'PCG7', 'PCG4', 'PCG5',
    'PCG2', 'FEFERI', 'CALLIOPE', 'DAVESPRITE', '?TG', 'PCC', 'CAC', 'CAA',
    'CAG', 'PGC', 'TT', 'TG', 'FAA', 'TC', 'TA', 'AA', 'AG', 'AT', 'ERIDAN',
    'TAVROSPRITE', 'JANE'
]

class Page(models.Model):
    """
    A randomly generated page
    """
    char_id = models.CharField(max_length=8, db_index=True, unique=True)

    title = models.TextField()

    image = models.TextField()

    # Any text before the dialoglog (if any)
    pre_dialog_text = models.TextField(default="")

    # Any dialoglog entries (if any)
    dialoglog = models.TextField(default="[]")

    # Any text after the dialoglog (if any)
    post_dialog_text = models.TextField(default="")

    # When generated
    datetime = models.DateTimeField(auto_now_add=True, db_index=True)

    # Has this page been saved
    # If not, it will automatically be deleted after a short while (about 24
    # hours)
    saved = models.BooleanField(default=False)
    score = models.IntegerField(default=0, db_index=True)

    @staticmethod
    def generate_page():
        """
        Generate a random page
        """
        page = Page()

        # Get image
        page.image = ComicImage.get_random_image()

        json_request = {}

        if random.randint(1, 10) >= 7:
            # Generate pre dialog text
            json_request["generate_pre_dialog"] = True

        json_request["characters"] = []

        if random.randint(1, 10) >= 5:
            # How many characters will there be in the conversation
            characters_in_conv = random.randint(1, 3)
            json_request["conversation_length"] = random.randint(1, 4)

            json_request["characters"] = random.sample(
                CHARACTER_NAMES, characters_in_conv
            )

        json_request["characters"] = json.dumps(json_request["characters"])

        if random.randint(1, 10) >= 6:
            # Generate post dialog text
            json_request["generate_post_dialog"] = True

        response = requests.post("http://127.0.0.1:5666/", data=json_request)
        response = json.loads(response.text)

        page.char_id = ''.join(random.SystemRandom().choice(
            string.ascii_letters + string.digits) for _ in range(8)
        )

        page.title = response["title"]

        if "dialoglog" in response:
            page.dialoglog = json.dumps(response["dialoglog"])

        if "pre_dialog_text" in response:
            page.pre_dialog_text = response["pre_dialog_text"]

        if "post_dialog_text" in response:
            page.post_dialog_text = response["post_dialog_text"]

        page.save()

        return page


#coding : utf8
#Author : taosenlin
#Time : 2020/5/25 14:22

from selenium import webdriver
import unittest
import time

class Testbaidu(unittest.TestCase):

    def setUp(self,url = 'https://www.baidu.com'):
        self.driver = webdriver.Chrome("E:\chromedriver.exe")
        self.url = url

    def tearDown(self):
        self.driver.quit()

    def test_baidu(self):
        # url = 'https://www.baidu.com'
        driver = self.driver
        driver.get(self.url)
        driver.find_element_by_id('kw').clear()
        driver.find_element_by_id('kw').send_keys('unittest')
        driver.find_element_by_id('su').click()
        time.sleep(3)
        title = driver.title
        self.assertEqual(title,'unittest_百度搜索')

if __name__ == '__main__':
    unittest.main()


import os
import time
import torch
import math
import cv2
import numpy as np
from PIL import Image
import scipy.io as sio
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from scipy.signal import convolve2d
from torch.nn import functional as F
from scipy.ndimage import measurements, interpolation
from utils.imresize import imresize

def preprocess_kernels(kernels, conf):
    # Load kernels if given files. if not just use the downscaling method from the configs.
    # output is a list of kernel-arrays or a a list of strings indicating downscaling method.
    # In case of arrays, we shift the kernels (see next function for explanation why).
    # Kernel is a .mat file (MATLAB) containing a variable called 'Kernel' which is a 2-dim matrix.
    if kernels is not None:
        return [kernel_shift(loadmat(kernel)['Kernel'], sf)
                for kernel, sf in zip(kernels, conf.scale_factors)]
    else:
        return [conf.downscale_method] * len(conf.scale_factors)

def kernel_shift(kernel, sf):
    # There are two reasons for shifting the kernel:
    # 1. Center of mass is not in the center of the kernel which creates ambiguity. There is no possible way to know
    #    the degradation process included shifting so we always assume center of mass is center of the kernel.
    # 2. We further shift kernel center so that top left result pixel corresponds to the middle of the sfXsf first
    #    pixels. Default is for odd size to be in the middle of the first pixel and for even sized kernel to be at the
    #    top left corner of the first pixel. that is why different shift size needed between odd and even size.
    # Given that these two conditions are fulfilled, we are happy and aligned, the way to test it is as follows:
    # The input image, when interpolated (regular bicubic) is exactly aligned with ground truth.

    # First calculate the current center of mass for the kernel
    current_center_of_mass = measurements.center_of_mass(kernel)

    # The second term ("+ 0.5 * ....") is for applying condition 2 from the comments above
    wanted_center_of_mass = np.array(kernel.shape) / 2 + 0.5 * (np.array(sf) - (np.array(kernel.shape) % 2))

    # Define the shift vector for the kernel shifting (x,y)
    shift_vec = wanted_center_of_mass - current_center_of_mass

    # Before applying the shift, we first pad the kernel so that nothing is lost due to the shift
    # (biggest shift among dims + 1 for safety)
    kernel = np.pad(kernel, np.int(np.ceil(np.max(np.abs(shift_vec)))) + 1, 'constant')

    # Finally shift the kernel and return
    return interpolation.shift(kernel, shift_vec)

def father_to_son(hr_father, kernel, sf):
    # Create son out of the father by downscaling and if indicated adding noise
    lr_son = imresize(hr_father, 1.0 / sf, kernel=kernel)
    return np.clip(lr_son, 0, 1)

def read_image(path):
    """Loads an image"""
    im = Image.open(path).convert('RGB')
    im = np.array(im, dtype=np.uint8)
    im = np.double(im) / 255.
    return im

def write_image(img, path):
    print('[*] img in write image', img.shape)
    print('[*] img squeezed ', img.dtype)
    if np.max(img) <= 1 :
        img = np.uint8(np.clip(np.round(img*255.), 0., 255.))
    im = Image.fromarray(img)
    im.save(path)

def create_gradient_map(im, window=5, percent=.97):
    """Create a gradient map of the image blurred with a rect of size window and clips extreme values"""
    # Calculate gradients
    gx, gy = np.gradient(rgb2gray(im))
    # Calculate gradient magnitude
    gmag, gx, gy = np.sqrt(gx ** 2 + gy ** 2), np.abs(gx), np.abs(gy)
    # Pad edges to avoid artifacts in the edge of the image
    gx_pad, gy_pad, gmag = pad_edges(gx, int(window)), pad_edges(gy, int(window)), pad_edges(gmag, int(window))
    lm_x, lm_y, lm_gmag = clip_extreme(gx_pad, percent), clip_extreme(gy_pad, percent), clip_extreme(gmag, percent)
    # Sum both gradient maps
    grads_comb = lm_x / lm_x.sum() + lm_y / lm_y.sum() + gmag / gmag.sum()
    # Blur the gradients and normalize to original values
    loss_map = convolve2d(grads_comb, np.ones(shape=(window, window)), 'same') / (window ** 2)
    # Normalizing: sum of map = numel
    return loss_map / np.mean(loss_map)

def create_probability_map(loss_map, crop):
    """Create a vector of probabilities corresponding to the loss map"""
    # Blur the gradients to get the sum of gradients in the crop
    blurred = convolve2d(loss_map, np.ones([crop // 2, crop // 2]), 'same') / ((crop // 2) ** 2)
    # Zero pad s.t. probabilities are NNZ only in valid crop centers
    prob_map = pad_edges(blurred, crop // 2)
    # Normalize to sum to 1
    prob_vec = prob_map.flatten() / prob_map.sum() if prob_map.sum() != 0 else np.ones_like(prob_map.flatten()) / \
                                                                               prob_map.flatten().shape[0]
    return prob_vec

def move2cpu(d):
    """Move data from gpu to cpu"""
    return d.detach().cpu().float().numpy()

def im2tensor(im_np):
    """Copy the image to the gpu & converts to range [-1,1]"""
    im_np = im_np / 255.0 if im_np.dtype == 'uint8' else im_np
    return torch.FloatTensor(np.transpose(im_np, (2, 0, 1)) * 2.0 - 1.0).unsqueeze(0).cuda()

def tensor2im(im_t):
    """Copy the tensor to the cpu & convert to range [0,255]"""
    im_np = np.clip(np.round((np.transpose(move2cpu(im_t).squeeze(0), (1, 2, 0)) + 1) / 2.0 * 255.0), 0, 255)
    return im_np.astype(np.uint8)

def rgb2gray(im):
    """Convert and RGB image to gray-scale"""
    return np.dot(im, [0.299, 0.587, 0.114]) if len(im.shape) == 3 else im

def pad_edges(im, edge):
    """Replace image boundaries with 0 without changing the size"""
    zero_padded = np.zeros_like(im)
    zero_padded[edge:-edge, edge:-edge] = im[edge:-edge, edge:-edge]
    return zero_padded

def clip_extreme(im, percent):
    """Zeroize values below the a threshold and clip all those above"""
    # Sort the image
    im_sorted = np.sort(im.flatten())
    # Choose a pivot index that holds the min value to be clipped
    pivot = int(percent * len(im_sorted))
    v_min = im_sorted[pivot]
    # max value will be the next value in the sorted array. if it is equal to the min, a threshold will be added
    v_max = im_sorted[pivot + 1] if im_sorted[pivot + 1] > v_min else v_min + 10e-6
    # Clip an zeroize all the lower values
    return np.clip(im, v_min, v_max) - v_min


import numpy as np
from gym.envs.box2d import LunarLander
from gym.envs.box2d.lunar_lander import ContactDetector, VIEWPORT_W, SCALE, VIEWPORT_H, INITIAL_RANDOM, LANDER_POLY, \
    LEG_AWAY, LEG_W, LEG_H, LEG_DOWN, LEG_SPRING_TORQUE
from gym.vector.utils import spaces

import Box2D
from Box2D.b2 import (edgeShape, circleShape, fixtureDef, polygonShape, revoluteJointDef, contactListener)

INITIAL_RANDOM = INITIAL_RANDOM

class LunarLanderRandomInit(LunarLander):
    smooth_random = True

    def reset(self):
        self._destroy()
        self.world.contactListener_keepref = ContactDetector(self)
        self.world.contactListener = self.world.contactListener_keepref
        self.game_over = False
        self.prev_shaping = None

        W = VIEWPORT_W / SCALE
        H = VIEWPORT_H / SCALE

        # terrain
        CHUNKS = 11
        height = self.np_random.uniform(0, H / 2, size=(CHUNKS + 1,))
        chunk_x = [W / (CHUNKS - 1) * i for i in range(CHUNKS)]
        self.helipad_x1 = chunk_x[CHUNKS // 2 - 1]
        self.helipad_x2 = chunk_x[CHUNKS // 2 + 1]
        self.helipad_y = H / 4
        height[CHUNKS // 2 - 2] = self.helipad_y
        height[CHUNKS // 2 - 1] = self.helipad_y
        height[CHUNKS // 2 + 0] = self.helipad_y
        height[CHUNKS // 2 + 1] = self.helipad_y
        height[CHUNKS // 2 + 2] = self.helipad_y
        smooth_y = [0.33 * (height[i - 1] + height[i + 0] + height[i + 1]) for i in range(CHUNKS)]

        self.moon = self.world.CreateStaticBody(shapes=edgeShape(vertices=[(0, 0), (W, 0)]))
        self.sky_polys = []
        for i in range(CHUNKS - 1):
            p1 = (chunk_x[i], smooth_y[i])
            p2 = (chunk_x[i + 1], smooth_y[i + 1])
            self.moon.CreateEdgeFixture(
                vertices=[p1, p2],
                density=0,
                friction=0.1)
            self.sky_polys.append([p1, p2, (p2[0], H), (p1[0], H)])

        self.moon.color1 = (0.0, 0.0, 0.0)
        self.moon.color2 = (0.0, 0.0, 0.0)

        if self.smooth_random:
            initial_x = np.clip(self.np_random.normal(loc=W / 2., scale=W / 3.),
                                a_min=0., a_max=W)
            initial_y = np.clip(self.np_random.normal(loc=H - (H - self.helipad_y) / 3., scale=self.helipad_y / 3.),
                                a_min=self.helipad_y, a_max=H)
            initial_angle = self.np_random.normal(loc=0., scale=np.pi / 3.)
        else:
            initial_x = self.np_random.uniform(0., W)
            initial_y = self.np_random.uniform(self.helipad_y, H)
            initial_angle = self.np_random.uniform(-np.pi / 2, np.pi / 2)

        self.lander = self.world.CreateDynamicBody(
            position=(initial_x, initial_y),
            angle=initial_angle,
            fixtures=fixtureDef(
                shape=polygonShape(vertices=[(x / SCALE, y / SCALE) for x, y in LANDER_POLY]),
                density=5.0,
                friction=0.1,
                categoryBits=0x0010,
                maskBits=0x001,  # collide only with ground
                restitution=0.0)  # 0.99 bouncy
        )
        self.lander.color1 = (0.5, 0.4, 0.9)
        self.lander.color2 = (0.3, 0.3, 0.5)
        self.lander.ApplyForceToCenter((
            self.np_random.uniform(-INITIAL_RANDOM, INITIAL_RANDOM),
            self.np_random.uniform(-INITIAL_RANDOM, INITIAL_RANDOM)
        ), True)

        self.legs = []
        for i in [-1, +1]:
            leg = self.world.CreateDynamicBody(
                position=(initial_x - i * LEG_AWAY / SCALE, initial_y),
                angle=(i * 0.05 + initial_angle),
                fixtures=fixtureDef(
                    shape=polygonShape(box=(LEG_W / SCALE, LEG_H / SCALE)),
                    density=1.0,
                    restitution=0.0,
                    categoryBits=0x0020,
                    maskBits=0x001)
            )
            leg.ground_contact = False
            leg.color1 = (0.5, 0.4, 0.9)
            leg.color2 = (0.3, 0.3, 0.5)
            rjd = revoluteJointDef(
                bodyA=self.lander,
                bodyB=leg,
                localAnchorA=(0, 0),
                localAnchorB=(i * LEG_AWAY / SCALE, LEG_DOWN / SCALE),
                enableMotor=True,
                enableLimit=True,
                maxMotorTorque=LEG_SPRING_TORQUE,
                motorSpeed=+0.3 * i  # low enough not to jump back into the sky
            )
            if i == -1:
                rjd.lowerAngle = +0.9 - 0.5  # The most esoteric numbers here, angled legs have freedom to travel within
                rjd.upperAngle = +0.9
            else:
                rjd.lowerAngle = -0.9
                rjd.upperAngle = -0.9 + 0.5
            leg.joint = self.world.CreateJoint(rjd)
            self.legs.append(leg)

        self.drawlist = [self.lander] + self.legs

        return self.step(np.array([0, 0]) if self.continuous else 0)[0]

class LunarLanderContinuousRandomInit(LunarLanderRandomInit):
    continuous = True

class LunarLanderNoRewardShaping(LunarLander):

    def step(self, action):
        prev_shaping = self.prev_shaping
        state, reward, done, d = super().step(action)
        shaping = self.prev_shaping
        if prev_shaping is not None:
            reward += prev_shaping - shaping
        return state, reward, done, d

class LunarLanderContinuousNoRewardShaping(LunarLanderNoRewardShaping):
    continuous = True

class LunarLanderRewardShapingAugmented(LunarLander):

    def __init__(self):
        super().__init__()
        self.observation_space = spaces.Box(-np.inf, np.inf, shape=(9,), dtype=np.float32)

    def step(self, action):
        state, reward, done, d = super().step(action)
        state = np.append(state, [self.prev_shaping * 1e-2], axis=-1)
        return state, reward, done, d

class LunarLanderContinuousRewardShapingAugmented(LunarLanderRewardShapingAugmented):
    continuous = True

class LunarLanderRandomInitRewardShapingAugmented(LunarLanderRandomInit, LunarLanderRewardShapingAugmented):
    def reset(self):
        return LunarLanderRandomInit.reset(self)

    def step(self, action):
        return super().step(action)

class LunarLanderRandomInitNoRewardShaping(LunarLanderRandomInit, LunarLanderNoRewardShaping):
    def reset(self):
        return LunarLanderRandomInit.reset(self)

    def step(self, action):
        return super().step(action)

class LunarLanderContinuousRandomInitNoRewardShaping(LunarLanderRandomInitNoRewardShaping):
    continuous = True

class LunarLanderContinuousRandomInitRewardShapingAugmented(LunarLanderRandomInitRewardShapingAugmented):
    continuous = True


import rclpy
from rclpy.node import Node
from std_msgs.msg import String
import os
import speech_recognition as sr
from gtts import gTTS
import threading
from dexhand_llm_control.openai_session import OpenAISession
import json

class LLMControlNode(Node):
    def __init__(self):
        super().__init__('llm_control')

        # Retreive the OpenAI key from the environment
        openai_key = os.environ.get('OPENAI_API_KEY')
        if openai_key is None:
            self.get_logger().error('OPENAI_API_KEY environment variable not set. Cannot continue.')
            exit(1)
        self.openai_session = OpenAISession(openai_key, self.get_logger())

        # Default publishers for hand
        self.extension_publisher = self.create_publisher(String, 'dexhand_finger_extension', 10)
        self.gesture_publisher = self.create_publisher(String, 'dexhand_gesture', 10)

        # Set up speech recognition
        self.recognizer = sr.Recognizer()

        # List the microphones - see if we can find a Respeaker
        self.microphone_index = 0  # Default to built in microphone
        for index, name in enumerate(sr.Microphone.list_microphone_names()):
            self.get_logger().debug("Microphone with name \"{1}\" found for `Microphone(device_index={0})`".format(index, name))
            if (name.lower().find("respeaker") != -1):
                self.microphone_index = index
                self.get_logger().info("Respeaker detected - Using microphone with name \"{1}\" for `Microphone(device_index={0})`".format(index, name))
                break

        self.get_logger().info("Using microphone with index {0} named {1}".format(self.microphone_index, sr.Microphone.list_microphone_names()[self.microphone_index]))
        self.microphone = sr.Microphone(device_index=self.microphone_index)
        self.recognizer.energy_threshold = 4000
        with self.microphone as source:
            self.recognizer.adjust_for_ambient_noise(self.microphone)

        # Start audio processing loop in a thread
        self.audio_thread = threading.Thread(target=self.message_processing_loop)
        self.audio_thread.start()

    # Basic wrapper for Google Text to Speech. Takes a string, speaks it, and then deletes the file.
    def textToSpeech(self, text):
        tts = gTTS(text=text, lang='en')
        tts.save("speech.mp3")
        os.system("mpg123 speech.mp3")
        os.remove("speech.mp3")

    # Takes the JSON response from OpenAI and formats it into a command for the hand
    def formatROSMessages(self, jsonmessage):

        # Publish a dexhand_gesture message to return to base pose
        msg = String()
        msg.data = "reset"
        self.gesture_publisher.publish(msg)

        # Is this a gesture or a finger message?
        if "gesture" in jsonmessage:
            msg = String()
            msg.data = jsonmessage["gesture"]
            self.gesture_publisher.publish(msg)

            self._logger.debug("Published dexhand_gesture message: {0}".format(msg.data))
        else:
            # Convert each finger to a dexhand_finger_extension message and publish it
            for finger, value in jsonmessage.items():
                msg = String()
                msg.data = "{0}:{1}".format(finger, value)
                self.extension_publisher.publish(msg)

                self._logger.debug("Published dexhand_finger_extension message: {0}".format(msg.data))

    # This is our main message processor that records audio from the user and
    # attempts to process it with OpenAI. It runs in a thread.
    def message_processing_loop(self):
        while rclpy.ok():
            with self.microphone as source:
                self.get_logger().info('Ready for audio input')
                audio = self.recognizer.listen(source)

                try:
                    # for testing purposes, we're just using the default API key
                    # to use another API key, use `r.recognize_google(audio, key="GOOGLE_SPEECH_RECOGNITION_API_KEY")`
                    # instead of `r.recognize_google(audio)`
                    query = self.recognizer.recognize_google(audio)

                    # We have a few special commands that we handle here without needing AI

                    # Reset the hand
                    if query == "reset hand":
                        self.get_logger().info('Resetting hand')
                        msg = String()
                        msg.data = "reset"
                        self.gesture_publisher.publish(msg)
                        continue

                    # Reset the context
                    if query == "reset context":
                        self.get_logger().info('Resetting context')
                        self.openai_session.resetContext()
                        msg = String()
                        msg.data = "reset"
                        self.gesture_publisher.publish(msg)

                        continue

                    # Send the query to OpenAI
                    jsonmessage,textmessage = self.openai_session.processPrompt(query)

                    # If the we received a jsonmessage, we format that into a command and send it to the hand
                    if jsonmessage is not None:
                        self.get_logger().info('Sending to DexHand: "%s"' % jsonmessage)
                        self.formatROSMessages(json.loads(jsonmessage))

                    # If we received a text message, we just speak it
                    if textmessage is not None:
                        self.get_logger().info('Speaking to user: "%s"' % textmessage)
                        self.textToSpeech(textmessage)

                except sr.UnknownValueError:
                    print("Google Speech Recognition could not understand audio")
                    #self.textToSpeech("I'm sorry, I didn't understand that. Try again.")

                except sr.RequestError as e:
                    print("Could not request results from Google Speech Recognition service; {0}".format(e))
                    self.textToSpeech("I'm sorry, there is an error with Google Speech Recognition.")

def main(args=None):
    rclpy.init(args=args)
    dexhand_node = LLMControlNode()
    rclpy.spin(dexhand_node)
    dexhand_node.destroy_node()
    rclpy.shutdown()

if __name__ == '__main__':
    main()


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import dash_bootstrap_components as dbc
import pandas as pd
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA
import plotly.express as px

app = dash.Dash(__name__, external_stylesheets=[dbc.themes.DARKLY])

app.layout = dbc.Container([
	dbc.Row(
		dbc.Col([
			html.Div(
				dbc.RadioItems(
					options = [
						{'label':'tsne only', 'value':1},
						{'label':'tsne + pca', 'value':2}
					],
					value = 1,
					id = 'radio',
					className = 'btn-group',
					labelClassName = 'btn btn-secondary',
					labelCheckedClassName = 'active',
				),
				className = 'radio-group',
				style = {'text-align':'center'}
			),
			dcc.Graph(id="scatter-plot"),
			html.Div(["PCA n_components: ", html.Div(id='display3')]),
			dcc.Slider(
				id='pca_comp',
				min=0, max=47, step=1,
				marks={a*10:str(a*10) for a in range(5)},
				value=20
			),
			html.Div(["Perplexity: ", html.Div(id='display')]),
			dcc.Slider(
				id='perplexity',
				min=0, max=100, step=1,
				marks={a*10:str(a*10) for a in range(11)},
				value=1
			),
			html.Div(["No of iteratons: ", html.Div(id='display2')]),
			dcc.Slider(
				id='iters',
				min=250, max=5000, step=50,
				marks={a*250:str(a*250) for a in range(4,21)},
				value=300
			),
		],
		width=12
		)
	)
])

df = pd.read_csv('../data/model_vus_preprocd.csv')
y=[]
for i in df.index:
	if pd.notna(df.loc[i,'gene']):
		y.append('Important Gene Mutation')
		continue
	if df.loc[i,'outcome'] == 0:
		y.append('No Mutation')
	else:
		y.append('VUS')
df = df.drop(['Sno','outcome','gene','Syndrome', 'gene name if VUS','Ethnicity  0-delhi NCR, west UP, haryana  1-eastern UP, Bihar 2- pahari 3-rajasthan 4-punjab 5-miscellanous'], axis=1)

@app.callback(
	Output("scatter-plot", "figure"),
	[Input("perplexity", "value"), Input('iters', 'value'), Input('pca_comp', 'value'), Input('radio', 'value')]
)
def update_bar_chart(slider_val, itersval, pcacomps, radio):
	pca_res = df
	if radio == 2:
		pca = PCA(n_components=pcacomps)
		pca_res = pca.fit_transform(df)
	tsne = TSNE(n_components=2, verbose=0, perplexity=slider_val, n_iter=itersval)
	tsne_results = tsne.fit_transform(pca_res)
	df['tsne-2d-one'] = tsne_results[:,0]
	df['tsne-2d-two'] = tsne_results[:,1]
	fig = px.scatter(df, x="tsne-2d-one", y="tsne-2d-two", template='plotly_dark', color=y)
	fig.layout.height=700
	return fig

@app.callback(
	Output('display', 'children'),
	[Input('perplexity', 'value')]
)
def update_display(val):
	return val

@app.callback(
	Output('display2', 'children'),
	[Input('iters', 'value')]
)
def update_display2(val):
	return val

app.run_server(debug=True)


from automaton import automata

class gramatica:
    def __init__(self, parts:list) -> None:
        self.name = parts.pop(0)
        tmp = parts.pop(0).split(";")
        self.Vn = tmp[0].split(",")
        self.Vt = tmp[1].split(",")
        self.Sn = tmp[2]
        self.glc = False
        self.Pr = []
        while len(parts) > 0:
            prod = parts.pop(0).split("->")
            prod[0] = prod[0].strip()
            prod[1] = prod[1].strip()
            self.Pr.append(prod)
            self.isGlc(prod)
        self.auto:automata = None

    def isGlc(self,prod):
        left = prod[0].split()
        numL = 0
        for e in left:
            for nt in self.Vn:
                if e == nt:
                    numL += 1
        right = prod[1].split()
        rNt = 0
        for e in right:
            for nt in self.Vn:
                if e == nt:
                    rNt += 1
        rT = 0
        for e in right:
            for t in self.Vt:
                if e == t:
                    rT += 1
        if numL == 1:
            if rNt > 1 or rT > 1:
                self.glc = True

    def showGrammar(self):
        print("\n------------------------------------------\n")
        print("Nombre de gramatica =",self.name)
        str = "{"
        for nt in self.Vn:
            str += nt+","
        str = str.removesuffix(",")
        str += "}"
        print("No Terminales =",str)
        str = "{"
        for t in self.Vt:
            str += t+","
        str = str.removesuffix(",")
        str += "}"
        print("Terminales =",str)
        print("No terminal inicial =",self.Sn)
        print("Producciones =\n")
        groups = []
        for prod in self.Pr:
            for g in groups:
                if g[0] == prod[0]:
                    g.append(prod[1])
                    break
            else:
                group = [prod[0],prod[1]]
                groups.append(group)
        for g in groups:
            if len(g) > 2:
                str = g.pop(0) + " -> "
                str += g.pop(0)
                for rule in g:
                    str += f" | {rule}"
                print(str)
            else:
                print(g[0],"->",g[1])
        print("\n------------------------------------------")


import json

from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt
from django.views.decorators.http import require_http_methods

from Defect_Management_backend.models import Requirement, Iteration

# 新增需求接口
@csrf_exempt
@require_http_methods(['POST'])
def requirement_add(request):
    response = {}
    try:
        form = json.loads(request.body)['form']
        pid = json.loads(request.body)['pid']  # 项目ID
        iteration_title = Iteration.objects.filter(id=form['iteration']).values('title')
        Requirement.objects.create(title=form['title'], priority=form['priority'], iteration=iteration_title, iterationId=form['iteration'],
                                   handler=form['handler'], start=form['start'], end=form['end'],
                                   desc=form['desc'], pid=pid)
        response['respCode'] = '000000'
        response['respMsg'] = 'success'
    except Exception as e:
        response['respCode'] = '999999'
        response['respMsg'] = str(e)
    return JsonResponse(response)

# 查询接口
@csrf_exempt
@require_http_methods(['GET'])
def requirement_info(request):
    response = {}
    try:
        pid = request.GET.get('pid')  # 项目ID
        rid = request.GET.get('rid')  # 需求ID
        iteration_id = request.GET.get('iterationId')    # 迭代ID
        if pid:
            info = Requirement.objects.filter(pid=pid).values()
        elif rid:
            info = Requirement.objects.filter(id=rid).values()
        else:
            info = Requirement.objects.filter(iterationId=iteration_id).values()
        response['respCode'] = '000000'
        response['respMsg'] = 'success'
        response['list'] = list(info)
    except Exception as e:
        response['respCode'] = '000000'
        response['respMsg'] = str(e)
    return JsonResponse(response)

# 修改接口
@csrf_exempt
@require_http_methods(['POST'])
def requirement_edit(request):
    response = {}
    try:
        rid = json.loads(request.body)['rid']
        form = json.loads(request.body)['form']
        iteration_title = Iteration.objects.filter(id=form['iteration']).values('title')
        Requirement.objects.filter(id=rid).update(title=form['title'], priority=form['priority'],
                                                  iterationId=form['iteration'], iteration=iteration_title,
                                                  handler=form['handler'], start=form['start'], end=form['end'],
                                                  desc=form['desc'])
        response['respCode'] = '000000'
        response['respMsg'] = 'success'
    except Exception as e:
        response['respCode'] = '999999'
        response['respMsg'] = str(e)
    return JsonResponse(response)

# 删除接口
@csrf_exempt
@require_http_methods(['GET'])
def requirement_del(request):
    response = {}
    try:
        rid = request.GET.get('rid')  # 需求ID
        Requirement.objects.filter(id=rid).delete()
        response['respCode'] = '000000'
        response['respMsg'] = 'success'
    except Exception as e:
        response['respCode'] = '000000'
        response['respMsg'] = str(e)
    return JsonResponse(response)

# 修改状态接口
@csrf_exempt
@require_http_methods(['POST'])
def requirement_update_state(request):
    response = {}
    try:
        rid = json.loads(request.body)['rid']
        state = json.loads(request.body)['newState']
        Requirement.objects.filter(id=rid).update(state=state)
        response['respCode'] = '000000'
        response['respMsg'] = 'success'
    except Exception as e:
        response['respCode'] = '999999'
        response['respMsg'] = str(e)
    return JsonResponse(response)


import sys

sys.path.append(f"ByteTrack")

import cv2

import os
import re
import glob
from tqdm import tqdm
import numpy as np
import json
from typing import Dict
from collections import defaultdict

from utils.utils import (
    postprocess,
    convert_xywh_to_xyxy,
    detections2boxes,
    match_detections_with_tracks,
    filter,
    extract_emotion_from_dict,
)
import supervision as sv

def emotion_images(conf: Dict) -> None:
    # Define the regex pattern for matching file extensions
    pattern = re.compile(r"\.(jpg|png|jpeg|bmp)$", re.IGNORECASE)

    # Get a list of all files in the folder
    all_files = glob.glob(os.path.join(conf["input_path"], "*"))

    # Filter the files based on the regex pattern
    img_files = [file for file in all_files if re.search(pattern, file)]
    box_annotator = sv.BoxAnnotator(color=sv.Color.white(), thickness=2, text_scale=1)
    person_emotion = defaultdict(str)

    for img_path in tqdm(img_files):
        # Load the image
        image_name = os.path.split(img_path)[1]
        image = cv2.imread(img_path)

        # face model prediction on single frame
        boxes, scores, classids, kpts = conf["face_model"].detect(image)
        boxes = np.asarray(list(map(convert_xywh_to_xyxy, boxes)))
        detections = sv.Detections(
            xyxy=boxes,
            confidence=scores,
            class_id=classids,
        )

        # tracks = byte_tracker.update(
        #     output_results=detections2boxes(detections=detections),
        #     img_info=image.shape,
        #     img_size=image.shape,
        # )

        # tracker_id = match_detections_with_tracks(detections=detections, tracks=tracks)

        detections.tracker_id = np.arange(
            0, len(detections)
        )  # Manual tracker for image

        # mask = np.array(
        #     [tracker_id is not None for tracker_id in detections.tracker_id],
        #     dtype=bool,
        # )

        # detections = filter(detections, mask)

        for xyxy, tracker_id in zip(detections.xyxy, detections.tracker_id):
            cropped_image = sv.crop(image, xyxy=xyxy)
            objs = conf["detector"].detect_emotions(cropped_image)[0]
            person_emotion[tracker_id] = extract_emotion_from_dict(objs["emotions"])

        labels = [
            f"#{tracker_id} {conf*100:0.2f} {person_emotion[tracker_id]}"
            for _, _, conf, class_id, tracker_id in detections
        ]

        with sv.ImageSink(
            target_dir_path=os.path.join(os.getcwd(), conf["save_path"]),
            overwrite=False,
        ) as sink:
            annotated_frame = box_annotator.annotate(
                image, labels=labels, detections=detections
            )

            sink.save_image(
                image=annotated_frame,
                image_name=f"{image_name}",
            )

def emotion_image(conf: Dict) -> None:
    image_name = os.path.split(conf["input_path"])[1]
    image = cv2.imread(conf["input_path"])
    objs = conf["detector"].detect_emotions(image)
    modified_frame = postprocess(image, objs)
    cv2.imwrite(os.path.join(conf["save_path"], f"{image_name}"), modified_frame)


import pyowm
city=input('Sizi hansi sher maraglandirir? : ')
owm=pyowm.OWM('e4b796328673811095c29edf76bf2e36')
observation = owm.weather_at_place(city)
w = observation.get_weather()
temperature=w.get_temperature('celsius')['temp']
wind=w.get_wind() ['speed']
humidity=w.get_humidity()
detalied=w.get_detailed_status()
print("bu sherde " + city + " tempuraturu ise " + str(temperature) + " Selsiya " + " Kuleyin sureti ise " + str(wind) + " Rutubeti ise " + str(humidity))
print("secdiyiniz seherde " + str(detalied))