from datetime import datetime

from aq_lib.event.event_type import EventType
from aq_lib.model.md_entry import MDEntry
from aq_lib.transport.serializable import Serializable

class Event(Serializable):
    def __init__(self, topic):
        self.topic = topic

# Event sent by cerebro to subscribe to MD
class MarketDataRequestEvent(Event):
    def __init__(self, topic=EventType.MARKET_DATA_REQUEST_EVENT.value, on_behalf_of_comp_id=None,
                 deliver_to_comp_id=None, symbols=None, md_req_id=None, subscription_request_type=None,
                 settlement_type=None, security_type=None, settlement_date=None, security_id=None, md_type=None,
                 routing_key=None, currency=None, settlement=None, market=None, description=None, execution_id=None,
                 negotiation_agent_id=None, trading_session=None, maturity_date=None, **kwargs):
        Event.__init__(self, topic)
        self.on_behalf_of_comp_id = on_behalf_of_comp_id
        self.deliver_to_comp_id = deliver_to_comp_id
        self.symbols = symbols
        self.md_req_id = md_req_id
        self.subscription_request_type = subscription_request_type
        self.routing_key = routing_key
        self.md_type = md_type
        self.settlement = settlement
        self.currency = currency
        self.security_id = security_id
        self.security_type = security_type
        self.settlement_date = settlement_date
        self.market = market
        self.description = description
        self.execution_id = execution_id
        self.negotiation_agent_id = negotiation_agent_id
        self.trading_session = trading_session
        self.maturity_date = maturity_date

class MarketDataSnapshotFullRefreshEvent(Event):
    def __init__(self, topic=EventType.MARKET_DATA_SNAPSHOT_FULL_REFRESH_EVENT.value, mdReqId=None, symbol=None,
                 md_type=None, securityId=None, securityType=None, settlType=None, settlDate=None, currency=None,
                 trading_session=None, entries=[], timestamp=None, negotiation_agent=None, **kwargs):
        Event.__init__(self, topic)
        self.mdReqId = mdReqId
        self.symbol = symbol
        self.md_type = md_type
        self.securityId = securityId
        self.securityType = securityType
        self.settlType = settlType
        self.settlDate = settlDate
        self.entries = self.build_entries(entries)
        self.timestamp = timestamp
        self.negotiation_agent = negotiation_agent
        self.trading_session = trading_session
        self.currency = currency

    @staticmethod
    def build_entries(new_entries):
        entries = []
        for entry in new_entries:
            entries.append(MDEntry(**entry))
        return entries

    def get_entry(self, entry_type):
        for entry in self.entries:
            if entry.entry_type == entry_type:
                return entry
        return None

    def addEntry(self, entry):
        self.entries.append(entry)

    def setEntries(self, entries):
        self.entries = entries

class ExecutionReportEvent(Event):
    def __init__(self, topic=EventType.EXECUTION_REPORT_EVENT.value, securityExchange=None, lastQty=None,
                 lastPrice=None, leavesQty=None, execType=None, settlType=None, transactTime=None, timeInForce=None,
                 text=None, symbol=None, side=None, price=None, avgPx=None, orderType=None, orderStatus=None,
                 orderQty=None, orderID=None, execID=None, settlement_date=None, currency=None, cumQty=None,
                 originalClientOrderId=None, account=None, clientOrderID=None, **kwargs):
        Event.__init__(self, topic)
        self.securityExchange = securityExchange
        self.lastQty = lastQty
        self.lastPrice = lastPrice
        self.leavesQty = leavesQty
        self.execType = execType
        self.settlType = settlType
        self.settlement_date = settlement_date
        self.transactTime = transactTime
        self.timeInForce = timeInForce
        self.text = text
        self.symbol = symbol
        self.side = side
        self.price = price
        self.avgPx = avgPx
        self.orderType = orderType
        self.orderStatus = orderStatus
        self.orderQty = orderQty
        self.orderID = orderID
        self.execID = execID
        self.currency = currency
        self.cumQty = cumQty
        self.originalClientOrderId = originalClientOrderId
        self.account = account
        self.clientOrderID = clientOrderID

class NewOrderEvent(Event):
    def __init__(self, topic=EventType.NEW_ORDER_EVENT.value, account=None, cl_ord_id=None, order_qty=None,
                 order_type=None, price=None, side=None, symbol=None, time_in_force=None, currency=None,
                 routing_key=None, exec_inst=None, security_type=None, settlement_type=None, settlement_date=None,
                 execution_id=None, negotiation_agent_id=None, min_qty=None, short_sell=None, **kwargs):
        Event.__init__(self, topic)
        self.account = account
        self.cl_ord_id = cl_ord_id
        self.order_qty = order_qty
        self.order_type = order_type
        self.price = price
        self.side = side
        self.symbol = symbol
        self.time_in_force = time_in_force
        self.routing_key = routing_key
        self.exec_inst = exec_inst
        self.currency = currency
        self.security_type = security_type
        self.settlement_type = settlement_type
        self.settlement_date = settlement_date
        self.execution_id = execution_id
        self.negotiation_agent_id = negotiation_agent_id
        self.min_qty = min_qty
        self.short_sell = short_sell

class OrderCancelReplaceRequestEvent(Event):
    def __init__(self, topic=EventType.ORDER_CANCEL_REPLACE_REQUEST_EVENT.value, account=None, cl_ord_id=None,
                 order_qty=None, order_type=None, orig_client_ord_id=None, orig_order_id=None, price=None, side=None,
                 symbol=None, time_in_force=None, security_type=None, settlement_type=None, currency=None,
                 settlement_date=None, routing_key=None, execution_id=None, negotiation_agent_id=None, short_sell=None,
                 **kwargs):
        Event.__init__(self, topic)
        self.account = account
        self.cl_ord_id = cl_ord_id
        self.order_qty = order_qty
        self.order_type = order_type
        self.orig_client_ord_id = orig_client_ord_id
        self.orig_order_id = orig_order_id
        self.price = price
        self.side = side
        self.symbol = symbol
        self.time_in_force = time_in_force
        self.order_type = order_type
        self.settlement_date = settlement_date
        self.security_type = security_type
        self.settlement_type = settlement_type
        self.routing_key = routing_key
        self.currency = currency
        self.execution_id = execution_id
        self.negotiation_agent_id = negotiation_agent_id
        self.short_sell = short_sell

class BulkNewOrderEvent(Event):
    def __init__(self, topic=EventType.BULK_NEW_ORDER_EVENT.value, orders=[]):
        Event.__init__(self, topic)
        self.orders = orders

class OrderCancelRequestEvent(Event):
    def __init__(self, topic=EventType.ORDER_CANCEL_REQUEST_EVENT.value, account=None, cl_ord_id=None, order_id=None,
                 order_qty=None, orig_cl_ord_id=None, side=None, symbol=None, security_type=None,
                 settlement_type=None, currency=None, settlement_date=None, routing_key=None, execution_id=None,
                 negotiation_agent_id=None, short_sell=None, **kwargs):
        Event.__init__(self, topic)
        self.account = account
        self.cl_ord_id = cl_ord_id
        self.order_id = order_id
        self.order_qty = order_qty
        self.orig_cl_ord_id = orig_cl_ord_id
        self.side = side
        self.symbol = symbol
        self.routing_key = routing_key
        self.security_type = security_type
        self.settlement_type = settlement_type
        self.settlement_date = settlement_date
        self.currency = currency
        self.execution_id = execution_id
        self.negotiation_agent_id = negotiation_agent_id
        self.short_sell = short_sell

class OrderMassCancelRequestEvent(Event):
    def __init__(self, topic=EventType.ORDER_MASS_CANCEL_REQUEST_EVENT.value, account=None, segment=None,
                 clientOrderId=None, **kwargs):
        Event.__init__(self, topic)
        self.account = account
        self.segment = segment
        self.clientOrderId = clientOrderId

class PositionUpdatedEvent(Event):
    def __init__(self, topic=EventType.POSITION_UPDATED_EVENT.value, symbol=None, account=None, size=None, price=None,
                 open_result=None, close_result=None, strategy_id=None, strategy_version=None, execution_id=None,
                 arquants_user=None, execution_type=None):
        Event.__init__(self, topic)
        self.symbol = symbol
        self.account = account
        self.size = size
        self.price = price
        self.open_result = open_result
        self.close_result = close_result
        self.strategy_id = strategy_id
        self.strategy_version = strategy_version
        self.execution_id = execution_id
        self.arquants_user = arquants_user
        self.execution_type = execution_type

class OrderUpdatedEvent(Event):
    def __init__(self, topic=EventType.LT_ORDER_UPDATED_EVENT.value, strategy_id=None, strategy_version=None,
                 execution_id=None, arquants_user=None, execution_type=None, account=None, symbol=None, side=None,
                 orderStatus=None, price=None, orderQty=None, cumQty=None, orderID=None, clientOrderID=None,
                 transactTime=None, ers=[], timeInForce=None, text=None, securityExchange=None, leavesQty=None,
                 execType=None, execID=None, **kwargs):
        Event.__init__(self, topic)
        self.strategy_id = strategy_id
        self.strategy_version = strategy_version
        self.execution_id = execution_id
        self.arquants_user = arquants_user
        self.execution_type = execution_type
        self.account = account
        self.symbol = symbol
        self.side = side
        self.orderStatus = orderStatus
        self.price = price
        self.orderQty = orderQty
        self.cumQty = cumQty
        self.orderID = orderID
        self.clientOrderID = clientOrderID
        self.transactTime = transactTime
        self.execID = execID
        self.execType = execType
        self.leavesQty = leavesQty
        self.securityExchange = securityExchange
        self.text = text
        self.timeInForce = timeInForce
        self.ers = ers
        self.time = str(datetime.now())

class LogsEvent(Event):
    def __init__(self, topic=EventType.LOGS_EVENT.value, arquants_user=None, execution_id=None, execution_type=None,
                 strategy_id=None, log_line=None, **kwargs):
        Event.__init__(self, topic)
        self.arquants_user = arquants_user
        self.execution_id = execution_id
        self.execution_type = execution_type
        self.strategy_id = strategy_id
        self.log_line = log_line

class ParameterChangeEvent(Event):
    def __init__(self, topic=EventType.PARAMETER_CHANGE_EVENT.value, parameters=dict(), execution_id=None,
                 arquants_user=None, strategy_id=None, strategy_version=None, execution_type=None, from_api=False,
                 **kwargs):
        Event.__init__(self, topic)
        self.parameters = parameters
        self.execution_id = execution_id
        self.arquants_user = arquants_user
        self.strategy_id = strategy_id
        self.strategy_version = strategy_version
        self.execution_type = execution_type
        self.from_api = from_api

class ManualOrderCancelRequestEvent(Event):
    def __init__(self, topic=EventType.MANUAL_ORDER_CANCEL_REQUEST_EVENT.value, order_id=None, **kwargs):
        Event.__init__(self, topic)
        self.order_id = order_id

class TradeExecutedEvent(Event):
    def __init__(self, topic=EventType.TRADE_EXECUTED_EVENT.value, strategy_id=None, strategy_version=None,
                 execution_id=None, arquants_user=None, execution_type=None, ts=None, symbol=None, side=None,
                 size=None, price=None, account=None, orderid=None, tradeid=None, settlType=None, **kwargs):
        Event.__init__(self, topic)

        self.strategy_id = strategy_id
        self.strategy_version = strategy_version
        self.execution_id = execution_id
        self.arquants_user = arquants_user
        self.execution_type = execution_type
        self.ts = ts
        self.symbol = symbol
        self.side = side
        self.size = size
        self.price = price
        self.account = account
        self.orderid = orderid
        self.tradeid = tradeid
        self.settlType = settlType

class BuildStrategyErrorEvent(Event):
    def __init__(self, topic=EventType.BUILD_STRATEGY_ERROR_EVENT.value, execution_id=None, arquants_user=None,
                 strategy_id=None, strategy_version=None, execution_type=None, error=None, **kwargs):
        Event.__init__(self, topic)
        self.execution_id = execution_id
        self.arquants_user = arquants_user
        self.strategy_id = strategy_id
        self.strategy_version = strategy_version
        self.execution_type = execution_type
        self.error = error

class LiveTradingStartedEvent(Event):
    def __init__(self, topic=EventType.LIVE_TRADING_STARTED_EVENT.value, execution_id=None, arquants_user=None,
                 status=None, **kwargs):
        Event.__init__(self, topic)
        self.execution_id = execution_id
        self.arquants_user = arquants_user
        self.status = status

class OrderStatusRequestEvent(Event):
    def __init__(self, topic=EventType.ORDER_STATUS_REQUEST_EVENT.value, client_order_id=None, side=None, **kwargs):
        Event.__init__(self, topic)
        self.client_order_id = client_order_id
        self.side = side

class PushNotificationEvent(Event):
    def __init__(self, topic=EventType.PUSH_NOTIFICATION_EVENT.value, title=None, body=None, user=None, device=None,
                 execution_id=None, strategy_id=None):
        Event.__init__(self, topic)
        self.title = title
        self.body = body
        self.user = user
        self.device = device
        self.execution_id = execution_id
        self.strategy_id = strategy_id

class StrategyStartedEvent(Event):
    """
    Evento para el risk module

    """

    def __init__(self, topic=EventType.STRATEGY_STARTED_EVENT.value,
                 execution_id=None,
                 strategy_version=None,
                 rate_limit_orders=10,
                 rate_limit_logs=5,
                 tickers=None,
                 user=None,
                 routing_key=None,
                 type=None,
                 strategy_id=None
                 ):
        Event.__init__(self, topic)
        self.strategy_version = strategy_version
        self.user = user
        self.tickers = tickers
        self.rate_limit_logs = rate_limit_logs
        self.rate_limit_orders = rate_limit_orders
        self.execution_id = execution_id
        self.routing_key = routing_key
        self.type = type
        self.strategy_id = strategy_id

class StrategyStatusChangeEvent(Event):
    """
    Evento enviado por las estrategias, y tambien por la API.

    El evento tiene un significado distinto en cada uno de estos dos casos:
    -Cuando el evento es enviado por la estrategia, se usa para notificar que hubo un cambio de estado en la estrategia.
    -Cuando el evento es enviado por la API, se usa para ordenarle a una estrategia en particular que cambie su estado.

    Posibles valores de estado:
     - ACTIVE
     - STOPPED
     - FINISHED
     - ERROR

    """

    def __init__(self, topic=EventType.STRATEGY_STATUS_CHANGE.value, status=None, from_api=None, execution_id=None):
        Event.__init__(self, topic)
        self.status = status
        self.from_api = from_api
        self.execution_id = execution_id


from __future__ import division
import os, re
from random import shuffle
from collections import Counter
import numpy as np
import pandas as pd
import parevol_tools as pt
from sklearn.decomposition import PCA
import functools
from sklearn.metrics.pairwise import euclidean_distances

def run_pca_permutation(iter = 10000, analysis = 'PCA', dataset = 'tenaillon'):
    if dataset == 'tenaillon':
        k = 3
        df_path = pt.get_path() + '/data/Tenaillon_et_al/gene_by_pop.txt'
        df = pd.read_csv(df_path, sep = '\t', header = 'infer', index_col = 0)
        df_array = df.as_matrix()
        df_out = open(pt.get_path() + '/data/Tenaillon_et_al/permute_' + analysis + '.txt', 'w')
        column_headers = ['Iteration', 'MCD', 'mean_angle', 'mean_dist', 'delta_L', 'x_stat']
        df_out.write('\t'.join(column_headers) + '\n')
        for i in range(iter):
            print(i)
            df_rndm = pd.DataFrame(data=pt.random_matrix(df_array), index=df.index, columns=df.columns)
            df_rndm_delta = pt.likelihood_matrix(df_rndm, 'Tenaillon_et_al').get_likelihood_matrix()
            if analysis == 'PCA':
                X = pt.hellinger_transform(df_rndm_delta)
                pca = PCA()
                df_rndm_delta_out = pca.fit_transform(X)
                #df_pca = pd.DataFrame(data=X_pca, index=df.index)
            mean_angle = pt.get_mean_angle(df_rndm_delta_out, k = k)
            mcd = pt.get_mean_centroid_distance(df_rndm_delta_out, k=k)
            mean_length = pt.get_euc_magnitude_diff(df_rndm_delta_out, k=k)
            mean_dist = pt.get_mean_pairwise_euc_distance(df_rndm_delta_out, k=k)
            x_stat = pt.get_x_stat(pca.explained_variance_[:-1])
            df_out.write('\t'.join([str(i), str(mcd), str(mean_angle), str(mean_dist), str(mean_length), str(x_stat)]) + '\n')
        df_out.close()

    elif dataset == 'good':
        k = 5
        df_path = pt.get_path() + '/data/Good_et_al/gene_by_pop.txt'
        df = pd.read_csv(df_path, sep = '\t', header = 'infer', index_col = 0)
        to_exclude = pt.complete_nonmutator_lines()
        to_exclude.append('p5')
        df_nonmut = df[df.index.str.contains('|'.join( to_exclude))]
        # remove columns with all zeros
        df_nonmut = df_nonmut.loc[:, (df_nonmut != 0).any(axis=0)]
        time_points = [ int(x.split('_')[1]) for x in df_nonmut.index.values]
        time_points_set = sorted(list(set([ int(x.split('_')[1]) for x in df_nonmut.index.values])))
        df_nonmut_array = df_nonmut.as_matrix()
        time_points_positions = {}
        for x in time_points_set:
            time_points_positions[x] =  [i for i,j in enumerate(time_points) if j == x]
        df_final = df_nonmut.iloc[time_points_positions[time_points_set[-1]]]

        df_out = open(pt.get_path() + '/data/Good_et_al/permute_' + analysis + '.txt', 'w')
        #column_headers = ['Iteration', 'Generation', 'MCD']
        column_headers = ['Iteration', 'Generation', 'MCD', 'mean_angle', 'delta_L', 'mean_dist']
        df_out.write('\t'.join(column_headers) + '\n')
        for i in range(iter):
            print("Iteration " + str(i))
            matrix_0 = df_nonmut.iloc[time_points_positions[time_points_set[0]]]
            matrix_0_rndm = pt.random_matrix(matrix_0.as_matrix())
            df_rndm_list = [pd.DataFrame(data=matrix_0_rndm, index=matrix_0.index, columns=matrix_0.columns)]
            # skip first time step
            for j, tp in enumerate(time_points_set[0:]):
                if j == 0:
                    continue
                df_tp_minus1 = df_nonmut[df_nonmut.index.str.contains('_' + str(time_points_set[j-1]))]
                df_tp = df_nonmut[df_nonmut.index.str.contains('_' + str(tp))]
                matrix_diff = df_tp.as_matrix() - df_tp_minus1.as_matrix()
                matrix_0_rndm = matrix_0_rndm +  pt.random_matrix(matrix_diff)
                df_0_rndm = pd.DataFrame(data=matrix_0_rndm, index=df_tp.index, columns=df_tp.columns)
                df_rndm_list.append(df_0_rndm)

            df_rndm = pd.concat(df_rndm_list)
            df_rndm_delta = pt.likelihood_matrix(df_rndm, 'Good_et_al').get_likelihood_matrix()
            if analysis == 'PCA':
                X = pt.hellinger_transform(df_rndm_delta)
                pca = PCA()
                matrix_rndm_delta_out = pca.fit_transform(X)
            elif analysis == 'cMDS':
                matrix_rndm_delta_bc = np.sqrt(pt.get_bray_curtis(df_rndm_delta.as_matrix()))
                matrix_rndm_delta_out = pt.cmdscale(matrix_rndm_delta_bc)[0]
            else:
                print("Analysis argument not accepted")
                continue

            df_rndm_delta_out = pd.DataFrame(data=matrix_rndm_delta_out, index=df_rndm_delta.index)
            for tp in time_points_set:
                df_rndm_delta_out_tp = df_rndm_delta_out[df_rndm_delta_out.index.str.contains('_' + str(tp))]
                df_rndm_delta_out_tp_matrix = df_rndm_delta_out_tp.as_matrix()
                mean_angle = pt.get_mean_angle(df_rndm_delta_out_tp_matrix, k = k)
                mcd = pt.get_mean_centroid_distance(df_rndm_delta_out_tp_matrix, k=k)
                mean_length = pt.get_euc_magnitude_diff(df_rndm_delta_out_tp_matrix, k=k)
                mean_dist = pt.get_mean_pairwise_euc_distance(df_rndm_delta_out_tp_matrix, k=k)
                df_out.write('\t'.join([str(i), str(tp), str(mcd), str(mean_angle), str(mean_length), str(mean_dist) ]) + '\n')

        df_out.close()

def run_pca_sample_size_permutation(iter = 10000, analysis = 'PCA', k =3):
    df_path = pt.get_path() + '/data/Tenaillon_et_al/gene_by_pop.txt'
    df = pd.read_csv(df_path, sep = '\t', header = 'infer', index_col = 0)
    df_array = df.as_matrix()
    sample_sizes = np.linspace(2, df.shape[0], num = 20, dtype = int)
    df_out = open(pt.get_path() + '/data/Tenaillon_et_al/sample_size_permute_' + analysis + '.txt', 'w')
    column_headers = ['Sample_size', 'Iteration', 'MCD', 'mean_angle', 'delta_L']
    df_out.write('\t'.join(column_headers) + '\n')
    for sample_size in sample_sizes:
        print("Sample size = "  + str(sample_size))
        for i in range(iter):
            print("Sample size = "  + str(sample_size) + ' Iteration = ' + str(i))
            df_sample = df.sample(n = sample_size)
            #df_sample = df_sample.loc[:, (df_sample != 0).any(axis=0)]
            df_sample_delta = pt.likelihood_matrix(df_sample, 'Tenaillon_et_al').get_likelihood_matrix()
            df_sample_delta = df_sample_delta.loc[:, (df_sample_delta != 0).any(axis=0)]
            X = pt.hellinger_transform(df_sample_delta)
            pca = PCA()
            df_sample_delta_out = pca.fit_transform(X)
            mcd = pt.get_mean_centroid_distance(df_sample_delta_out, k=k)
            mean_angle = pt.get_mean_angle(df_sample_delta_out, k = k)
            mean_length = pt.get_euclidean_distance(df_sample_delta_out, k=k)

            df_out.write('\t'.join([str(sample_size), str(i), str(mcd), str(mean_angle), str(mean_length)]) + '\n')

    df_out.close()

def get_F_stat_pairwise(pca_array, groups, k=3):
    # assuming only two groups
    # groups is a nested list, each list containing row integers for a given group
    X = pca_array[:,0:k]
    between_var = 0
    within_var = 0
    K = len(groups)
    N = np.shape(X)[0]
    euc_sq_dists = np.square(euclidean_distances(X, X))
    SS_T = sum( np.sum(euc_sq_dists, axis =1)) /  (N*2)
    euc_sq_dists_group1 = euc_sq_dists[groups[0][:, None], groups[0]]
    SS_W = sum( np.sum(euc_sq_dists_group1, axis =1)) /  (len(groups[0])*2)
    # between groups sum-of-squares
    SS_A = SS_T - SS_W

    return (SS_A / (K-1)) / (SS_W / (N-K) )

def get_F_stat_mcd(pca_array, groups, k=3):
    X = pca_array[:,0:k]
    mcd_overall = pt.get_mean_centroid_distance(X)
    centroid_overall = np.mean(X, axis = 0)
    between_var = 0
    within_var = 0
    K = len(groups)
    N = np.shape(X)[0]

    for group in groups:
        #euc_dists_group = euc_dists[group[:, None], group]
        X_group = X[group, :]
        mcd_group = pt.get_mean_centroid_distance(X_group)
        groups_values = []
        centroid_group = np.mean(X_group, axis = 0)
        #between_var += ((mcd_group - mcd_group) ** 2) * len(groups)
        between_var += ((centroid_group - centroid_overall) ** 2) * len(groups)

        #centroid_distances = np.sqrt(np.sum(np.square(X_group - np.mean(X_group, axis = 0)), axis=1))
        #within_var += sum( (centroid_distances - mcd_group) ** 2 )

        #within_var += sum(np.sqrt(np.sum(np.square(X_group - np.mean(X_group, axis = 0)), axis=1)))
        within_var += sum(np.sqrt(np.sum(np.square(X_group - centroid_group), axis=1)))

    return (between_var / (K-1)) / (within_var / (N-K) )

def two_treats_sim(iter1 = 1000, iter2 = 1000, alpha = 0.05):
    genes = 10
    pops1 = pops2 = 10
    shape = 1
    scale = 1
    muts1 = muts2 = 20
    to_reshuffle = [0, 5, 10, 15, 20]

    for reshuf in to_reshuffle:
        p_vales = []
        for i in range(iter1):
            #print(i)
            rates = np.random.gamma(shape, scale=scale, size=genes)
            rates1 = rates.copy()
            # permute rates
            shuffle(rates[:reshuf])
            rates2 = rates.copy()
            list_dicts1 = [Counter(np.random.choice(genes, size = muts1, replace = True, p = rates1 / sum(rates1))) for i in range(pops1) ]
            list_dicts2 = [Counter(np.random.choice(genes, size = muts2, replace = True, p = rates2 / sum(rates2))) for i in range(pops2) ]

            df1 = pd.DataFrame(list_dicts1)
            df2 = pd.DataFrame(list_dicts2)
            df = pd.concat([df1, df2])
            df = df.fillna(0)
            count_matrix = df.values
            groups = [ np.asarray(list(range(0, pops1))), np.asarray(list(range(pops1, pops1+pops2))) ]
            pca = PCA()
            X = pt.hellinger_transform(count_matrix)
            pca_fit = pca.fit_transform(X)
            F = get_F_stat_pairwise(pca_fit, groups)
            F_list = []
            for j in range(iter2):
                count_matrix_n0 = pt.random_matrix(count_matrix)
                X_n0 = pt.hellinger_transform(count_matrix_n0)
                pca_fit_n0 = pca.fit_transform(X_n0)
                F_list.append(get_F_stat_pairwise(pca_fit_n0, groups))

            p_vales.append((len([x for x in F_list if x > F]) +1)  / (iter2+1))

        power = (len([k for k in p_vales if k < alpha]) +1)  / (iter1+1)
        print('Reshuffle = ' + str(reshuf) + ', Power ' +  str(power))

    #fig = plt.figure()
    #plt.hist(F_list, bins=50,  alpha=0.8, color = '#175ac6')
    #plt.axvline(F, color = 'red', lw = 3, ls = '--')
    #plt.tight_layout()
    #fig_name = pt.get_path() + '/figs/test_F.png'
    #fig.savefig(fig_name, bbox_inches = "tight", pad_inches = 0.4, dpi = 600)
    #plt.close()

    #print(np.mean(F_list))

#two_treats_sim()
#test_stats()
#run_pca_permutation()
#get_likelihood_matrices()
#run_pca_permutation(dataset='tenaillon', iter =10000)


import gspread
from oauth2client.service_account import ServiceAccountCredentials

# use creds to create a client to interact with the Google Drive API
scope = ['https://spreadsheets.google.com/feeds']
credentials = ServiceAccountCredentials.from_json_keyfile_name('client_secret.json', scope)
client = gspread.authorize(credentials)

# Find a workbook by name and open the first sheet
# Make sure you use the right name here.
sheet = client.open_by_url("https://docs.google.com/spreadsheets/d/1FEaaJdhnomaV2fPBwO0fLfqNK6_bnncfy7SIWvOPZFQ/edit?usp=drive_web&ouid=103704144020418407446").sheet1

# Extract and print all of the values
list_of_hashes = sheet.get_all_records()
print(list_of_hashes)


import numpy as np
from scipy.stats import f
import matplotlib.pyplot as plt

x=np.arange(0,10,.001)
plt.plot(x,f.pdf(x,1,1), x,f.pdf(x,3,8), x,f.pdf(x,8,3), x,f.pdf(x, 40,40))

plt.xlim(0,10)
plt.ylim(0,1.5)

plt.xlabel('$x$',fontsize=20)
plt.ylabel('$P(X=x | K_1,K_2)$',fontsize=18)

plt.title('Fisher-Distribution',fontsize=20)
plt.legend(['$K_1=1, K_2=1$','$K_1=3, K_2=8$', '$K_1=8 ,K_2=3$', '$K_1=40, K_2=40$'])
plt.show()


import pytest
from .neighbouring import *
from .helpers_for_tests import *

def test_create_Neighbouring_instance():
    with pytest.raises(TypeError):
        neighbouring = Neighbouring([1.,2.3,5.2])

def test_neighbour_structur_is_created():
    array = np.random.rand(5,3)
    r_cutoff = 3.0
    neighbouring = NeighbouringCellLinkedLists(particle_positions=array,radius=r_cutoff, box_size=1.0)
    assert neighbouring._neighbourlist != None
    # usually we would not call private (with _name) attributes from outside

def test_create_Neighbouring_Cell_Linked_Lists_instance():

    with pytest.raises(TypeError):
        n = NeighbouringCellLinkedLists("something")

def test_equivalence_CLL_PL():
    box_size = float(7.6)
    nr_particles = 25
    pid = 5 % nr_particles
    particle_pos = np.random.rand(nr_particles, 3)*box_size
    NL_PL = NeighbouringPrimitiveLists(particle_pos, radius=1.2, box_size=box_size)
    NL_CLL = NeighbouringCellLinkedLists(particle_pos, radius=1.2, box_size=box_size)
    assert set(NL_CLL.get_particles_within_radius(pid)[0]) == set(NL_PL.get_particles_within_radius(pid))


from tkinter import *
from tkinter import ttk
import tkinter.messagebox
from tkinter import filedialog
from tkinter.messagebox import showinfo
from goto import with_goto
import shutil
import csv
import json
import os
from pathlib import Path
import sys
sys.path.insert(0, './TibiaBOT/bot')
from main import *

def food():
    if (var1.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\eat_food\eat_food.exe",'x')
    else:
        os.system("taskkill /im eat_food.exe /F")

def ring():
    if (var11.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\auto_ring\auto_ring.exe",'x')
    else:
        os.system("taskkill /im auto_ring.exe /F")

def player_on_screen():
    if (var12.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\player_on_screen\player_on_screen.exe", 'a')
    else:
        os.system("taskkill /IM player_on_screen.exe /F")

def private_mensage():
    if (var13.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\private_mensage\private_mensage.exe", 'a')
    else:
        os.system("taskkill /IM private_mensage.exe /F")

def anti_bot():
    if (var15.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\anti_bot\anti_bot.exe", 'a')
    else:
        os.system("taskkill /IM anti_bot.exe /F")

def haste():
    if (var2.get() == 1):
        os.spawnl(os.P_DETACH, r"dist/auto_haste/auto_haste.exe", 'x')
    else:
        os.system("taskkill /im auto_haste.exe /F")

def cura_vida():
    if (var3.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\cura_vida\cura_vida.exe", 'x')
    else:
        os.system("taskkill /im cura_vida.exe /F")
    with open(r'parametros.json', 'r') as read_file:
        ler = json.load(read_file)
    ler['cv'] = var3.get()
    with open(r'parametros.json', 'w') as f:
        json.dump(ler, f)

def mana_train():
    if (var4.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\mana_train\mana_train.exe", 'a')
    else:
        os.system("taskkill /IM mana_train.exe /F")

def paralyze():
    if (var6.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\auto_anti_paralyze\auto_anti_paralyze.exe", 'x')
    else:
        os.system("taskkill /im auto_anti_paralyze.exe /F ")

def utamo():
    if (var7.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\auto_utamo\auto_utamo.exe", 'x')
        #ativar_janela('tibia')
    else:
        os.system("taskkill /im auto_utamo.exe /F")

def cura_mana():
    with open(r'parametros.json', 'r') as read_file:
        ler = json.load(read_file)
    if (var8.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\cura_mana\cura_mana.exe", 'x')
    else:
        os.system("taskkill /im cura_mana.exe /F")
    ler['cm'] = var8.get()
    with open(r'parametros.json', 'w') as f:
        json.dump(ler, f)

def ler_var(col):
    with open(r'parametros.json', 'r') as read_file:
        ler = json.load(read_file)
        val = ler[col]
    return val

def auto_atack():
    if (var9.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\auto_atack\auto_atack.exe", 'x')
    else:
        os.system("taskkill /im auto_atack.exe /F")
    with open(r'parametros.json', 'r') as read_file:
        ler = json.load(read_file)
    ler['auto_atack'] = var9.get()
    with open(r'parametros.json', 'w') as f:
        json.dump(ler, f)

def auto_loot():
    if (var14.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\auto_loot\auto_loot.exe", 'x')
    else:
        os.system("taskkill /im auto_loot.exe /F")

def follow_on():
    if (var16.get() == 1):
        os.spawnl(os.P_DETACH, r"dist\follow_on\follow_on.exe", 'x')
    else:
        os.system("taskkill /im follow_on.exe /F")

def cavebot():
    if (var10.get() == 1):
        #os.spawnl(os.P_DETACH, r"dist\cavebot\cavebot.exe", 'x')
        os.spawnl(os.P_DETACH, r"dist\script_reader\script_reader.exe", 'x')
    else:
        #os.system("taskkill /im cavebot.exe /F")
        os.system("taskkill /im script_reader.exe /F")

def atualizar():
    with open(r'parametros.json', 'r') as read_file:
        ler = json.load(read_file)
    ler['ef_ht']=ef_ht.get()
    ler['cv_perc']=float(cv_perc.get())
    ler['cv_ht']=cv_ht.get()
    ler['cv_perc_sup']=float(cv_perc_sup.get())
    ler['cv_ht_sup']=cv_ht_sup.get()
    ler['mt_perc']=float(mt_perc.get())
    ler['mt_ht']=mt_ht.get()
    ler['ah_ht']=ah_ht.get()
    ler['ap_ht']=ap_ht.get()
    ler['au_ht']=au_ht.get()
    ler['cm_perc']=float(cm_perc.get())
    ler['cm_ht']=cm_ht.get()
    ler['ar_ht'] = ar_ht.get()
    ler['spell_um_nm'] = spell_um_nm.get()
    ler['spell_dois_nm'] = spell_dois_nm.get()
    ler['spell_tres_nm'] = spell_tres_nm.get()
    ler['spell_um_ht'] = spell_um_ht.get()
    ler['spell_dois_ht'] = spell_dois_ht.get()
    ler['spell_tres_ht'] = spell_tres_ht.get()
    ler['min_mana_atack'] = float(min_mana_atack.get())
    with open(r'parametros.json', 'w') as f:
        json.dump(ler, f)
    tkinter.messagebox.showinfo('Tibia BOT','Hotkeys atualizadas')
    frame1.focus_set()

def background():

    with open('parametros_back.json', 'r') as read_file:
        ler = json.load(read_file)

    if (var5.get() == 1):
        ler['back'] = 1
    else:
        ler['back'] = 0

    with open('parametros_back.json', 'w') as f:
        json.dump(ler, f)

def selecionar_script():
    pasta = ''
    folder_selected = filedialog.askdirectory()
    for i in range(1,len(folder_selected)):
        x = folder_selected[-i]
        y = x.find('/')
        if y==-1:
            pasta = x+pasta
        else:
            break

    with open(r'parametros.json', 'r') as read_file:
        ler = json.load(read_file)
    ler['script'] = pasta
    with open(r'parametros.json', 'w') as f:
        json.dump(ler, f)

    btn_txt.set(f'script: {pasta}')

    return pasta

def limpar_script():
    entry10.delete(0, END)
    frame11.delete(CURRENT, END)

@with_goto
def delete_script():
    pasta=script_name.get()
    if pasta=='' or pasta == None:
        goto .fim
    limpar_script()
    path = "./scripts/"+pasta
    isExist = os.path.exists(path)
    if isExist:
        shutil.rmtree(path)
    showinfo(title='Informação', message='Script deletado')
    goto .encerrar
    label .fim
    showinfo(title='Informação', message='Selecione script a ser deletado')
    label .encerrar

@with_goto
def salvar_script():
    pasta=script_name.get()
    if pasta=='' or pasta == None:
        goto .fim
    path = "./scripts/"+pasta
    r=frame11.get('1.0','end')
    if len(r)<5:
        goto .fim
    isExist = os.path.exists(path)
    if not isExist:
        os.makedirs(path)

    f = open(path+f'/{pasta}.csv', 'w+')
    writer = csv.writer(f)
    if r[0:1]==f'\n':
        r=r[1:len(r)]
    if r[(len(r)-2):len(r)]==f'\n\n':
        r=r[0:(len(r)-2)]
    writer.writerow([r.replace(f'\n\n','\n')])
    f.close()

    frame11.delete(CURRENT, END)
    ficheiro = open(f'./scripts/{pasta}/{pasta}.csv', 'r')
    ler = csv.reader(ficheiro, delimiter=';')
    lista = list(ler)
    for i in range(0, len(lista) - 1):
        c = lista[i][0]
        if c != '' and c != r'\n':
            frame11.insert(END, c)
            if i < len(lista) - 2:
                frame11.insert(END, '\n')

    showinfo(title='Informação', message='Script salvo')
    goto .encerrar
    label .fim
    showinfo(title='Informação', message='Script vazio')
    label .encerrar

def abrir_script():
    root = Toplevel(window)
    root.title('Listbox')
    rootdir = 'scripts'
    langs = ()
    for file in os.listdir(rootdir):
        d = os.path.join(rootdir, file)
        if os.path.isdir(d):
            if d.replace("scripts\\", "") != "__pycache__":
                langs = langs + (d.replace("scripts\\", ""),)
    var = Variable(value=langs)
    listbox = Listbox(
        root,
        listvariable=var,
        height=6,
        selectmode=EXTENDED)
    listbox.pack(expand=True, fill=BOTH, side=LEFT)
    # link a scrollbar to a list
    scrollbar = ttk.Scrollbar(
        root,
        orient=VERTICAL,
        command=listbox.yview
    )
    listbox['yscrollcommand'] = scrollbar.set
    scrollbar.pack(side=LEFT, expand=True, fill=Y)

    def items_selected(event):
        # get selected indices
        selected_indices = listbox.curselection()
        # get selected items
        selected_langs = ",".join([listbox.get(i) for i in selected_indices])
        msg = f'You selected: {selected_langs}'

        entry10.delete(0,END)
        entry10.insert(0,selected_langs)
        frame11.delete(CURRENT, END)
        ficheiro = open(f'./scripts/{selected_langs}/{selected_langs}.csv', 'r')
        ler = csv.reader(ficheiro, delimiter=';')
        lista = list(ler)
        for i in range(0, len(lista)-1):
            c = lista[i][0]
            if c!='' and c!=r'\n':
                frame11.insert(END,c)
                if i<len(lista)-2:
                    frame11.insert(END, '\n')

        showinfo(title='Information', message=msg)
        root.destroy()

    listbox.bind('<<ListboxSelect>>', items_selected)

    #Preenche Codigo

def limpa_programa():
    os.system("taskkill /im cura_vida.exe /F")
    os.system("taskkill /im cura_mana.exe /F")
    os.system("taskkill /im mana_train.exe /F")
    os.system("taskkill /im auto_haste.exe /F")
    os.system("taskkill /im eat_food.exe /F")
    os.system("taskkill /im auto_atack.exe /F")
    os.system("taskkill /im cavebot.exe /F")
    os.system("taskkill /im auto_ring.exe /F")
    os.system("taskkill /im player_on_screen.exe /F")
    os.system("taskkill /im private_mensage.exe /F")
    os.system("taskkill /im anti_bot.exe /F")
    os.system("taskkill /im follow_on.exe /F")
    os.system("taskkill /im script_reader.exe /F")

def kill_programa():
    os.system("taskkill /im cura_vida.exe /F")
    os.system("taskkill /im cura_mana.exe /F")
    os.system("taskkill /im mana_train.exe /F")
    os.system("taskkill /im auto_haste.exe /F")
    os.system("taskkill /im eat_food.exe /F")
    os.system("taskkill /im auto_atack.exe /F")
    os.system("taskkill /im cavebot.exe /F")
    os.system("taskkill /im auto_ring.exe /F")
    os.system("taskkill /im player_on_screen.exe /F")
    os.system("taskkill /im private_mensage.exe /F")
    os.system("taskkill /im anti_bot.exe /F")
    os.system("taskkill /im follow_on.exe /F")
    os.system("taskkill /im script_reader.exe /F")
    os.system("taskkill /im python.exe /F")

def stop_script():
    os.system("taskkill /im auto_atack.exe /F")
    os.system("taskkill /im cavebot.exe /F")
    os.system("taskkill /im follow_on.exe /F")
    os.system("taskkill /im script_reader.exe /F")

##################################################
# Window de Scripts
##################################################

def abrir_lista_scripts():
    # create the root window
    root = Toplevel(window)

    root.title('Listbox')

    # create a list box
    # langs = ('Java', 'C#', 'C', 'C++', 'Python',
    #         'Go', 'JavaScript', 'PHP', 'Swift')
    rootdir = 'scripts'
    langs = ()
    for file in os.listdir(rootdir):
        d = os.path.join(rootdir, file)
        if os.path.isdir(d):
            if d.replace("scripts\\", "") != "__pycache__":
                langs = langs + (d.replace("scripts\\", ""),)

    var = Variable(value=langs)

    listbox = Listbox(
        root,
        listvariable=var,
        height=6,
        selectmode=EXTENDED)

    listbox.pack(expand=True, fill=BOTH, side=LEFT)

    # link a scrollbar to a list
    scrollbar = ttk.Scrollbar(
        root,
        orient=VERTICAL,
        command=listbox.yview
    )

    listbox['yscrollcommand'] = scrollbar.set

    scrollbar.pack(side=LEFT, expand=True, fill=Y)

    def items_selected(event):
        # get selected indices
        selected_indices = listbox.curselection()
        # get selected items
        selected_langs = ",".join([listbox.get(i) for i in selected_indices])
        msg = f'Script {selected_langs} selecionado'

        with open(r'parametros.json', 'r') as read_file:
            ler = json.load(read_file)
        ler['script'] = str(selected_langs)
        with open(r'parametros.json', 'w') as f:
            json.dump(ler, f)
        btn_txt.set(f'script: {str(selected_langs)}')

        showinfo(title='Information', message=msg)
        root.destroy()

    listbox.bind('<<ListboxSelect>>', items_selected)

##################################################
# Configurações da janela
##################################################

limpa_programa()

window = Tk()
window.title('Tibia BOT')
window.geometry('580x380')
window.configure(bg='#dddddd')

menubar = Menu(window)
window.config(menu=menubar)
filemenu = Menu(menubar, tearoff=0)
filemenu.add_command(label="Atualizar Hotkeys", command=atualizar)
#filemenu.add_command(label="Open", command=atualizar)
#filemenu.add_command(label="Save", command=atualizar)
filemenu.add_separator()
filemenu.add_command(label="Exit", command=kill_programa)
menubar.add_cascade(label="File", menu=filemenu)

helpmenu = Menu(menubar, tearoff=0)
helpmenu.add_command(label="Help Index", command=atualizar)
helpmenu.add_command(label="About...", command=atualizar)
menubar.add_cascade(label="Help", menu=helpmenu)

tabControl = ttk.Notebook (window)
tab1 = ttk.Frame (tabControl)
tab2 = ttk.Frame (tabControl)
tab3 = ttk.Frame (tabControl)
tab4 = ttk.Frame (tabControl)
tabControl.add (tab1, text = 'Main')
tabControl.add (tab2, text = 'Target')
tabControl.add (tab3, text = 'Cavebot')
tabControl.add (tab4, text = 'Editor Cavebot')
tabControl.pack(expand = 1, fill ="both")

##################################################
# Frame comandos principais
##################################################
frame1 = Label(tab1)
frame1.pack()
frame2 = LabelFrame(frame1, text='On/Off')

ef_ht = StringVar(value=ler_var('ef_ht'))
cv_perc = StringVar(value=ler_var('cv_perc'))
cv_ht = StringVar(value=ler_var('cv_ht'))
cv_perc_sup = StringVar(value=ler_var('cv_perc_sup'))
cv_ht_sup = StringVar(value=ler_var('cv_ht_sup'))
mt_perc = StringVar(value=ler_var('mt_perc'))
mt_ht = StringVar(value=ler_var('mt_ht'))
ah_ht = StringVar(value=ler_var('ah_ht'))
ap_ht = StringVar(value=ler_var('ap_ht'))
au_ht = StringVar(value=ler_var('au_ht'))
cm_perc = StringVar(value=ler_var('cm_perc'))
cm_ht = StringVar(value=ler_var('cm_ht'))
ar_ht = StringVar(value=ler_var('ar_ht'))
spell_um_nm = StringVar(value=ler_var('spell_um_nm'))
spell_dois_nm = StringVar(value=ler_var('spell_dois_nm'))
spell_tres_nm = StringVar(value=ler_var('spell_tres_nm'))
spell_um_ht = StringVar(value=ler_var('spell_um_ht'))
spell_dois_ht = StringVar(value=ler_var('spell_dois_ht'))
spell_tres_ht = StringVar(value=ler_var('spell_tres_ht'))
min_mana_atack = StringVar(value=ler_var('min_mana_atack'))
back = IntVar

##Labels

label = Label(frame1, text="Food").grid(row=0, column=0)
entry = Entry(frame1 ,textvariable=ef_ht).grid(row=0, column=1)

label = Label(frame1, text="Auto Haste").grid(row=1, column=0)
entry = Entry(frame1, textvariable=ah_ht).grid(row=1, column=1)

label = Label(frame1, text="Anti Paralyze").grid(row=2, column=0)
entry = Entry(frame1, textvariable=ap_ht).grid(row=2, column=1)

label = Label(frame1, text="Auto Utamo").grid(row=3, column=0)
entry = Entry(frame1, textvariable=au_ht).grid(row=3, column=1)

label = Label(frame1, text="Cura Vida").grid(row=4, column=0)
entry = Entry(frame1, textvariable=cv_perc).grid(row=4, column=1)
entry = Entry(frame1, textvariable=cv_ht).grid(row=4, column=2)
entry = Entry(frame1, textvariable=cv_perc_sup).grid(row=5, column=1)
entry = Entry(frame1, textvariable=cv_ht_sup).grid(row=5, column=2)

label = Label(frame1, text="Mana Train").grid(row=6, column=0)
entry = Entry(frame1, textvariable=mt_perc).grid(row=6, column=1)
entry = Entry(frame1, textvariable=mt_ht).grid(row=6, column=2)

label = Label(frame1, text="Cura Mana").grid(row=7, column=0)
entry = Entry(frame1, textvariable=cm_perc).grid(row=7, column=1)
entry = Entry(frame1, textvariable=cm_ht).grid(row=7, column=2)

label = Label(frame1, text="Auto Ring").grid(row=8, column=0)
entry = Entry(frame1, textvariable=ar_ht).grid(row=8, column=1)

#Butões
var1 = IntVar()
var2 = IntVar()
var3 = IntVar()
var4 = IntVar()
var5 = IntVar()
var6 = IntVar()
var7 = IntVar()
var8 = IntVar()
var11 = IntVar()
var12 = IntVar()
var13 = IntVar()
var15 = IntVar()
Checkbutton(frame2, text='Food', variable=var1, onvalue=1, offvalue=0,command=food).pack(anchor=W)
Checkbutton(frame2, text='Haste', variable=var2, onvalue=1, offvalue=0,command=haste).pack(anchor=W)
Checkbutton(frame2, text='Anti Paralyze', variable=var6, onvalue=1, offvalue=0,command=paralyze).pack(anchor=W)
Checkbutton(frame2, text='Utamo', variable=var7, onvalue=1, offvalue=0,command=utamo).pack(anchor=W)
Checkbutton(frame2, text='Cura Vida', variable=var3, onvalue=1, offvalue=0,command=cura_vida).pack(anchor=W)
Checkbutton(frame2, text='Cura Mana', variable=var8, onvalue=1, offvalue=0,command=cura_mana).pack(anchor=W)
Checkbutton(frame2, text='Mana Train', variable=var4, onvalue=1, offvalue=0,command=mana_train).pack(anchor=W)
Checkbutton(frame2, text='Background', variable=var5, onvalue=1, offvalue=0,command=background).pack(anchor=W)
Checkbutton(frame2, text='Auto Ring', variable=var11, onvalue=1, offvalue=0,command=ring).pack(anchor=W)
Checkbutton(frame2, text='Player Alert', variable=var12, onvalue=1, offvalue=0,command=player_on_screen).pack(anchor=W)
Checkbutton(frame2, text='Private Mensage', variable=var13, onvalue=1, offvalue=0,command=private_mensage).pack(anchor=W)
Checkbutton(frame2, text='Anti Bot', variable=var15, onvalue=1, offvalue=0,command=anti_bot).pack(anchor=W)

frame2.grid(row=0, column=3, rowspan=8, padx=30)

Button1 = Button(frame1, text="Atualizar HotKeys", command=atualizar).grid(row=0, column=2,rowspan=2,ipadx=5,ipady=5)

##################################################
# Frame Target
##################################################
frame3 = Label(tab2, bg='#dddddd')
frame3.pack()
frame4 = LabelFrame(frame3, text='On/Off')

label = Label(frame3, text="Mana Min.").grid(row=0, column=0)
entry = Entry(frame3, textvariable=min_mana_atack).grid(row=0, column=1)

label = Label(frame3, text="Magia 1").grid(row=1, column=0)
entry = Entry(frame3, textvariable=spell_um_nm).grid(row=1, column=1)
entry = Entry(frame3, textvariable=spell_um_ht).grid(row=1, column=2)

label = Label(frame3, text="Magia 2").grid(row=2, column=0)
entry = Entry(frame3, textvariable=spell_dois_nm).grid(row=2, column=1)
entry = Entry(frame3, textvariable=spell_dois_ht).grid(row=2, column=2)

label = Label(frame3, text="Magia 3").grid(row=3, column=0)
entry = Entry(frame3, textvariable=spell_tres_nm).grid(row=3, column=1)
entry = Entry(frame3, textvariable=spell_tres_ht).grid(row=3, column=2)

var9 = IntVar()
Checkbutton(frame4, text='target', variable=var9, onvalue=1, offvalue=0,command=auto_atack).pack(anchor=W)
var14 = IntVar()
Checkbutton(frame4, text='auto loot', variable=var14, onvalue=1, offvalue=0,command=auto_loot).pack(anchor=W)
var16 = IntVar()
Checkbutton(frame4, text='follow on', variable=var16, onvalue=1, offvalue=0,command=follow_on).pack(anchor=W)
frame4.grid(row=0, column=3, rowspan=8, padx=30)

##################################################
# Frame Cavebot
##################################################

open('cavebot.txt', 'w').close()

frame5 = Label(tab3, bg='#dddddd')
frame5.pack()
Button2 = Button(frame5, text="Encerrar", command=kill_programa).grid(row=0, column=2,rowspan=2,ipadx=5,ipady=5)

frame8 = Label(tab3)
frame8.pack()

frame7 = LabelFrame(tab3, text='')
frame7.pack()
btn_txt=StringVar()
Button3 = Button(frame7, textvariable=btn_txt, command=abrir_lista_scripts).grid(row=0, column=3,rowspan=3,ipadx=5,ipady=5)
btn_txt.set("selecionar script")

frame6 = LabelFrame(frame5, text='On/Off')
var10 = IntVar()
Checkbutton(frame6, text='cavebot', variable=var10, onvalue=1, offvalue=0,command=cavebot).pack(anchor=W)
frame6.grid(row=0, column=3, rowspan=8, padx=30)

frame9 = Label(tab3)
frame9.pack()

st = Label(tab3,justify=LEFT)
st.pack()
def atualizar():
    f = open("cavebot.txt").readlines()
    st.config(text=f)
    st.after(1000,atualizar)
atualizar()

##################################################
# Frame Editor de Cavebot
##################################################

frame10 = Label(tab4, bg='#dddddd')
frame10.pack()
script_name=StringVar()
label10= Label(frame10, text="Nome do Script")
label10.grid(row=0, column=0)
entry10 = Entry(frame10,text='' ,textvariable=script_name)
entry10.grid(row=0, column=1)
text10 = Label(frame10,text='     ',bg='#dddddd')
text10.grid(row=0, column=2)
Button10 = Button(frame10,text='Salvar',command=salvar_script)
Button10.grid(row=0, column=3)
Button11 = Button(frame10,text='Abrir',command=abrir_script)
Button11.grid(row=0, column=4)
Button12 = Button(frame10,text='Novo',command=limpar_script)
Button12.grid(row=0, column=5)
Button13 = Button(frame10,text='Delete',command=delete_script)
Button13.grid(row=0, column=6)

frame11 = Label(tab4)
frame11.pack()

langs = ('goto_label','ajustar_zoom','andar_waypoint','click_sqm','press_tecla','say','comprar_item','vender_itens','label','verifica_andar','verifica_supply','verifica_ammo','target_on','target_off','sleep','check_supply_vai','check_ammo_vai','stop_script','depositar_dp','haste_on','haste_off')
dct = {'goto_label':'goto(<nome da label>)','ajustar_zoom':'ajustar_zoom(<nivel do zoom>)','andar_waypoint':'andar("<nome do waypoint>")','click_sqm':'click_sqm(<sqm>,<botao>,<vezes>)','press_tecla':'press_tecla("<tecla>",<vezes>)','say':'say("<texto>")','comprar_item':'comprar_item(<"nome da img">,<"abreviacao">,<"qtde">)','vender_itens':'vender_itens()','label':'label_<nome da label>','verifica_andar':'andar_get()','verifica_supply':'check_supply("<nome do supply>")','verifica_ammo':'check_ammo()','target_on':'target_on()','target_off':'target_off()','sleep':'sleep(<segundos>)','check_supply_vai':'check_supply_vai(<supply>,<valor>,<label>)','check_ammo_vai':'check_ammo_vai(<ammo>,<valor>,<label>)','stop_script':'stop_script()','depositar_dp':'deposit_dp()','haste_on':'haste_on()','haste_off':'haste_off()'}

var = Variable(value=langs)

listbox = Listbox(
    tab4,
    listvariable=var,
    height=6,
    selectmode=EXTENDED)

listbox.pack(expand=True, fill=BOTH, side=LEFT,anchor='center')

# link a scrollbar to a list
scrollbar = ttk.Scrollbar(
    tab4,
    orient=VERTICAL,
    command=listbox.yview
)

listbox['yscrollcommand'] = scrollbar.set

scrollbar.pack(side=LEFT, expand=True, fill=Y)

@with_goto
def items_selected(event):
    # get selected indices
    selected_indices = listbox.curselection()
    # get selected items
    selected_langs = ",".join([listbox.get(i) for i in selected_indices])
    key_lst=list(dct.keys())
    values_lst=list(dct.values())
    if selected_langs!='' and selected_langs!=f'n':
        key_value=key_lst.index(selected_langs)
        index_value=values_lst[key_value]

        frame11.insert(END,index_value+';\n')

listbox.bind('<<ListboxSelect>>', items_selected)

frame11 = Text(tab4)
frame11.pack()

##################################################

window.mainloop()

######################################################################
#Comandos antes de fechar
######################################################################

kill_programa()


# -*- coding: utf-8 -*-
import requests
from threading import Thread
import time

url = 'https://www.mari0er.club/'
exitFlag = 0

class myThread (Thread):   #继承父类threading.Thread
    def __init__(self,id, url):
        super().__init__()
        self.id = id
        self.url = url
        self.start()
    def run(self):#把要执行的代码写到run函数里面 线程在创建后会直接运行run函数
        i = 0
        while i<10000:
            r = requests.get(self.url)
            print ('this is thread %s'%self.id,self.url)
            i += 1
            # time.sleep(1)
# 创建新线程
star = time.time()
for i in range(0,10):
    myThread(i,url)

# # 开启线程
# thread1.start()
# thread2.start()
end = time.time()
print (star - end)


from keras.layers import Input, Dense, LeakyReLU, Concatenate, Reshape, Conv2D, Flatten
from keras.models import Model
from keras.optimizers import Adam
import keras.backend as K
import tensorflow as tf

class CriticNetwork(object):
    def __init__(self, sess, state_size, action_size, tau, lr):
        self.sess = sess
        self.tau = tau
        self.lr = lr
        self.action_size = action_size
        self.state_size = state_size

        K.set_session(sess)

        # Now create the model
        self.model, self.action, self.state = self.create_critic_network()
        self.action_grads = tf.gradients(self.model.output, self.action)
        self.sess.run(tf.initialize_all_variables())

    def gradients(self, states, actions):
        return self.sess.run(self.action_grads, feed_dict={
            self.state: states,
            self.action: actions
        })[0]

    # def target_train(self):
    #     critic_weights = self.model.get_weights()
    #     critic_target_weights = self.target_model.get_weights()
    #     for i in range(len(critic_weights)):
    #         critic_target_weights[i] = self.tau * critic_weights[i] + (1 - self.tau) * critic_target_weights[i]
    #     self.target_model.set_weights(critic_target_weights)

    def create_critic_network(self):
        state_input = Input(shape=(self.state_size,))
        action_input = Input(shape=(self.action_size,))

        main_network = Dense(2048)(state_input)
        main_network = LeakyReLU()(main_network)

        concat_network = Concatenate()([main_network, action_input])
        concat_network = Dense(2048)(concat_network)
        concat_network = LeakyReLU()(concat_network)
        concat_network = Dense(2048)(concat_network)
        concat_network = LeakyReLU()(concat_network)

        value_prediction = Dense(1, name='value_prediction')(concat_network)
        critic = Model(inputs=[state_input, action_input], outputs=value_prediction)
        critic.compile(optimizer=Adam(lr=self.lr), loss='mse')
        critic.summary()

        return critic, action_input, state_input


import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange, repeat

########################################################################################

class Residual(nn.Module):
    def __init__(self, fn):
        super().__init__()
        self.fn = fn

    def forward(self, x, **kwargs):
        return self.fn(x, **kwargs) + x

class PreNorm(nn.Module):
    def __init__(self, dim, fn):
        super().__init__()
        self.norm = nn.LayerNorm(dim)
        self.fn = fn

    def forward(self, x, **kwargs):
        return self.fn(self.norm(x), **kwargs)

class FeedForward(nn.Module):
    def __init__(self, dim, hidden_dim, dropout=0.):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(dim, hidden_dim),
            nn.ReLU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, dim),
            nn.Dropout(dropout)
        )

    def forward(self, x):
        return self.net(x)

class Attention(nn.Module):
    def __init__(self, dim, heads=8, dropout=0.):
        super().__init__()
        self.heads = heads
        self.scale = dim ** -0.5

        self.to_qkv = nn.Linear(dim, dim * 3, bias=False)
        self.to_out = nn.Sequential(
            nn.Linear(dim, dim),
            nn.Dropout(dropout)
        )

    def forward(self, x, mask=None):
        b, n, _, h = *x.shape, self.heads
        qkv = self.to_qkv(x).chunk(3, dim=-1)
        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=h), qkv)

        dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale

        if mask is not None:
            mask = F.pad(mask.flatten(1), (1, 0), value=True)
            assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions'
            mask = mask[:, None, :] * mask[:, :, None]
            dots.masked_fill_(~mask, float('-inf'))
            del mask

        attn = dots.softmax(dim=-1)

        out = torch.einsum('bhij,bhjd->bhid', attn, v)
        out = rearrange(out, 'b h n d -> b n (h d)')
        out = self.to_out(out)
        return out

class Transformer(nn.Module):
    def __init__(self, dim, depth, heads, mlp_dim, dropout):
        super().__init__()
        self.layers = nn.ModuleList([])
        for _ in range(depth):
            self.layers.append(nn.ModuleList([
                Residual(PreNorm(dim, Attention(dim, heads=heads, dropout=dropout))),
                Residual(PreNorm(dim, FeedForward(dim, mlp_dim, dropout=dropout)))
            ]))

    def forward(self, x, mask=None):
        for attn, ff in self.layers:
            x = attn(x, mask=mask)
            x = ff(x)
        return x

class Seq_Transformer(nn.Module):
    def __init__(self, *, patch_size, dim, depth, heads, mlp_dim, channels=1, dropout=0.1):
        super().__init__()
        patch_dim = channels * patch_size
        self.patch_to_embedding = nn.Linear(patch_dim, dim)
        self.c_token = nn.Parameter(torch.randn(1, 1, dim))
        self.transformer = Transformer(dim, depth, heads, mlp_dim, dropout)
        self.to_c_token = nn.Identity()

    def forward(self, forward_seq):
        x = self.patch_to_embedding(forward_seq)
        b, n, _ = x.shape
        c_tokens = repeat(self.c_token, '() n d -> b n d', b=b)
        x = torch.cat((c_tokens, x), dim=1)
        x = self.transformer(x)
        c_t = self.to_c_token(x[:, 0])
        return c_t


# pip install requests
# pip install matplotlib

import requests
import urllib.request
from urllib import parse
import json
import matplotlib.pyplot as plt
import numpy as np

# https://www.data.go.kr 에서 발급 받은 API_KEY
API_AUTH_KEY = "API_KEY"

"""
https://www.data.go.kr/dataset/15000581/openapi.do

3. 시도별 실시간 측정정보 조회
시도명을 검색조건으로 하여 시도별 측정소목록에 대한
일반 항목과 CAI 최종 실시간 측정값과 지수 정보 조회 기능을 제공하는 시도별 실시간 측정정보 조회

Request Parameter
numOfRows   10      0   한 페이지 결과 수
pageNo      1       0   페이지 번호
sidoName    세종    1   시도 이름 (서울, 부산, 대구, 인천, 광주, 대전, 울산, 경기, 강원, 충북, 충남, 전북, 전남, 경북, 경남, 제주, 세종)
ver         1.3     0   버전별 상세 결과 참고문서 참조

http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getCtprvnRltmMesureDnsty?
serviceKey=API_AUTH_KEY
&numOfRows=10
&sidoName=%EC%84%B8%EC%A2%85
&ver=1.3
"""

# 시도별 실시간 측정정보를 조회 할 수 있는 REST BASE_URL
getCtprvnRltmMesureDnsty = "http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getCtprvnRltmMesureDnsty"

# 시도별 실시간 측정정보를 요청할때 붙은 파라미터 값들을 이용하여, url 를 만들어 주는 함수
def make_url_getCtprvnRltmMesureDnsty(sidoName, numOfRows=10):
    return "%s?serviceKey=%s&numOfRows=%d&sidoName=%s&ver=1.3&_returnType=json" % (getCtprvnRltmMesureDnsty, API_AUTH_KEY, numOfRows, parse.quote(sidoName))

# 세종시의 실시간 측정정보를 요청하는 url을 생성
target_url = make_url_getCtprvnRltmMesureDnsty('세종')

with urllib.request.urlopen(target_url) as url:  # 웹페이지에 접속해서 정보를 가져올 수 있는 urllib를 활용하여, json 정보를 가져옴
    data = json.loads(url.read().decode(url.headers.get_content_charset()))  # json 정보를 처리하고 쉽도록 decode 함 > dictionary type 과 같은 형태로 반환됨.

    """
    {'list': [{..., 'stationName': '신흥동', ...}, {..., 'stationName': '아름동', ...}, {...}, {...}]}
    """

    # 여러 동이 나오지만, 학교가 있는 아름동만 추출하기 위한 코드
    my_place = ''
    for sub_data in data['list']:
        if sub_data['stationName'] == '아름동':
            my_place = sub_data
            break

    # json 으로 받은 많은 데이터중에 출력하지 않을 정보들을 제외
    data_column = list(my_place)
    no_use_column = "stationName dataTime dataTerm mangName pageNo resultCode resultMsg rnum serviceKey sidoName stationCode stationName totalCount ver _returnType"

    print("[ 실시간 공기질 정보 | %s | %s]" % (my_place['stationName'], my_place['dataTime']))
    print("[통합대기환경지수] %s 등급" % my_place['khaiGrade'])

    for column_name in data_column:
        if column_name not in no_use_column.split():
            # print("%s: %s" % (column_name, my_place[column_name]))
            if  'Grade' in column_name:
                if 'co' in column_name:
                    print("%s: \t%s등급" % ("[일산화탄소]", my_place[column_name]))
                elif 'no2' in column_name:
                    print("%s: \t%s등급" % ("[이산화질소]", my_place[column_name]))
                elif 'pm10Grade1h' in column_name:
                    print("%s: \t%s등급" % ("[PM10]", my_place[column_name]))
                elif 'pm25Grade1h' in column_name:
                    print("%s: \t%s등급" % ("[PM25]", my_place[column_name]))
                elif 'so2' in column_name:
                    print("%s: \t%s등급" % ("[아황산가스]", my_place[column_name]))
                elif 'o3' in column_name:
                    print("%s: \t%s등급" % ("[오존]", my_place[column_name]))

"""
https://www.data.go.kr/dataset/15000581/openapi.do

5. 시도별 실시간 평균정보 조회

시도별 측정소목록에 대한 일반 항목의 시간 및 일평균 자료 및 지역 평균 정보를 제공하는
시도별 실시간 평균정보 조회

numOfRows	    한 페이지 결과 수   4	0	10	한 페이지 결과 수
pageNo	        페이지 번호         4	0	1	페이지 번호
itemCode	    항목명	            10	1	PM10	측정항목 구분(SO2, CO, O3, NO2, PM10, PM25)
dataGubun	    자료 구분	        10	1	HOUR	요청 자료 구분(시간평균 : HOUR, 일평균 : DAILY)
searchCondition	데이터 기간	        10	0	MONTH	요청 데이터기간 (일주일 : WEEK, 한달 : MONTH)

http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getCtprvnMesureLIst?
itemCode=PM10&dataGubun=DAILY
&searchCondition=MONTH
&pageNo=1
&numOfRows=10
&ServiceKey=서비스키
"""

# 시도별 실시간 평균 정보를 조회하기 위한 BASE_URL
getCtprvnMesureList = "http://openapi.airkorea.or.kr/openapi/services/rest/ArpltnInforInqireSvc/getCtprvnMesureLIst"

# 시도별 실시간 평균 정보를 조회하기 위해 파라미터를 기준으로 요청할 url을 만들어주는 함수
def make_url_getCtprvnMesureList(itemCode, dataGubun, searchCondition = 'WEEK'):
    return "%s?serviceKey=%s&itemCode=%s&dataGubun=%s&searchCondition=%s&_returnType=json" % (getCtprvnMesureList, API_AUTH_KEY, itemCode, dataGubun, searchCondition)

target_url = make_url_getCtprvnMesureList('PM10', 'DAILY', 'WEEK')

with urllib.request.urlopen(target_url) as url:
    data = json.loads(url.read().decode(url.headers.get_content_charset()))

    sort_data = data['list'][::-1]

    in_date = []
    city_name = []
    city_air_contidion_data = []

    for city in list(sort_data[0]):
        if city not in "_returnType dataGubun dataTerm dataTime itemCode numOfRows pageNo resultCode resultMsg searchCondition serviceKey totalCount".split():
            city_name.append(city)

    for i in range(len(city_name)):
        city_air_contidion_data.append([])

    for sub_data in sort_data:
        in_date.append(sub_data['dataTime'][5:10])
        for city in list(city_name):
            city_air_contidion_data[city_name.index(city)].append(int(sub_data[city]))

    for i in range(0, len(city_name)):
        if city_name[i] in "seoul sejong jeju".split():
            plt.plot(in_date, city_air_contidion_data[i], label=city_name[i])

    plt.title('PM10', loc='center')
    plt.xlabel('date')
    plt.ylabel('value')
    plt.legend()
    plt.grid(True)

    plt.show()


from __future__ import print_function

import getopt
import glob
import os
import re
import string
import sys
import xml

from xml.dom.minidom import parse

try:
  _basestr = basestring
except NameError:
  _basestr = str

# Dictionary of subtitution args
substitution_args_context = {}

class XacroException(Exception):
  pass

def isnumber(x):
  return hasattr(x, '__int__')

# Better pretty printing of xml
# Taken from http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-and-silly-whitespace/
def fixed_writexml(self, writer, indent="", addindent="", newl=""):
  # indent = current indentation
  # addindent = indentation to add to higher levels
  # newl = newline string
  writer.write(indent + "<" + self.tagName)

  attrs = self._get_attributes()
  a_names = list(attrs.keys())
  a_names.sort()

  for a_name in a_names:
    writer.write(" %s=\"" % a_name)
    xml.dom.minidom._write_data(writer, attrs[a_name].value)
    writer.write("\"")
  if self.childNodes:
    if len(self.childNodes) == 1 \
       and self.childNodes[0].nodeType == xml.dom.minidom.Node.TEXT_NODE:
      writer.write(">")
      self.childNodes[0].writexml(writer, "", "", "")
      writer.write("</%s>%s" % (self.tagName, newl))
      return
    writer.write(">%s" % (newl))
    for node in self.childNodes:
      # skip whitespace-only text nodes
      if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
         not node.data.strip():
        continue
      node.writexml(writer, indent + addindent, addindent, newl)
    writer.write("%s</%s>%s" % (indent, self.tagName, newl))
  else:
    writer.write("/>%s" % (newl))

# replace minidom's function with ours
xml.dom.minidom.Element.writexml = fixed_writexml

class Table:

  def __init__(self, parent=None):
    self.parent = parent
    self.table = {}

  def __getitem__(self, key):
    if key in self.table:
      return self.table[key]
    elif self.parent:
      return self.parent[key]
    else:
      raise KeyError(key)

  def __setitem__(self, key, value):
    self.table[key] = value

  def __contains__(self, key):
    return \
        key in self.table or \
        (self.parent and key in self.parent)

class QuickLexer(object):

  def __init__(self, **res):
    self.str = ""
    self.top = None
    self.res = []
    for k, v in res.items():
      self.__setattr__(k, len(self.res))
      self.res.append(v)

  def lex(self, str):
    self.str = str
    self.top = None
    self.next()

  def peek(self):
    return self.top

  def next(self):
    result = self.top
    self.top = None
    for i in range(len(self.res)):
      m = re.match(self.res[i], self.str)
      if m:
        self.top = (i, m.group(0))
        self.str = self.str[m.end():]
        break
    return result

def first_child_element(elt):
  c = elt.firstChild
  while c:
    if c.nodeType == xml.dom.Node.ELEMENT_NODE:
      return c
    c = c.nextSibling
  return None

def next_sibling_element(elt):
  c = elt.nextSibling
  while c:
    if c.nodeType == xml.dom.Node.ELEMENT_NODE:
      return c
    c = c.nextSibling
  return None

# Pre-order traversal of the elements
def next_element(elt):
  child = first_child_element(elt)
  if child:
    return child
  while elt and elt.nodeType == xml.dom.Node.ELEMENT_NODE:
    next = next_sibling_element(elt)
    if next:
      return next
    elt = elt.parentNode
  return None

# Pre-order traversal of all the nodes
def next_node(node):
  if node.firstChild:
    return node.firstChild
  while node:
    if node.nextSibling:
      return node.nextSibling
    node = node.parentNode
  return None

def child_nodes(elt):
  c = elt.firstChild
  while c:
    yield c
    c = c.nextSibling

all_includes = []

# Deprecated message for <include> tags that don't have <xacro:include> prepended:
deprecated_include_msg = """DEPRECATED IN HYDRO:
  The <include> tag should be prepended with 'xacro' if that is the intended use
  of it, such as <xacro:include ...>. Use the following script to fix incorrect
  xacro includes:
     sed -i 's/<include/<xacro:include/g' `find . -iname *.xacro`"""

include_no_matches_msg = """Include tag filename spec \"{}\" matched no files."""

## @throws XacroException if a parsing error occurs with an included document
def process_includes(doc, base_dir):
  namespaces = {}
  previous = doc.documentElement
  elt = next_element(previous)
  while elt:
    # Xacro should not use plain 'include' tags but only namespaced ones. Causes conflicts with
    # other XML elements including Gazebo's <gazebo> extensions
    is_include = False
    if elt.tagName == 'xacro:include' or elt.tagName == 'include':

      is_include = True
      # Temporary fix for ROS Hydro and the xacro include scope problem
      if elt.tagName == 'include':

        # check if there is any element within the <include> tag. mostly we are concerned
        # with Gazebo's <uri> element, but it could be anything. also, make sure the child
        # nodes aren't just a single Text node, which is still considered a deprecated
        # instance
        if elt.childNodes and not (len(elt.childNodes) == 1 and
                                   elt.childNodes[0].nodeType == elt.TEXT_NODE):
          # this is not intended to be a xacro element, so we can ignore it
          is_include = False
        else:
          # throw a deprecated warning
          print(deprecated_include_msg, file=sys.stderr)

    # Process current element depending on previous conditions
    if is_include:
      filename_spec = eval_text(elt.getAttribute('filename'), {})
      if not os.path.isabs(filename_spec):
        filename_spec = os.path.join(base_dir, filename_spec)

      if re.search('[*[?]+', filename_spec):
        # Globbing behaviour
        filenames = sorted(glob.glob(filename_spec))
        if len(filenames) == 0:
          print(include_no_matches_msg.format(filename_spec), file=sys.stderr)
      else:
        # Default behaviour
        filenames = [filename_spec]

      for filename in filenames:
        global all_includes
        all_includes.append(filename)
        try:
          with open(filename) as f:
            try:
              included = parse(f)
            except Exception as e:
              raise XacroException(
                  "included file \"%s\" generated an error during XML parsing: %s" %
                  (filename, str(e)))
        except IOError as e:
          raise XacroException("included file \"%s\" could not be opened: %s" % (filename, str(e)))

        # Replaces the include tag with the elements of the included file
        for c in child_nodes(included.documentElement):
          elt.parentNode.insertBefore(c.cloneNode(deep=True), elt)

        # Grabs all the declared namespaces of the included document
        for name, value in included.documentElement.attributes.items():
          if name.startswith('xmlns:'):
            namespaces[name] = value

      elt.parentNode.removeChild(elt)
      elt = None
    else:
      previous = elt

    elt = next_element(previous)

  # Makes sure the final document declares all the namespaces of the included documents.
  for k, v in namespaces.items():
    doc.documentElement.setAttribute(k, v)

# Returns a dictionary: { macro_name => macro_xml_block }
def grab_macros(doc):
  macros = {}

  previous = doc.documentElement
  elt = next_element(previous)
  while elt:
    if elt.tagName == 'macro' or elt.tagName == 'xacro:macro':
      name = elt.getAttribute('name')

      macros[name] = elt
      macros['xacro:' + name] = elt

      elt.parentNode.removeChild(elt)
      elt = None
    else:
      previous = elt

    elt = next_element(previous)
  return macros

# Returns a Table of the properties
def grab_properties(doc):
  table = Table()

  previous = doc.documentElement
  elt = next_element(previous)
  while elt:
    if elt.tagName == 'property' or elt.tagName == 'xacro:property':
      name = elt.getAttribute('name')
      value = None

      if elt.hasAttribute('value'):
        value = elt.getAttribute('value')
      else:
        name = '**' + name
        value = elt  # debug

      bad = string.whitespace + "${}"
      has_bad = False
      for b in bad:
        if b in name:
          has_bad = True
          break

      if has_bad:
        sys.stderr.write('Property names may not have whitespace, ' + '"{", "}", or "$" : "' +
                         name + '"')
      else:
        table[name] = value

      elt.parentNode.removeChild(elt)
      elt = None
    else:
      previous = elt

    elt = next_element(previous)
  return table

def eat_ignore(lex):
  while lex.peek() and lex.peek()[0] == lex.IGNORE:
    lex.next()

def eval_lit(lex, symbols):
  eat_ignore(lex)
  if lex.peek()[0] == lex.NUMBER:
    return float(lex.next()[1])
  if lex.peek()[0] == lex.SYMBOL:
    try:
      key = lex.next()[1]
      value = symbols[key]
    except KeyError as ex:
      raise XacroException("Property wasn't defined: %s" % str(ex))
    if not (isnumber(value) or isinstance(value, _basestr)):
      if value is None:
        raise XacroException("Property %s recursively used" % key)
      raise XacroException("WTF2")
    try:
      return int(value)
    except:
      try:
        return float(value)
      except:
        # prevent infinite recursion
        symbols[key] = None
        result = eval_text(value, symbols)
        # restore old entry
        symbols[key] = value
        return result
  raise XacroException("Bad literal")

def eval_factor(lex, symbols):
  eat_ignore(lex)

  neg = 1
  if lex.peek()[1] == '-':
    lex.next()
    neg = -1

  if lex.peek()[0] in [lex.NUMBER, lex.SYMBOL]:
    return neg * eval_lit(lex, symbols)
  if lex.peek()[0] == lex.LPAREN:
    lex.next()
    eat_ignore(lex)
    result = eval_expr(lex, symbols)
    eat_ignore(lex)
    if lex.next()[0] != lex.RPAREN:
      raise XacroException("Unmatched left paren")
    eat_ignore(lex)
    return neg * result

  raise XacroException("Misplaced operator")

def eval_term(lex, symbols):
  eat_ignore(lex)

  result = 0
  if lex.peek()[0] in [lex.NUMBER, lex.SYMBOL, lex.LPAREN] \
          or lex.peek()[1] == '-':
    result = eval_factor(lex, symbols)

  eat_ignore(lex)
  while lex.peek() and lex.peek()[1] in ['*', '/']:
    op = lex.next()[1]
    n = eval_factor(lex, symbols)

    if op == '*':
      result = float(result) * float(n)
    elif op == '/':
      result = float(result) / float(n)
    else:
      raise XacroException("WTF")
    eat_ignore(lex)
  return result

def eval_expr(lex, symbols):
  eat_ignore(lex)

  op = None
  if lex.peek()[0] == lex.OP:
    op = lex.next()[1]
    if not op in ['+', '-']:
      raise XacroException("Invalid operation. Must be '+' or '-'")

  result = eval_term(lex, symbols)
  if op == '-':
    result = -float(result)

  eat_ignore(lex)
  while lex.peek() and lex.peek()[1] in ['+', '-']:
    op = lex.next()[1]
    n = eval_term(lex, symbols)

    if op == '+':
      result = float(result) + float(n)
    if op == '-':
      result = float(result) - float(n)
    eat_ignore(lex)
  return result

def eval_text(text, symbols):

  def handle_expr(s):
    lex = QuickLexer(IGNORE=r"\s+",
                     NUMBER=r"(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?",
                     SYMBOL=r"[a-zA-Z_]\w*",
                     OP=r"[\+\-\*/^]",
                     LPAREN=r"\(",
                     RPAREN=r"\)")
    lex.lex(s)
    return eval_expr(lex, symbols)

  def handle_extension(s):
    return ("$(%s)" % s)

  results = []
  lex = QuickLexer(DOLLAR_DOLLAR_BRACE=r"\$\$+\{",
                   EXPR=r"\$\{[^\}]*\}",
                   EXTENSION=r"\$\([^\)]*\)",
                   TEXT=r"([^\$]|\$[^{(]|\$$)+")
  lex.lex(text)
  while lex.peek():
    if lex.peek()[0] == lex.EXPR:
      results.append(handle_expr(lex.next()[1][2:-1]))
    elif lex.peek()[0] == lex.EXTENSION:
      results.append(handle_extension(lex.next()[1][2:-1]))
    elif lex.peek()[0] == lex.TEXT:
      results.append(lex.next()[1])
    elif lex.peek()[0] == lex.DOLLAR_DOLLAR_BRACE:
      results.append(lex.next()[1][1:])
  return ''.join(map(str, results))

# Expands macros, replaces properties, and evaluates expressions
def eval_all(root, macros, symbols):
  # Evaluates the attributes for the root node
  for at in root.attributes.items():
    result = eval_text(at[1], symbols)
    root.setAttribute(at[0], result)

  previous = root
  node = next_node(previous)
  while node:
    if node.nodeType == xml.dom.Node.ELEMENT_NODE:
      if node.tagName in macros:
        body = macros[node.tagName].cloneNode(deep=True)
        params = body.getAttribute('params').split()

        # Parse default values for any parameters
        defaultmap = {}
        for param in params[:]:
          splitParam = param.split(':=')

          if len(splitParam) == 2:
            defaultmap[splitParam[0]] = splitParam[1]
            params.remove(param)
            params.append(splitParam[0])

          elif len(splitParam) != 1:
            raise XacroException("Invalid parameter definition")

        # Expands the macro
        scoped = Table(symbols)
        for name, value in node.attributes.items():
          if not name in params:
            raise XacroException("Invalid parameter \"%s\" while expanding macro \"%s\"" %
                                 (str(name), str(node.tagName)))
          params.remove(name)
          scoped[name] = eval_text(value, symbols)

        # Pulls out the block arguments, in order
        cloned = node.cloneNode(deep=True)
        eval_all(cloned, macros, symbols)
        block = cloned.firstChild
        for param in params[:]:
          if param[0] == '*':
            while block and block.nodeType != xml.dom.Node.ELEMENT_NODE:
              block = block.nextSibling
            if not block:
              raise XacroException("Not enough blocks while evaluating macro %s" %
                                   str(node.tagName))
            params.remove(param)
            scoped[param] = block
            block = block.nextSibling

        # Try to load defaults for any remaining non-block parameters
        for param in params[:]:
          if param[0] != '*' and param in defaultmap:
            scoped[param] = defaultmap[param]
            params.remove(param)

        if params:
          raise XacroException("Parameters [%s] were not set for macro %s" %
                               (",".join(params), str(node.tagName)))
        eval_all(body, macros, scoped)

        # Replaces the macro node with the expansion
        for e in list(child_nodes(body)):  # Ew
          node.parentNode.insertBefore(e, node)
        node.parentNode.removeChild(node)

        node = None
      elif node.tagName == 'arg' or node.tagName == 'xacro:arg':
        name = node.getAttribute('name')
        if not name:
          raise XacroException("Argument name missing")
        default = node.getAttribute('default')
        if default and name not in substitution_args_context['arg']:
          substitution_args_context['arg'][name] = default

        node.parentNode.removeChild(node)
        node = None

      elif node.tagName == 'insert_block' or node.tagName == 'xacro:insert_block':
        name = node.getAttribute('name')

        if ("**" + name) in symbols:
          # Multi-block
          block = symbols['**' + name]

          for e in list(child_nodes(block)):
            node.parentNode.insertBefore(e.cloneNode(deep=True), node)
          node.parentNode.removeChild(node)
        elif ("*" + name) in symbols:
          # Single block
          block = symbols['*' + name]

          node.parentNode.insertBefore(block.cloneNode(deep=True), node)
          node.parentNode.removeChild(node)
        else:
          raise XacroException("Block \"%s\" was never declared" % name)

        node = None
      elif node.tagName in ['if', 'xacro:if', 'unless', 'xacro:unless']:
        value = eval_text(node.getAttribute('value'), symbols)
        try:
          if value == 'true': keep = True
          elif value == 'false': keep = False
          else: keep = float(value)
        except ValueError:
          raise XacroException(
              "Xacro conditional evaluated to \"%s\". Acceptable evaluations are one of [\"1\",\"true\",\"0\",\"false\"]"
              % value)
        if node.tagName in ['unless', 'xacro:unless']: keep = not keep
        if keep:
          for e in list(child_nodes(node)):
            node.parentNode.insertBefore(e.cloneNode(deep=True), node)

        node.parentNode.removeChild(node)
      else:
        # Evals the attributes
        for at in node.attributes.items():
          result = eval_text(at[1], symbols)
          node.setAttribute(at[0], result)
        previous = node
    elif node.nodeType == xml.dom.Node.TEXT_NODE:
      node.data = eval_text(node.data, symbols)
      previous = node
    else:
      previous = node

    node = next_node(previous)
  return macros

# Expands everything except includes
def eval_self_contained(doc):
  macros = grab_macros(doc)
  symbols = grab_properties(doc)
  eval_all(doc.documentElement, macros, symbols)

def print_usage(exit_code=0):
  print("Usage: %s [-o <output>] <input>" % 'xacro.py')
  print("       %s --deps       Prints dependencies" % 'xacro.py')
  print("       %s --includes   Only evalutes includes" % 'xacro.py')
  sys.exit(exit_code)

def set_substitution_args_context(context={}):
  substitution_args_context['arg'] = context

def open_output(output_filename):
  if output_filename is None:
    return sys.stdout
  else:
    return open(output_filename, 'w')

def main():
  try:
    opts, args = getopt.gnu_getopt(sys.argv[1:], "ho:", ['deps', 'includes'])
  except getopt.GetoptError as err:
    print(str(err))
    print_usage(2)

  just_deps = False
  just_includes = False

  output_filename = None
  for o, a in opts:
    if o == '-h':
      print_usage(0)
    elif o == '-o':
      output_filename = a
    elif o == '--deps':
      just_deps = True
    elif o == '--includes':
      just_includes = True

  if len(args) < 1:
    print("No input given")
    print_usage(2)

  # Process substitution args
  # set_substitution_args_context(load_mappings(sys.argv))
  set_substitution_args_context((sys.argv))

  f = open(args[0])
  doc = None
  try:
    doc = parse(f)
  except xml.parsers.expat.ExpatError:
    sys.stderr.write("Expat parsing error.  Check that:\n")
    sys.stderr.write(" - Your XML is correctly formed\n")
    sys.stderr.write(" - You have the xacro xmlns declaration: " +
                     "xmlns:xacro=\"http://www.ros.org/wiki/xacro\"\n")
    sys.stderr.write("\n")
    raise
  finally:
    f.close()

  process_includes(doc, os.path.dirname(args[0]))
  if just_deps:
    for inc in all_includes:
      sys.stdout.write(inc + " ")
    sys.stdout.write("\n")
  elif just_includes:
    doc.writexml(open_output(output_filename))
    print()
  else:
    eval_self_contained(doc)
    banner = [
        xml.dom.minidom.Comment(c) for c in [
            " %s " % ('=' * 83),
            " |    This document was autogenerated by xacro from %-30s | " % args[0],
            " |    EDITING THIS FILE BY HAND IS NOT RECOMMENDED  %-30s | " % "",
            " %s " % ('=' * 83)
        ]
    ]
    first = doc.firstChild
    for comment in banner:
      doc.insertBefore(comment, first)

    open_output(output_filename).write(doc.toprettyxml(indent='  '))
    print()

if __name__ == '__main__':
  main()


#!/usr/bin/env python
# coding: utf-8

#
# # Project: Investigate the reason of no-show appointment
#
# ## Table of Contents
#

#
Introduction

#
Data Wrangling

#
Exploratory Data Analysis

#
Conclusions

#

#
# ## Introduction
#
# ### Dataset Description
#
#
# This dataset collects information from 100k medical appointments in Brazil and is focused on the question of
# whether or not patients show up for their appointment. A number of characteristics about the patient are included in each row.
# ● ‘ScheduledDay’ tells us on what day the patient set up their appointment. ● ‘Neighborhood’ indicates the location of the hospital. ●
# ‘Scholarship’ indicates whether or not the patient is enrolled in Brasilian welfare program Bolsa Família.
# ● Be careful about the encoding of the last column: it says ‘No’ if the patient showed up to their appointment, and ‘Yes’ if they did not show up.
#
#

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.pyplot as plt
import seaborn as sns
get_ipython().run_line_magic('matplotlib', 'inline')

# In[ ]:

# Upgrade pandas to use dataframe.explode() function.
get_ipython().system('pip install --upgrade pandas==0.25.0')

# In[ ]:

# In[ ]:

# Load your data and print out a few lines. Perform operations to inspect data
#   types and look for instances of missing or possibly errant data.
df = pd.read_csv('noshowappointments-kagglev2-may-2016.csv')
pd.options.display.max_rows = 9999
df.head()

# In[ ]:

#explore the data
df.shape

# the data contain 110527 appointment and 14 different columns

# In[ ]:

df.describe()

# In[ ]:

df.dtypes

# this is the type pf data

# it appears that about32% people recived sms which is low and we must re-check the sms campaign,
# also the mean age is about 37 , probaply we have one or more row having error because of age which is(-1), we will check it
# also, the percentage of alcoholic, handcapped , chronic diseases and scolar shipped patient is low

# In[ ]:

#identify the error
error=df.query("Age==-1")
error

# this row is having error in entering age we may remove it in cleaning data to prevent error

# In[ ]:

#ispecting missing data
df.info()

# there is no missing data

# In[ ]:

#find if tere is a duplicate data
df.duplicated().sum()

# there is no duplicate rows(data) found

# In[ ]:

#find the no of unique patient ID
df['PatientId'].nunique()

# we have only 62299 unique patient id,
#   110527-62299= 48228 that  mean we have 48228 duplicate patient id who had different appointment dates

# In[ ]:

df['Age'].max()

# As we can see the maximum age is 115 it may be a reason of not attednding

# In[ ]:

df.query('Age==115')

# will there is only 5 patient IDs with this age 2 of them has showed, 5 patient is not a signicant number, also appear that two has a unique IDs number and both of them has showed up at the end

# In[ ]:

df.sort_values(['Age'], ascending=[False])

# will it appear that the age Doesnot affect attendance except for young people because of parental care, we still need further investigation

# In[ ]:

#find the most repeated data
df.mode().head(1)

# it appears from mode that most of patient is female and the most appointment and scheduled date is in july 2016, however most people donot recieve a sms, also Most of people is children come with their parents(high parental care) (age =0)

# **cleaning data**
#
# From the data description and questions to answer, I've determined that some of the dataset columns are not necessary for the analysis process and will therefore be removed.
# PatientId,Sms_received,AppointmentID
# This will help to process the Data Analysis Faster.
# also we will remove age error and some misspelling.
# also we will remove duplicated id with duplicated data show
#

# In[ ]:

#Removing -1 age
df.drop(index=99832,inplace=True)

# In[ ]:

df.describe()

# here we have our age cleaned

# In[ ]:

#correction of column names
df.rename(columns={'Hipertension':'Hypertenstion'},inplace=True)
df.rename(columns={'No-show':'no_show'},inplace=True)
df.head()

# In[ ]:

#remove duplicated id and duplicated no-show status
df.drop_duplicates(['PatientId','no_show'],inplace=True)
df.shape

# In[ ]:

df.drop(['PatientId','AppointmentID','AppointmentDay','ScheduledDay'],axis=1,inplace=True)
df.head()
#we dropped appointment date and scheduled date because the time frame is small and not significant

#
# ## Exploratory Data Analysis
#
# > **Tip**: Now that you've trimmed and cleaned your data, you're ready to move on to exploration. **Compute statistics** and **create visualizations**
#
#
#
#
# **###General look###**

# In[ ]:

# Use this, and more code cells, to explore your data. Don't forget to add
#   Markdown cells to document your observations and findings.
df.hist(figsize=(25,7));

# ### Dividing the patient into two groups

# In[ ]:

#Dividing the patient into two groups(show and no show)
show=df.no_show=='No'
noshow=df.no_show=='Yes'
df[show].count(),df[noshow].count()

# In[ ]:

df[show]

# In[ ]:

df[noshow]

# it appears that the number of patient that attended the clinic after appointment is 54154 while who don't is 17663

# In[ ]:

df[show].mean(),df[noshow].mean()

# this show that the attended patient who recieved SMS is about 30 % while who don't is about 45%
# so we need to re-check the message sent to the patient

# In[ ]:

df[show].describe(),df[noshow].describe()

# from the previous data it shows that the average age is not signifcance to attendance or not this and other data will be discussed in details

# In[ ]:

df[show].mode(),df[noshow].mode()

# this data show that regarding to attendance the most neighbourhood shown is jardim camburai the same for non attendance
# also most patient arenot cronically ill or handcapped
# also, Most of the didnot recieve a sms

# In[ ]:

#let's divide people according to ther gender
male=df.Gender=='M'
female=df.Gender=='F'
df[male].count(),df[female].count()

# the number of males is 25350 while the number of female is 46466

# # INVISTIGATION and Questions On the Affecting Factor For The Attendance rate

# In[ ]:

df[male].mean(),df[female].mean()

# as you can see the numbers are very close so it's not a signifant one
# let's see the percentantage of attendance and absence in the gragh

# In[ ]:

#does the gender affect attendance
#we showed in previous that the female is more than male in appointment and gender
#let us see in graph to assure the percentage of attendance
plt.figure(figsize=[15,6])
plt.style.use('ggplot')
df['Gender'][show].value_counts(normalize=True).plot(kind='pie',label='show')
plt.legend();
plt.title('compare between attendance of gender')
plt.xlabel('Gender')
plt.ylabel('Patients Number');

#the female is more attendance because they made more appointment , we can confirm by calculating the absence
#if it's the same percentage so the gender is not affecting attendance

# In[ ]:

#let us see the percentage of absent
plt.figure(figsize=[15,6])
df['Gender'][noshow].value_counts(normalize=True).plot(kind='pie',label='noshow')
plt.legend();
plt.title('compare between absence of gender')
plt.xlabel('Gender')
plt.ylabel('Patients Number');

#as expected gender has no influence on presence or absence

# does the scolarships affect the attendance?

# In[ ]:

df.Scholarship[show].mean()

# In[ ]:

df.Scholarship[noshow].mean()

# as it shown the difeernce in both values of prescence is insignificance
# we will show it gragh

# In[ ]:

plt.figure(figsize=[20,10])
df.Scholarship[show].value_counts().plot(kind='bar',color='green',label='show',fontsize=20)
df.Scholarship[noshow].value_counts().plot(kind='bar',color='red',label='noshow',fontsize=20)
plt.legend();

# Does the handcap affect the attendance?

# In[ ]:

df.Handcap[show].mean()

# In[ ]:

df.Handcap[noshow].mean()

# the difference in mean between both numbers is very vey low which show the the handcap is not affecting the attendance

# In[ ]:

plt.figure(figsize=[60,20])
df.Age[show].value_counts().plot(kind='bar',color='green',label='show',fontsize=30)

plt.legend();

# we can see from here there s no effect of age in the show of patient except for children till 2 years(show high parentral care) and for old people show low show
# we need to see the percentage compared to no show for patients

# In[ ]:

plt.figure(figsize=[60,20])
df.Age[show].value_counts().plot(kind='bar',color='green',label='show',fontsize=30)
df.Age[noshow].value_counts().plot(kind='bar',color='red',label='noshow',fontsize=30)
plt.legend();

# here we can see the percentage of show to no show is the same with the exception of children who has higher percntage of attendance
# and the percentage of the old people is relatively lower which can allow us to provide services to help to bring the old people for the appointment and save their lives

# In[ ]:

plt.figure(figsize=[15,8])
df.Neighbourhood[show].value_counts().plot(kind='bar',color='green',label='show')
df.Neighbourhood[noshow].value_counts().plot(kind='bar',color='red',label='noshow')
plt.legend();
plt.title('compare according to neig.iloc[1-71815]hbourhood')
plt.xlabel('Neighbourhood')
plt.ylabel('Patients Number');

# In[ ]:

df[show].groupby('Gender').mean()['Age'],df[noshow].groupby('Gender').mean()['Age']

# it shown that the mean age for both gender is almost the same let's see in gragh

# In[ ]:

plt.figure(figsize=[15,7])
df[show].groupby('Gender').mean()['Age'].plot(kind='bar',color='green',label='show'),df[noshow].groupby('Gender').mean()['Age'].plot(kind='bar',color='red',label='noshow')
plt.legend()
plt.ylabel('mean age');

# the mean of ages is approximatly same so there is no correlation between gender ages and attendance

# In[ ]:

plt.figure(figsize=[15,7])
df[show].groupby(['Hypertenstion','Diabetes']).mean()['Age'].plot(kind='bar',color='green',label='show'),
df[noshow].groupby(['Hypertenstion','Diabetes']).mean()['Age'].plot(kind='bar',color='red',label='noshow')
plt.legend();
plt.title('Compare Acc. to mean ages and chronic diseases attendance',fontsize=25,loc = 'left')
plt.xticks(fontsize=20,rotation=360)
plt.annotate('Healthy patient',xy=(0,30),fontsize=14,xytext=(0,45),arrowprops=dict(facecolor='yellow'),color='green')
plt.annotate('Diabetic',xy=(1,53),fontsize=14,xytext=(1,65),arrowprops=dict(facecolor='yellow'),color='green')
plt.annotate('Hypertensive',xy=(2,59),fontsize=14,xytext=(2,66),arrowprops=dict(facecolor='yellow'),color='green')
plt.annotate('Both Diesases',xy=(3,62),fontsize=14,xytext=(2.7,69),arrowprops=dict(facecolor='yellow'),color='green')
plt.ylim([0,72])
plt.xlim([-1,4])
plt.axhline(y=31,xmin=-1,xmax=0.15,linewidth=2,marker=">",ms=7,linestyle= 'dashed',color='blue')
plt.axhline(y=53,xmin=-1,xmax=0.35,linewidth=2,marker=">",ms=7,linestyle= 'dashed',color='blue')
plt.axhline(y=60,xmin=-1,xmax=0.55,linewidth=2,marker=">",ms=7,linestyle= 'dashed',color='blue')
plt.axhline(y=63,xmin=-1,xmax=0.75,linewidth=2,marker=">",ms=7,linestyle= 'dashed',color='blue')
plt.xlabel('chronic diseases')
plt.ylabel('mean age');

# In[ ]:

df[show].groupby(['Hypertenstion','Diabetes']).mean()['Age']
df[noshow].groupby(['Hypertenstion','Diabetes']).mean()['Age']

# (0,0) mean the patient is healthy
# (0,1) mean the patient is Diabetic
# (1,0)mean the patient is hypertensive
# (1,1) mean the patient has both diseases
# we see that there s no correlation between disease and attendendance

# In[ ]:

df[show].groupby(['Hypertenstion','Diabetes']).mean()['Age'],df[noshow].groupby(['Hypertenstion','Diabetes']).mean()

# this Data confirm the shown in presentation, so chronic diseases is not affecting attendance

# In[ ]:

#we will remake sure that the age doesnot affect attendance by histogram
#does age affect attendance
plt.figure(figsize=[20,5])
df['Age'][show].hist(alpha=.5,bins=15,color='green',label='show')
df['Age'][noshow].hist(alpha=.5,bins=15,color='red',label='noshow')
plt.legend();
plt.title('compare different ages')
plt.xlabel('Age')
plt.ylabel('Patients Number');

# it show that young people (children) attend more which show the high parental care as it appeared in showing the mode value

# In[ ]:

#does receiveing sms affect the attendance
#we showed preveously that they are inversiouly propotional and we need to revise the texting team and the message sent but let us make sure by plotting
plt.figure(figsize=[20,5])
df['SMS_received'][show].hist(alpha=.5,bins=15,color='green',label='show')
df['SMS_received'][noshow].hist(alpha=.5,bins=15,color='red',label='noshow')
plt.legend();
plt.title('compare according to receiving SMS')
plt.xlabel('SMS')
plt.ylabel('Patients Number');

# as it appears more than 75 percent of patient who didnot recieve message has attended while about 50 percent of patient who recieved didnot recieve
# so there is an inverse propotion relation
# and we need to Re-check the sent text

# In[ ]:

#effect of neighbourhood
plt.figure(figsize=[15,8])
df.Neighbourhood[show].value_counts().plot(kind='bar',color='green',label='show')
df.Neighbourhood[noshow].value_counts().plot(kind='bar',color='red',label='noshow')
plt.legend();
plt.title('compare according to neighbourhood')
plt.xlabel('Neighbourhood')
plt.ylabel('Patients Number');

# jardam camburai has the highest attendance and show in number of patients

# In[ ]:

#mean age differ from one place to another?
plt.figure(figsize=[20,5])
df[show].groupby('Neighbourhood').Age.mean().plot(kind='bar',color='green',label='show')
df[noshow].groupby('Neighbourhood').Age.mean().plot(kind='bar',color='red',label='noshow')
plt.legend();
plt.title('compare according to neighbourhood age')
plt.xlabel('Neighbourhood')
plt.ylabel('Mean age');

# there is different ages shown in different neighboorhood which relatively affect attendance

# In[ ]:

#Does SMS goes to every neighboorhood on the same way and its effect?
plt.figure(figsize=[20,5])
df[show].groupby('Neighbourhood').SMS_received.mean().plot(kind='bar',color='green',label='show')
df[noshow].groupby('Neighbourhood').SMS_received.mean().plot(kind='bar',color='red',label='noshow')
plt.legend();
plt.title('compare according to neighbourhood SMS recieving')
plt.xlabel('Neighbourhood')
plt.ylabel('Patient Number');

# as shown some neighborhood who recieved message Made response while others is not

#
# ## Conclusions
#
# neighboorhood has a great effect in number of patient and attendance percentage,
# The number of patient in neighboorhood is affected by ages and sometimes sms,
# in general we see the more the patient grow the less they attend the resirvation as from 0  to 3 years there is great attendance which show the great parent care and the old people show slightly low attendance may be because they can't and we can facilitate their coming or we can home appointment for severly ill patient and help to save their lives,
# but overall the age hasn't a great effect in attendance except for children and old people alittle bit,
# female has higher resirvation and attendance rate but the percentage with male is almost the same
# limitation:No clear correlation between attendance and gender also chronic diseases doesnot affect it, handcapped patients and enrollment in welfare program also has little affect in enrollment
#
#

# Rescources that I used are many but most from: www.stackoverflow.com
# ,www.geeksforgeeks.org,
# www.stackoverflow.com,
# www.towardsdatascience.com,
# www.tutorialspoint.com,
# www.pandas.pydata.org

# In[ ]:

from subprocess import call
call(['python', '-m', 'nbconvert', 'Investigate_a_Dataset.ipynb'])