import networkx as nx
import numpy as np
import pandas as pd
from scipy.stats import hypergeom
from scipy.stats import binom
from scipy.linalg import eigvals
#TODO: MODIFY THE IMPORTS TO EXTERNAL IMPORTS

from .topology import node_degree

def closeness_connected_components(adj, neuron_properties=[], directed=False, return_sum=True):
    """Compute the closeness of each connected component of more than 1 vertex

    Parameters
    ----------
    adj : array_like
        Adjacency matrix of the graph
    directed : bool
        If `True`, will be computed using strongly connected components and directed closeness.
    return_sum : bool
        If `True`, only one list will be returned, by summing over all the connected components.

    Returns
    -------
    array_like
        A single array( if `return_sum=True`) or a list of arrays of shape `n`, containting closeness of vertices in that component, or 0 if the vertex is not in the component. Closeness cannot be zero otherwise.

    """
    from sknetwork.ranking import Closeness
    from scipy.sparse.csgraph import connected_components

    matrix=adj.toarray()
    if directed:
        n_comp, comp = connected_components(matrix, directed=True, connection="strong")
    else:
        n_comp, comp = connected_components(matrix, directed=False)
        matrix = matrix + matrix.T  # we need to make the matrix symmetric

    closeness = Closeness()  # if matrix is not symmetric automatically use directed
    n = matrix.shape[0]
    all_c = []
    for i in range(n_comp):
        c = np.zeros(n)
        idx = np.where(comp == i)[0]
        sub_mat = matrix[np.ix_(idx, idx)].tocsr()
        if sub_mat.getnnz() > 0:
            c[idx] = closeness.fit_transform(sub_mat)
            all_c.append(c)
    if return_sum:
        all_c = np.array(all_c)
        return np.sum(all_c, axis=0)
    else:
        return all_c

def communicability(adj, neuron_properties):
    pass

def closeness(adj, neuron_properties, directed=False):
    """Compute closeness centrality using sknetwork on all connected components or strongly connected
    component (if directed==True)"""
    return closeness_connected_components(adj, directed=directed)

def centrality(self, sub_gids, kind="closenesgit rebases", directed=False):
    """Compute a centrality of the graph. `kind` can be 'betweeness' or 'closeness'"""
    if kind == "closeness":
        return self.closeness(sub_gids, directed)
    else:
        ValueError("Kind must be 'closeness'!")
        #TODO:  Implement betweeness

def connected_components(adj,neuron_properties=[]):
    """Returns a list of the size of the connected components of the underlying undirected graph on sub_gids,
    if None, compute on the whole graph"""

    matrix=adj.toarray()
    matrix_und = np.where((matrix+matrix.T) >= 1, 1, 0)
    # TODO: Change the code from below to scipy implementation that seems to be faster!
    G = nx.from_numpy_matrix(matrix_und)
    return [len(c) for c in sorted(nx.connected_components(G), key=len, reverse=True)]

def core_number(adj, neuron_properties=[]):
    """Returns k core of directed graph, where degree of a vertex is the sum of in degree and out degree"""
    # TODO: Implement directed (k,l) core and k-core of underlying undirected graph (very similar to this)
    G = nx.from_numpy_matrix(adj.toarray())
    # Very inefficient (returns a dictionary!). TODO: Look for different implementation
    return nx.algorithms.core.core_number(G)

    # TODO: Filtered simplex counts with different weights on vertices (coreness, intersection)
    #  or on edges (strength of connection).

def density(adj, neuron_properties=[]):
    #Todo: Add #cells/volume as as possible spatial density
    adj=adj.astype('bool').astype('int')
    return adj.sum() / np.prod(adj.shape)

def __make_expected_distribution_model_first_order__(adj, direction="efferent"):
    #TODO: Document, utility function used in COMMON NEIGHBOURS ANALYSIS
    if direction == "efferent":
        N = adj.sum(axis=1).mean()
        M = adj.shape[1]
    elif direction == "afferent":
        N = adj.sum(axis=0).mean()
        M = adj.shape[0]
    else:
        raise ValueError("Unknown direction: {0}".format(direction))
    expected = hypergeom(M, N, N)
    return expected

def distribution_number_of_common_neighbors(adj, neuron_properties=None, direction="efferent"):
    # TODO: Document, utility function used in COMMON NEIGHBOURS ANALYSIS
    adj = adj.tocsc().astype(int)
    if direction == "efferent":
        cn = adj * adj.transpose()
    elif direction == "afferent":
        cn = adj.transpose() * adj
    else:
        raise ValueError("Unknown direction: {0}".format(direction))
    cn = np.array(cn.todense())
    cn = cn[np.triu_indices_from(cn, 1)]
    bins = np.arange(0, cn.max() + 2)
    return pd.Series(np.histogram(cn, bins=bins)[0], index=bins[:-1])

def normalized_distribution_of_common_neighbors(adj, neuron_properties=None, direction="efferent"):
    # TODO: Document, utility function used in COMMON NEIGHBOURS ANALYSIS
    data = distribution_number_of_common_neighbors(adj, neuron_properties, direction=direction)
    expected = __make_expected_distribution_model_first_order__(adj, direction=direction)
    expected = expected.pmf(data.index) * data.sum()
    expected = pd.Series(expected, index=data.index)
    return (data - expected) / (data + expected)

def overexpression_of_common_neighbors(adj, neuron_properties=None, direction="efferent"):
    # TODO: Document, utility function used in COMMON NEIGHBOURS ANALYSIS
    data = distribution_number_of_common_neighbors(adj, neuron_properties, direction=direction)
    data_mean = (data.index.values * data.values).sum() / data.values.sum()
    ctrl = __make_expected_distribution_model_first_order__(adj, direction=direction)
    ctrl_mean = ctrl.mean()
    return (data_mean - ctrl_mean) / (data_mean + ctrl_mean)

def common_neighbor_weight_bias(adj, neuron_properties=None, direction="efferent"):
    # TODO: Document, utility function used in COMMON NEIGHBOURS ANALYSIS
    adj_bin = (adj.tocsc() > 0).astype(int)
    if direction == "efferent":
        cn = adj_bin * adj_bin.transpose()
    elif direction == "afferent":
        cn = adj_bin.transpose() * adj_bin

    return np.corrcoef(cn[adj > 0],
                       adj[adj > 0])[0, 1]

def common_neighbor_connectivity_bias(adj, neuron_properties=None, direction="efferent",
                                      cols_location=None, fit_log=False):
    # TODO: Document, utility function used in COMMON NEIGHBOURS ANALYSIS
    import statsmodels.formula.api as smf
    from patsy import ModelDesc
    from scipy.spatial import distance

    if adj.dtype == bool:
        adj = adj.astype(int)

    if direction == "efferent":
        cn = adj * adj.transpose()
    elif direction == "afferent":
        cn = adj.transpose() * adj

    input_dict = {"CN": cn.toarray().flatten(),
                  "Connected": adj.astype(bool).toarray().flatten()}

    if fit_log:
        input_dict["CN"] = np.log10(input_dict["CN"] + fit_log)
    formula_str = "CN ~ Connected"
    if cols_location is not None:
        formula_str = formula_str + " + Distance"
        dmat = distance.squareform(distance.pdist(neuron_properties[cols_location].values))
        input_dict["Distance"] = dmat.flatten()
    sm_model = ModelDesc.from_formula(formula_str)

    sm_result = smf.ols(sm_model, input_dict).fit()

    pval = sm_result.pvalues.get("Connected[T.True]", 1.0)
    mdl_intercept = sm_result.params["Intercept"]
    mdl_added = sm_result.params.get("Connected[T.True]", 0.0)
    mdl_distance = sm_result.params.get("Distance", 0.0)
    return pval, mdl_added / mdl_intercept, 100 * mdl_distance / mdl_intercept

###Computing connection probabilities

def connection_probability_within(adj, neuron_properties, max_dist=200, min_dist=0,
                                  columns=["ss_flat_x", "ss_flat_y"]):
    #TODO: Does neuron_properties here assumes the right data structure?
    if isinstance(neuron_properties, tuple):
        nrn_pre, nrn_post = neuron_properties
    else:
        nrn_pre = neuron_properties;
        nrn_post = neuron_properties
    D = distance.cdist(nrn_pre[columns], nrn_post[columns])
    mask = (D > min_dist) & (D <= max_dist)  # This way with min_dist=0 a neuron with itself is excluded
    return adj[mask].mean()

def connection_probability(adj, neuron_properties):
    #TODO: Does neuron_properties here assumes the right data structure?
    exclude_diagonal = False
    if isinstance(neuron_properties, tuple):
        nrn_pre, nrn_post = neuron_properties
        if len(nrn_pre) == len(nrn_post):
            if (nrn_pre["gid"] == nrn_post["gid"]).all():
                exclude_diagonal = True
    else:
        exclude_diagonal = True

    if not exclude_diagonal:
        return adj.astype(bool).mean()
    assert adj.shape[0] == adj.shape[1], "Inconsistent shape!"
    n_pairs = adj.shape[0] * (adj.shape[1] - 1)
    return adj.astype(bool).sum() / n_pairs

#TODO ADD CODE FROM CLUSTER TO COMPUTE PROBABILITY OF CONNECTION PER PATHWAY OR ANY OTHER PROPERTIE ON THE NEURON_PROPERTY.

#TODO: Checked up to here ... clean the code below
# ##Degree based analyses
#TODO UPDATE THE RICH CLUB IMPLEMENTATION

def gini_curve(m, nrn, direction='efferent'):
    m = m.tocsc()
    if direction == 'afferent':
        degrees = np.array(m.sum(axis=0)).flatten()
    elif direction == 'efferent':
        degrees = np.array(m.sum(axis=1).flatten())
    else:
        raise Exception("Unknown value for argument direction: %s" % direction)
    cs = np.cumsum(np.flipud(sorted(degrees))).astype(float) / np.sum(degrees)
    return pd.Series(cs, index=np.linspace(0, 1, len(cs)))

def gini_coefficient(m, nrn, direction='efferent'):
    gc = gini_curve(m, nrn, direction=direction)
    A = gc.index.values
    B = gc.values
    return np.sum(np.diff(A) * (B[:-1] + B[1:]) / 2.0)

def _analytical_expected_gini_curve(m, direction='efferent'):
    if direction == 'afferent':
        N = m.shape[0] - 1
        C = m.shape[1] * N
    elif direction == 'efferent':
        N = m.shape[1] - 1
        C = m.shape[0] * N
    P = m.nnz / C
    # Only using degrees, not distribution of weigthts. TODO: Fix that
    x = np.arange(N, -1, -1)
    p = binom.pmf(x, N, P)
    A = np.cumsum(p) / p.sum()
    B = np.cumsum(p * x) / np.sum(p * x)
    return pd.Series(B, index=A)

def normalized_gini_coefficient(m, nrn, direction='efferent'):
    gc = gini_coefficient(m, nrn, direction=direction)
    ctrl = _analytical_expected_gini_curve(m, direction=direction)
    A = ctrl.index.values
    B = ctrl.values
    return 2 * (gc - np.sum(np.diff(A) * (B[:-1] + B[1:]) / 2.0))

def _bin_degrees(degrees):
    nbins = np.maximum(int(len(degrees) * 0.1), np.minimum(len(degrees), 30))
    mx = np.nanmax(degrees);
    mn = np.nanmin(degrees)
    bins = np.linspace(mn, mx + 1E-6 * (mx - mn), nbins + 1)
    degrees = np.digitize(degrees, bins=bins) - 1
    udegrees = np.arange(nbins)
    ret_x = 0.5 * (bins[:-1] + bins[1:])
    return ret_x, udegrees, degrees

def rich_club_curve(m, direction='efferent'):
    m = m.tocsc()
    if direction == 'afferent':
        degrees = np.array(m.sum(axis=0)).flatten()
    elif direction == 'efferent':
        degrees = np.array(m.sum(axis=1)).flatten()
    else:
        raise Exception("Unknown value for argument direction: %s" % direction)

    if m.dtype == bool:
        udegrees = np.arange(1, degrees.max() + 1)
        ret_x = udegrees
    else:
        ret_x, udegrees, degrees = _bin_degrees(degrees)

    edge_counter = lambda i: (degrees >= i).sum() * ((degrees >= i).sum() - 1)  # number of pot. edges
    mat_counter = lambda i: m[np.ix_(degrees >= i, degrees >= i)].sum()  # number of actual edges
    ret = (np.array([mat_counter(i) for i in udegrees]).astype(float)
           / np.array([edge_counter(i) for i in udegrees]))
    return pd.Series(ret, index=ret_x)

def efficient_rich_club_curve(M, direction="efferent", pre_calculated_richness=None, sparse_bin_set=False):
    M = M.tocoo()
    shape = M.shape
    M = pd.DataFrame.from_dict({"row": M.row, "col": M.col})
    if pre_calculated_richness is not None:
        deg = pre_calculated_richness
    elif direction == "efferent":
        deg = M["row"].value_counts()
    elif direction == "afferent":
        deg = M["col"].value_counts()
    elif direction == "both":
        D = pd.DataFrame({'row': np.zeros(shape[0]), 'col': np.zeros(shape[0])}, np.arange(shape[0])[-1::-1])
        D['row'] = D['row'] + M["row"].value_counts()
        D['col'] = D['col'] + M["col"].value_counts()
        D = D.fillna(0)
        D = D.astype(int)
        deg = D['row'] + D['col']
    else:
        raise ValueError()

    if sparse_bin_set == False:
        degree_bins = np.arange(deg.max() + 2)
    elif sparse_bin_set == True:
        degree_bins = np.unique(np.append(deg, [0, deg.max() + 1]))
    degree_bins_rv = degree_bins[-2::-1]
    nrn_degree_distribution = np.histogram(deg.values, bins=degree_bins)[0]
    nrn_cum_degrees = np.cumsum(nrn_degree_distribution[-1::-1])
    nrn_cum_pairs = nrn_cum_degrees * (nrn_cum_degrees - 1)

    deg_arr = np.zeros(shape[0], dtype=int)
    deg_arr[deg.index.values] = deg.values

    deg = None

    con_degree = np.minimum(deg_arr[M["row"].values], deg_arr[M["col"].values])
    M = None
    con_degree = np.histogram(con_degree, bins=degree_bins)[0]

    cum_degrees = np.cumsum(con_degree[-1::-1])

    return pd.Series(cum_degrees / nrn_cum_pairs, degree_bins_rv)[::-1]

def _analytical_expected_rich_club_curve(m, direction='efferent'):
    assert m.dtype == bool, "This function only works for binary matrices at the moment"
    indegree = np.array(m.sum(axis=0))[0]
    outdegree = np.array(m.sum(axis=1))[:, 0]

    if direction == 'afferent':
        degrees = indegree
    elif direction == 'efferent':
        degrees = outdegree
    else:
        raise Exception("Unknown value for argument direction: %s" % direction)

    udegrees = np.arange(1, degrees.max() + 1)
    edge_counter = lambda i: (degrees >= i).sum() * ((degrees >= i).sum() - 1)
    res_mn = []
    res_sd = []
    for deg in udegrees:
        valid = np.nonzero(degrees >= deg)[0]
        i_v = indegree[valid]
        i_sum_all = indegree.sum() - i_v
        i_sum_s = i_v.sum() - i_v
        o_v = outdegree[valid]
        S = np.array([hypergeom.stats(_ia, _is, o)
                      for _ia, _is, o in zip(i_sum_all, i_sum_s, o_v)])
        res_mn.append(np.sum(S[:, 0]) / edge_counter(deg))
        res_sd.append(np.sqrt(S[:, 1].sum()) / edge_counter(deg))  # Sum the variances, but divide the std
    df = pd.DataFrame.from_dict({"mean": np.array(res_mn),
                                 "std": np.array(res_sd)})
    df.index = udegrees
    return df

def generate_degree_based_control(M, direction="efferent"):
    """
    A shuffled version of a connectivity matrix that aims to preserve degree distributions.
    If direction = "efferent", then the out-degree is exactly preserved, while the in-degree is
    approximately preseved. Otherwise it's the other way around.
    """
    if direction == "efferent":
        M = M.tocsr()
        idxx = np.arange(M.shape[1])
        p_out = np.array(M.mean(axis=0))[0]
    elif direction == "afferent":
        M = M.tocsc()
        idxx = np.arange(M.shape[0])
        p_out = np.array(M.mean(axis=1))[:, 0]
    else:
        raise ValueError()

    for col in range(M.shape[1]):
        p = p_out.copy()
        p[col] = 0.0
        p = p / p.sum()
        a = M.indptr[col]
        b = M.indptr[col + 1]
        M.indices[a:b] = np.random.choice(idxx, b - a, p=p, replace=False)
    return M

def _randomized_control_rich_club_curve(m, direction='efferent', n=10):
    res = []
    for _ in range(n):
        m_shuf = generate_degree_based_control(m, direction=direction)
        res.append(efficient_rich_club_curve(m_shuf))
    res = pd.concat(res, axis=1)
    #TODO: Something is wrong here. rr is not defined. Should it be res?
    #      But changing rr to res causing
    df = pd.DataFrame.from_dict(
        {
            "mean": np.nanmean(res, axis=1),
            "std": np.nanstd(res, axis=1)
        }
    )
    df.index = res.index
    return df

def normalized_rich_club_curve(m, direction='efferent', normalize='std',
                               normalize_with="shuffled", **kwargs):
    assert m.dtype == bool, "This function only works for binary matrices at the moment"
    data = efficient_rich_club_curve(m, direction=direction)
    A = data.index.values
    B = data.values
    if normalize_with == "analytical":
        ctrl = _analytical_expected_rich_club_curve(m, direction=direction)
    elif normalize_with == "shuffled":
        ctrl = _randomized_control_rich_club_curve(m, direction=direction)
    Ar = ctrl.index.values
    mn_r = ctrl["mean"].values
    sd_r = ctrl["std"].values

    if normalize == 'mean':
        return pd.Series(B[:len(mn_r)] / mn_r, index=A[:len(mn_r)])
    elif normalize == 'std':
        return pd.Series((B[:len(mn_r)] - mn_r) / sd_r, index=A[:len(mn_r)])
    else:
        raise Exception("Unknown normalization: %s" % normalize)

def rich_club_coefficient(m, **kwargs):
    Bn = normalized_rich_club_curve(m, normalize='std', **kwargs).values
    return np.nanmean(Bn)

#*************************************************************************#
#Code taken from TriDy

##
## HELPER FUNCTIONS (STRUCTURAL)
##

def nx_to_np(directed_graph):
    """Converts networkx digraph to numpy array of the adjacency matrix

    Parameters
    ----------
    directed_graph : networkx DiGraph
        a directed graph

    Returns
    -------
    numpy array
        the adjaceny matrix of the DiGraph as a numpy array
    """
    return nx.to_numpy_array(directed_graph,dtype=int)

def np_to_nx(adjacency_matrix):
    """Converts numpy array of an adjacency matrix to a networkx digraph

    Parameters
    ----------
    adjacency_matrix : numpy array
        the adjaceny matrix of the DiGraph as a numpy array

    Returns
    -------
    networkx DiGraph
            a directed graph

    """
    return nx.from_numpy_array(adjacency_matrix,create_using=nx.DiGraph)

def largest_strongly_connected_component(adjacency_matrix):
    """Computes the largest strongly connected component of the graph with adjacency matrix adjacency_matrix,
        and returns the adjacency matrix of said component

    Parameters
    ----------
    adjacency_matrix : numpy array
        the adjaceny matrix of the DiGraph as a numpy array

    Returns
    -------
    numpy array
        The adjacency matrix of the largest strongly connected component

    """
    current_tribe_nx = np_to_nx(adjacency_matrix)
    largest_comp = max(nx.strongly_connected_components(current_tribe_nx), key=len)
    current_tribe_strong_nx = current_tribe_nx.subgraph(largest_comp)
    current_tribe_strong = nx_to_np(current_tribe_strong_nx)
    return current_tribe_strong

##
## HELPER FUNCTIONS (SPECTRAL)
##

def spectral_gap(matrix, thresh=10, param='low'):
#  In: matrix
# Out: float
    current_spectrum = spectrum_make(matrix)
    current_spectrum = spectrum_trim_and_sort(current_spectrum, threshold_decimal=thresh)
    return spectrum_param(current_spectrum, parameter=param)

def spectrum_make(matrix):
#  In: matrix
# Out: list of complex floats
    assert np.any(matrix) , 'Error (eigenvalues): matrix is empty'
    eigenvalues = eigvals(matrix)
    return eigenvalues

def spectrum_trim_and_sort(spectrum, modulus=True, threshold_decimal=10):
#  In: list of complex floats
# Out: list of unique (real or complex) floats, sorted by modulus
    if modulus:
        return np.sort(np.unique(abs(spectrum).round(decimals=threshold_decimal)))
    else:
        return np.sort(np.unique(spectrum.round(decimals=threshold_decimal)))

def spectrum_param(spectrum, parameter):
#  In: list of complex floats
# Out: float
    assert len(spectrum) != 0 , 'Error (eigenvalues): no eigenvalues (spectrum is empty)'
    if parameter == 'low':
        if spectrum[0]:
            return spectrum[0]
        else:
            assert len(spectrum) > 1 , 'Error (low spectral gap): spectrum has only zeros, cannot return nonzero eigval'
            return spectrum[1]
    elif parameter == 'high':
        assert len(spectrum) > 1 , 'Error (high spectral gap): spectrum has one eigval, cannot return difference of top two'
        return spectrum[-1]-spectrum[-2]
    elif parameter == 'radius':
        return spectrum[-1]

##
## NONSPECTRAL PARAMETER FUNCTIONS
##

def ccc(matrix):
    """Computes the classical clustering coefficient of a directed graph

    Parameters
    ----------
    matrix : numpy array
        the adjaceny matrix of a directed graph

    Returns
    -------
    float
        the clustering coefficient

    References
    ----------
    The formula is taken from the following paper.

    ..[1] G. Fagiolo, "Clustering in complex directed networks", 2006;
           [DOI: 10.1103/PhysRevE.76.026107](https://doi.org/10.1103/PhysRevE.76.026107).

    """
    deg = node_degree(matrix)[0]
    numerator = np.linalg.matrix_power(matrix+np.transpose(matrix),3)[0][0]
    denominator = 2*(deg*(deg-1)-2*reciprocal_connections_adjacency(matrix, chief_only=True))
    if denominator == 0:
        return 0
    return numerator/denominator

# degree type parameters

# tribe size

def tribe_size(matrix):
    return len(matrix)

# reciprocal connections

def reciprocal_connections(matrix, chief_only=False):
    #TODO: MERGE THIS WITH RC FUNCTION
    if chief_only:
        rc_count = np.count_nonzero(np.multiply(matrix[0],np.transpose(matrix)[0]))
    else:
        rc_count = np.count_nonzero(np.multiply(matrix,np.transpose(matrix)))//2
    return rc_count

##
## SPECTRAL PARAMETER FUNCTIONS
##

# adjacency spectrum

def asg(matrix, gap='high'):
    return spectral_gap(matrix, param=gap)

# transition probability spectrum

def tpsg(matrix, in_deg=False, gap='high'):
#  in: tribe matrix
# out: float
    current_matrix = tps_matrix(matrix, in_deg=in_deg)
    return spectral_gap(current_matrix, param=gap)

def tps_matrix(matrix, in_deg=False):
#  in: tribe matrix
# out: transition probability matrix
    current_size = len(matrix)
    if in_deg:
        degree_vector = node_degree(matrix, direction='IN').values
    else:
        degree_vector = node_degree(matrix, direction='OUT').values
    inverted_degree_vector = [0 if not d else 1/d for d in degree_vector]
    return np.matmul(np.diagflat(inverted_degree_vector),matrix)

# chung laplacian spectrum
# source 1: Laplacians and the Cheeger inequality for directed graph (Fan Chung, 2005)
# source 2: https://networkx.org/documentation/stable/reference/generated/networkx.linalg.laplacianmatrix.directed_laplacian_matrix.html

def clsg(matrix, is_strongly_conn=False, gap='low'):
#  in: tribe matrix
# out: float
    chung_laplacian_matrix = cls_matrix_fromadjacency(matrix, is_strongly_conn=is_strongly_conn)
    return spectral_gap(chung_laplacian_matrix, param=gap)

def cls_matrix_fromadjacency(matrix, is_strongly_conn=False):
#  in: numpy array
# out: numpy array
    matrix_nx = np_to_nx(matrix)
    return cls_matrix_fromdigraph(matrix_nx, matrix=matrix, matrix_given=True, is_strongly_conn=is_strongly_conn)

def cls_matrix_fromdigraph(digraph, matrix=np.array([]), matrix_given=False, is_strongly_conn=False):
#  in: networkx digraph
# out: numpy array
    digraph_sc = digraph
    matrix_sc = matrix
    # Make sure is strongly connected
    if not is_strongly_conn:
        largest_comp = max(nx.strongly_connected_components(digraph), key=len)
        digraph_sc = digraph.subgraph(largest_comp)
        matrix_sc = nx_to_np(digraph_sc)
    elif not matrix_given:
        matrix_sc = nx_to_np(digraph_sc)
    # Degeneracy: scc has size 1
    if not np.any(matrix_sc):
        return np.array([[0]])
    # Degeneracy: scc has size 2
    elif np.array_equal(matrix_sc,np.array([[0,1],[1,0]],dtype=int)):
        return np.array([[1,-0.5],[-0.5,1]])
    # No degeneracy
    else:
        return nx.directed_laplacian_matrix(digraph_sc)

# bauer laplacian spectrum
# source: Normalized graph Laplacians for directed graphs (Frank Bauer, 2012)

def blsg(matrix, reverse_flow=False, gap='high'):
#  in: tribe matrix
# out: float
    bauer_laplacian_matrix = bls_matrix(matrix, reverse_flow=reverse_flow)
    return spectral_gap(bauer_laplacian_matrix, param=gap)

def bls_matrix(matrix, reverse_flow=False):
#  in: tribe matrix
# out: bauer laplacian matrix
    #non_quasi_isolated = [i for i in range(len(matrix)) if matrix[i].any()]
    #matrix_D = np.diagflat([np.count_nonzero(matrix[nqi]) for nqi in non_quasi_isolated])
    #matrix_W = np.diagflat([np.count_nonzero(np.transpose(matrix)[nqi]) for nqi in non_quasi_isolated])
    #return np.subtract(np.eye(len(non_quasi_isolated),dtype=int),np.matmul(inv(matrix_D),matrix_W))
    current_size = len(matrix)
    return np.subtract(np.eye(current_size,dtype='float64'),tps_matrix(matrix, in_deg=(not reverse_flow)))

##
## TO BE DELETED ONCE EVERYTHING WORKS
##

# # degree
# def degree(chief_index, matrix, vertex_index=0):
#     current_tribe = tribe(chief_index, matrix)
#     return degree_adjacency(current_tribe, vertex_index=vertex_index)

# def degree_adjacency(matrix, vertex_index=0):
#     return in_degree_adjacency(matrix, vertex_index=vertex_index)+out_degree_adjacency(matrix, vertex_index=vertex_index)

# # in-degree
# def in_degree(chief_index, matrix, vertex_index=0):
#     current_tribe = tribe(chief_index, matrix)
#     return in_degree_adjacency(current_tribe, vertex_index=vertex_index)

# def in_degree_adjacency(matrix, vertex_index=0):
#     return np.count_nonzero(np.transpose(matrix)[vertex_index])

# # out-degree
# def out_degree(chief_index, matrix, vertex_index=0):
#     current_tribe = tribe(chief_index, matrix)
#     return out_degree_adjacency(current_tribe, vertex_index=vertex_index)

# def out_degree_adjacency(matrix, vertex_index=0):
#     return np.count_nonzero(matrix[vertex_index])


import numpy as np
import cv2

window_main = 'Chairs'
img_in = cv2.imread('chairs.jpg', 1)
img_out = np.array(img_in)
dx = 5
dy = 5
detection_tol = 10
win_width = 1000
win_height = 800
chairpos = []

class queue:
    a = []

    def enqueue(self, x):
        self.a = self.a + [x]
        return True

    def dequeue(self):
        output = self.a[0]
        self.a = self.a[1:]
        return output

def undo():
    global window_main
    global img_in
    global img_out
    global chairpos

    if len(chairpos) == 0:
        return True

    chair = chairpos[-1]
    chairpos = chairpos[:-1]

    for p in chair:
        x = p[0]
        y = p[1]

        cv2.circle(img_out, (x, y), 1, (int(img_in[y, x][0]), int(img_in[y, x][1]), int(img_in[y, x][2])), -1)

    cv2.imshow(window_main, img_out)

    return True

def compare_color(x1, y1, x2, y2):
    global img_in
    global img_out
    global detection_tol

    c1 = np.array(img_in[y1, x1], dtype=int)
    c2 = np.array(img_in[y2, x2], dtype=int)

    if np.sum(np.abs(c2 - c1)) > detection_tol:
        return False

    return True

def detect_chair(x_0, y_0):
    global window_main
    global img_in
    global img_out
    global dx
    global dy
    global chairpos

    chair = []

    q = queue()
    q.enqueue([x_0, y_0])

    while len(q.a) > 0:
        point = q.dequeue()
        x = point[0]
        y = point[1]
        chair = chair + [[x, y]]

        cv2.circle(img_out, (x, y), 1, (0, 0, 255), -1)

        if compare_color(x, y, x + dx, y) and not np.array_equal(img_out[y, x + dx], [0, 0, 255]):
            q.enqueue([x + dx, y])
            cv2.circle(img_out, (x + dx, y), 1, (0, 0, 255), -1)

        if compare_color(x, y, x - dx, y) and not np.array_equal(img_out[y, x - dx], [0, 0, 255]):
            q.enqueue([x - dx, y])
            cv2.circle(img_out, (x - dx, y), 1, (0, 0, 255), -1)

        if compare_color(x, y, x, y + dy) and not np.array_equal(img_out[y + dy, x], [0, 0, 255]):
            q.enqueue([x, y + dy])
            cv2.circle(img_out, (x, y + dy), 1, (0, 0, 255), -1)

        if compare_color(x, y, x, y - dy) and not np.array_equal(img_out[y - dy, x], [0, 0, 255]):
            q.enqueue([x, y - dy])
            cv2.circle(img_out, (x, y - dy), 1, (0, 0, 255), -1)

    chairpos = chairpos + [chair]

    return True

def mouse_click(event, x, y, flags, param):
    global window_main
    global img_in
    global img_out

    if event == cv2.EVENT_LBUTTONDOWN:
        detect_chair(x, y)
        cv2.imshow(window_main, img_out)

    return True

def main():
    global window_main
    global chairpos
    global img_in
    global img_out
    global win_width
    global win_height

    cv2.namedWindow(window_main, 0)
    cv2.setMouseCallback(window_main, mouse_click)
    cv2.resizeWindow(window_main, win_width, win_height)

    if img_in is None:
        print('Could not find the picture specified!')
        return True

    cv2.imshow(window_main, img_out)

    while 1 == 1:
        k = cv2.waitKey(0)

        if k == ord('z'):
            undo()
        if k == 27:
            break

    cv2.destroyAllWindows()

    output = open('output.txt', 'w')
    for chair in chairpos:
        line = str(len(chair)) + '\n'
        for p in chair:
            x = p[0]
            y = p[1]
            c = img_in[y, x]
            line = line + str(x) + ',' + str(y) + ',' + str(c[0]) + ',' + str(c[1]) + ',' + str(c[2]) + '\n';

        output.write(line)

    output.close()

    return True

if __name__ == '__main__':
    main()


from django.db.models import AutoField
from django.db import models
from django.core.exceptions import PermissionDenied
from django.contrib import admin

class Callable:
    def __init__(self, anycallable):
        self.__call__ = anycallable

def copy_model_instance(obj):
    """ Django snippit 1040
    Create a copy of a model instance.
    Works in model inheritance case where instance.pk = None is not good enough, since the subclass instance refers to the parent_link's primary key during save.
    M2M relationships are currently not handled, i.e. they are not copied.
    """
    initial = dict([(f.name, getattr(obj, f.name))
                    for f in obj._meta.fields
                    if not isinstance(f, AutoField) and\
                       not f in obj._meta.parents.values()])
    return obj.__class__(**initial)

def log_admin_entry(request, obj, state, message=""):
    """
    Make a log entry in django admin.
    request: Django request - must have user. Will do nothing without user.
    obj: Any django object
    state: django.contrib.admin.models. ADDITION, DELETION, or CHANGE
    message: optional message for log
    """
    from django.contrib.admin.models import LogEntry
    from django.contrib.contenttypes.models import ContentType
    if request.user and hasattr(request.user,"pk"):
        LogEntry.objects.log_action(
            user_id         = request.user.pk,
            content_type_id = ContentType.objects.get_for_model(obj).pk,
            object_id       = obj.pk,
            object_repr     = unicode(obj),
            action_flag     = state,
            change_message  = message
        )

class Struct(object):
    def __unicode__(self):
        return ""

class CharNullField(models.CharField):
    description = "CharField that stores NULL but returns ''"
    def to_python(self, value):  #this is the value right out of the db, or an instance
       if isinstance(value, models.CharField): #if an instance, just return the instance
              return value
       if value==None:   #if the db has a NULL (==None in Python)
              return ""  #convert it into the Django-friendly '' string
       else:
              return value #otherwise, return just the value
    def get_db_prep_value(self, value, *args, **kwargs):  #catches value right before sending to db
       if value=="":     #if Django tries to save '' string, send the db None (NULL)
            return None
       else:
            return super(CharNullField, self).get_db_prep_value(value, *args, **kwargs)

class ReadPermissionModelAdmin(admin.ModelAdmin):
    """ based on http://gremu.net/blog/2010/django-admin-read-only-permission/
    Admin model that allows users to read
    """
    def has_change_permission(self, request, obj=None):
        if getattr(request, 'readonly', False):
            return True
        return super(ReadPermissionModelAdmin, self).has_change_permission(request, obj)

    def changelist_view(self, request, extra_context=None):
        try:
            return super(ReadPermissionModelAdmin, self).changelist_view(
                request, extra_context=extra_context)
        except PermissionDenied:
            pass
        perm_name = '%s.view_%s' % (self.model._meta.app_label, unicode.lower(unicode(self.model._meta.object_name)))
        if request.method == 'POST' or not perm_name in request.user.get_all_permissions():
            # Only allow POST if export to xls and nothing evil
            # It's not all that secure, but we assume authenticated users aren't trying to hack things.
            if not (request.POST['action'] == 'export_simple_selected_objects' and len(request.POST) <= 4) :
                raise PermissionDenied
        request.readonly = True
        return super(ReadPermissionModelAdmin, self).changelist_view(
            request, extra_context=extra_context)

    def change_view(self, request, object_id, extra_context=None):
        try:
            return super(ReadPermissionModelAdmin, self).change_view(
                request, object_id, extra_context=extra_context)
        except PermissionDenied:
            pass
        if request.method == 'POST':
            raise PermissionDenied
        request.readonly = True
        return super(ReadPermissionModelAdmin, self).change_view(
            request, object_id, extra_context=extra_context)


from django.core.management.base import BaseCommand, CommandError
import requests
import json
from data.models import Nursery, Device, Sample, Param

class Command(BaseCommand):
    help = 'Traer los datos de los dispositivos'

    def json_load_byteified(self, file_handle):
        return _byteify(
            json.load(file_handle, object_hook=self._byteify),
            ignore_dicts=True
        )

    def json_loads_byteified(self, json_text):
        return self._byteify(
            json.loads(json_text, object_hook=self._byteify),
            ignore_dicts=True
        )

    def _byteify(self, data, ignore_dicts=False):
        # if this is a unicode string, return its string representation
        if isinstance(data, unicode):
            return data.encode('utf-8')
        # if this is a list of values, return list of byteified values
        if isinstance(data, list):
            return [self._byteify(item, ignore_dicts=True) for item in data]
        # if this is a dictionary, return dictionary of byteified keys and values
        # but only if we haven't already byteified it
        if isinstance(data, dict) and not ignore_dicts:
            return {
                self._byteify(key, ignore_dicts=True): self._byteify(value, ignore_dicts=True)
                for key, value in data.iteritems()
            }
        # if it's anything else, return it in its original form
        return data

    def handle(self, *args, **options):
        devices = Device.objects.all()
        params = {"c": "Temperatura",
                  "%": "Humedad",
                  "z": "Zona de riego",
                  "m3": "Volumen de rociado",
                  "r1": "Producto regado 1",
                  "r2": "Producto regado 2",
                  "so": "Grado de sombra",
                  "s": "Status operativo del vivero",
                  "pr1": "Periodo de regado producto 1",
                  "pr2": "Periodo de regado producto 2"}
        for device in devices:
            # try:
            r = requests.get(device.url)
            data = self.json_loads_byteified(r.content)
            nursery = Nursery.objects.get(fiscal_key=data["v"]["c"])
            for i in data:
                if i != "v":
                    param = Param.objects.get(name__contains=params[i])
                    if i == "%" or i == "c":
                        Sample.objects.create(value=data[i]["v"], unit=i, state_transducer=data[i]["er"],
                                              duration=data[i]["d"], state_transmission=data[i]["en"],
                                              nursery=nursery,
                                              param=param
                                              )
                    else:
                        Sample.objects.create(value=data[i]["v"], unit="placeholder", state_transducer=data[i]["er"],
                                              duration=data[i]["d"], state_transmission=data[i]["en"],
                                              nursery=nursery,
                                              param=param
                                              )
                        # self.stdout.write(self.style.SUCCESS('Successfully'))
                        # except :
                        #   self.stdout.write(self.style.ERROR('ERROR'))


# -*- coding: utf-8 -*-

from __future__ import unicode_literals

import json

import six
from django import forms
from django.utils.translation import ugettext_lazy as _
from django.contrib.postgres.forms import JSONField

__all__ = ['JanySONField']

class JanySONField(JSONField):
    default_error_messages = {
        'invalid': _("JSON deserialization error: %(error)s"),
        'invalid_json_type': _("Invalid JSON type"),
        'invalid_value': _("%(field)s (%(type)s)"
                           " - invalid value: %(value)s"),
        'non_declared_fields': _("Non-declared field(s): %(fields)s"),
    }

    def to_python(self, value):
        value = value.strip()
        if value in self.empty_values:
            return None
        try:
            return json.loads(value)
        except ValueError as error:
            raise forms.ValidationError(
                self.error_messages['invalid'],
                code='invalid',
                params={'error': error},
            )

    def prepare_value(self, value):
        if isinstance(value, six.text_type):
            return value
        return json.dumps(
            value,
            ensure_ascii=False,
            indent=4,
            separators=(',', ': '),
            sort_keys=True
        )


import os.path

from imgui_bundle import immvision, immapp, imgui_md
from imgui_bundle.demos_python import demo_utils
import cv2  # type: ignore

def fill_inspector():
    os.path.dirname(__file__)
    image_files = ["dmla.jpg", "house.jpg", "tennis.jpg", "world.jpg"]
    for image_file in image_files:
        img = cv2.imread(f"{demo_utils.demos_assets_folder()}/images/{image_file}")
        immvision.inspector_add_image(img, legend=image_file)

@immapp.static(inited=False)
def demo_gui():
    if not demo_gui.inited:
        fill_inspector()
        demo_gui.inited = True

    imgui_md.render_unindented(
        """Call *immvision.inspector_add_image()* anywhere - for example, at different steps inside an image processing algorithm. Later, call *immvision.inspector_show()*, and it will show all the collected images."""
    )
    immvision.inspector_show()

def main():
    demo_utils.set_hello_imgui_demo_assets_folder()
    immapp.run(demo_gui, window_size=(1000, 800), with_markdown=True)

if __name__ == "__main__":
    main()


import itertools
from enum import Enum
import datetime
from statistics import mean

import paho.mqtt.client as mqtt

from map.elements.effector import Effector

'''
Singleton instanced as such to apply a localization that depends on the mode.
It will compute the position and communicate it accordingly.
'''
class Localization:

    def rssiThreshold(self):
        return -65
    def timeThreshold(self):
        return 2000
    def getType(self):
        pass
    def topic(self, effector: Effector):
        pass
    def build_messages(self, **kwargs):
        pass
    def send(self, **kwargs):
        client = kwargs["client"]
        for effectorMessage in self.build_messages(**kwargs):
            effector, message = effectorMessage
            if effector:
                client.publish(self.topic(effector).lower(), message)
    def compute(self, mac_dict, mac, origin):
        pass

class NeighboursLocalization(Localization):
    def topic(self, effector):
        return "directions/effector/neighbour/effector_{}".format(effector.idx)
    def build_messages(self, **kwargs):
        # each pair (node, device) has a message of the form: (node, device, is_close)
        nodes, devices, devices_dict = kwargs["nodes"].nodes, kwargs["devices"], kwargs["devices_dict"]
        # filtering: only in case the device is navigating we compute the localization
        active_devices = [device for device in devices if "status" in devices_dict[device] and devices_dict[device]["status"] == Status.NAVIGATING]
        messages = []
        for node, device in itertools.product(*[nodes, active_devices]):
            # check values that do not differ much in time
            anchor_id = node.idx    # before: node.mac.lower()
            if device in devices_dict and anchor_id in devices_dict[device]:
                current_vals = list(devices_dict[device][anchor_id])
                recent_values = list(map(lambda x: x["value"], filter(
                    lambda x: x["timestamp"] + datetime.timedelta(milliseconds=self.timeThreshold()) > datetime.datetime.now(),
                    current_vals)))
            else:
                recent_values = []
            # localization: node is close if threshold is greater than <rssiThreshold>
            close = 1 if len(recent_values) > 0 and mean(recent_values) > self.rssiThreshold() else 0
            messages.append((node, "{}${}".format(device, close)))
        return messages
    def getType(self):
        return LocalizationType.NEIGHBOURS

class NodeLocalization(Localization):
    def topic(self, effector):
        return "directions/effector/activate/effector_{}".format(effector.idx)
    def build_messages(self, **kwargs):
        # each pair (node, device) has a message of the form: (node, device, is_close)
        # effectors, nodes, devices, devices_dict = kwargs["effectors"], kwargs["nodes"], kwargs["devices"], kwargs["devices_dict"]
        sd_instance, devices, devices_dict = kwargs["sd_instance"], kwargs["devices"], kwargs["devices_dict"]

        # filtering: only in case the device is navigating we compute the localization
        active_devices = [device for device in devices if
                          "status" in devices_dict[device] and devices_dict[device]["status"] == Status.NAVIGATING]

        messages = []
        close_anchors = {}
        best_anchors = {}
        # find first each pair (device, node)
        for device_key in active_devices:
            found, best_val = False, self.rssiThreshold()
            closest_anchor = None
            for node in sd_instance.raw_anchors():
                anchor_id = node.idx    # before: node.mac.lower()
                current_vals = list(devices_dict[device_key][anchor_id])
                recent_values = list(map(lambda x: x["value"], filter(lambda x: x["timestamp"] + datetime.timedelta(milliseconds=self.timeThreshold()) > datetime.datetime.now(), current_vals)))
                rssi_val = -100
                if len(recent_values) > 0:
                    rssi_val = mean(recent_values)
                if rssi_val > best_val:
                    best_val = rssi_val
                    closest_anchor = node
            if closest_anchor is not None:
                best_anchors[device_key] = closest_anchor

        # for each device, use the closest anchor
        for device_key in best_anchors.keys():
            node = best_anchors[device_key]
            color = devices_dict[device_key]['color']
            device_building = [b for b in sd_instance.buildings if devices_dict[device_key]['id_building']==b.id][0]

            is_first_time = False
            device_destination = device_building.findPoI(devices_dict[device_key]['id_POI'])
            # only first time: last_anchor is the last encountered anchor; first_anchor is the first encountered anchor
            if 'last_anchor' not in devices_dict[device_key]:
                devices_dict[device_key]['last_anchor'] = node
            if 'first_anchor' not in devices_dict[device_key]:
                devices_dict[device_key]['first_anchor'] = node

            # this feature is currently not used.
            # if True, last_anchor is the last encountered (gets updated)
            # if False, last_anchor is the same as the first encountered
            update_last_anchor = False
            if update_last_anchor:
                if node.idx != devices_dict[device_key]['last_anchor'].idx:
                    devices_dict[device_key]['last_anchor'] = node
            device_origin = devices_dict[device_key]['last_anchor']

            # localization: node is close if threshold is greater than <rssiThreshold> and no other node is close
            effectors_to_activate, face_to_show, relative_message_to_show, absolute_message_to_show = \
                device_building.toActivate(node, device_destination, device_origin)

            from map.elements.planimetry.point_type import Direction, MessageDirection

            if not devices_dict[device_key]['shown_first']:
                face_to_show, relative_message_to_show = Direction.ALL, MessageDirection.START

            first_anchor_show_all_faces = False

            if effectors_to_activate is not None:
                if first_anchor_show_all_faces and devices_dict[device_key]['shown_first'] \
                        and node.idx == devices_dict[device_key]['first_anchor'].idx\
                        and relative_message_to_show != MessageDirection.ARRIVED:
                    faces, directions = device_building.portDirectionsToFaces(absolute_message_to_show)
                else:
                    faces, directions = [face_to_show], [relative_message_to_show]
                for face_to_show, relative_message_to_show in zip(faces, directions):
                    messages.append((effectors_to_activate, "{}$1${}${}${}".format(device_key, face_to_show, relative_message_to_show, color)))
                all_effectors = device_building.raw_effectors()
                for remaining_effector in all_effectors:
                    if remaining_effector.idx != effectors_to_activate.idx:
                        # relevant only for smartphone effectors. Masters do not show the message persistently
                        messages.append(
                            (remaining_effector, "{}$0${}${}${}".format(device_key, face_to_show, relative_message_to_show, color)))

            else:
                print("INFO: no effector to activate for device {}".format(device_key))

            devices_dict[device_key]['shown_first'] = True

        return messages
    def getType(self):
        return LocalizationType.NODE
'''
Factory for the localization type.
Builds depending on the enum
'''
class LocalizationFactory:
    class __LocalizationFactory:
        def __init__(self):
            pass

        def build(self, mode):
            if mode == LocalizationType.NEIGHBOURS:
                return NeighboursLocalization()
            elif mode == LocalizationType.NODE:
                return NodeLocalization()
            elif mode == LocalizationType.ACCURATE:
                raise Exception("Unsupported localization")
            else:
                raise Exception("Unexisting localization")

    __instance = None

    def __init__(self):
        if not LocalizationFactory.__instance:
            LocalizationFactory.__instance = LocalizationFactory.__LocalizationFactory()

    def __getattr__(self, name):
        return getattr(self.__instance, name)

    def getInstance(self):
        return LocalizationFactory.__instance

class LocalizationType(Enum):
    NEIGHBOURS = 1      # computes list of neighbours and communicates to all of them
    NODE = 2            # localization is done by associating to a device a node
    ACCURATE = 3        # more precise localization. TODO

class Status(Enum):
    TO_INIT = 1
    NAVIGATING = 2
    INACTIVE = 3


#!/usr/bin/env python
# -*- encoding: utf-8 -*-

import json
import os

from fastapi import HTTPException, status

from fastapp.schemas.appinfo import ApkInfo, IpaInfo
from fastapp.core.config import settings

from plugins.apk_parse3.apk_parse3.apk import APK
from plugins.ipa_parser import IosIpa as IpaParser

"""
解析apk文件

@File    :   app_info.py
@Time    :   2021/03/25 15:26:30
@Author  :   snc
"""

class AppInfo(object):
    def __init__(self, file: str):
        self.file = file

    def apk_parser(self) -> ApkInfo:

        apkf = APK(self.file)
        if not apkf.is_valid_APK():
            raise HTTPException(
                status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail='不支持该媒体类型')

        app_info = ApkInfo(
            package=apkf.package,
            file_md5=apkf.file_md5,
            cert_md5=apkf.get_cert_md5(),
            file_size=apkf.file_size,
            androidversion=apkf.androidversion,
            sdk_version=apkf.get_target_sdk_version(),
            libraries=apkf.get_libraries(),
            files=apkf.get_files(),
            files_types=apkf.get_files_types(),
            main_activity=apkf.get_main_activity(),
            activities=apkf.get_activities(),
            services=apkf.get_services(),
            receivers=apkf.get_receivers(),
            providers=apkf.get_providers(),
            permissions=apkf.get_permissions()
        )

        # app_info = apkf.get_file(apkf.get_app_icon())
        # app_info=apkf.get_certificate()

        return app_info

    def ipa_parser(self) -> IpaInfo:
        ipaf = IpaParser(self.file)
        app_info = ipaf.get_info()
        app_info = IpaInfo(**app_info)

        return app_info

    def apk_parser_go(self):
        parser_path = os.path.join(
            settings.BASE_PATH, 'plugins', 'app_parser.exe')

        run = os.popen(parser_path+' '+self.file)
        info = run.buffer.read().decode(encoding='utf8')
        run.close()

        info = json.loads(info)

        return info


# SPDX-License-Identifier: MIT
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple

from ..decodestate import DecodeState
from ..encodestate import EncodeState
from ..exceptions import DecodeError, EncodeError, odxraise, odxrequire
from ..odxlink import OdxLinkDatabase, OdxLinkId, OdxLinkRef
from ..odxtypes import ParameterValue
from .parameter import Parameter, ParameterType

if TYPE_CHECKING:
    from ..diaglayer import DiagLayer
    from ..table import Table
    from ..tablerow import TableRow

@dataclass
class TableKeyParameter(Parameter):

    odx_id: OdxLinkId
    table_ref: Optional[OdxLinkRef]
    table_snref: Optional[str]
    table_row_snref: Optional[str]
    table_row_ref: Optional[OdxLinkRef]

    def __post_init__(self) -> None:
        self._table: "Table"
        self._table_row: Optional["TableRow"] = None
        if self.table_ref is None and self.table_snref is None and \
           self.table_row_ref is None and self.table_row_snref is None:
            odxraise("Either a table or a table row must be defined.")

    @property
    def parameter_type(self) -> ParameterType:
        return "TABLE-KEY"

    def _build_odxlinks(self) -> Dict[OdxLinkId, Any]:
        result = super()._build_odxlinks()

        result[self.odx_id] = self

        return result

    def _resolve_odxlinks(self, odxlinks: OdxLinkDatabase) -> None:
        super()._resolve_odxlinks(odxlinks)

        # Either table_ref or table_row_ref will be defined
        if self.table_ref:
            if TYPE_CHECKING:
                self._table = odxlinks.resolve(self.table_ref, Table)
            else:
                self._table = odxlinks.resolve(self.table_ref)

        if self.table_row_ref:
            if TYPE_CHECKING:
                self._table_row = odxlinks.resolve(self.table_row_ref, TableRow)
            else:
                self._table_row = odxlinks.resolve(self.table_row_ref)
            self._table = self._table_row.table

    def _resolve_snrefs(self, diag_layer: "DiagLayer") -> None:
        super()._resolve_snrefs(diag_layer)

        if self.table_snref is not None:
            ddd_spec = diag_layer.diag_data_dictionary_spec
            self._table = ddd_spec.tables[self.table_snref]
        if self.table_row_snref is not None:
            # make sure that we know the table to which the table row
            # SNREF is relative to.
            table = odxrequire(
                self._table, "If a table-row short name reference is defined, a "
                "table must also be specified.")
            self._table_row = table.table_rows[self.table_row_snref]

    @property
    def table(self) -> "Table":
        if self._table is not None:
            return self._table
        if self._table_row is not None:
            return self._table_row.table
        odxraise(f'Could not resolve the table of {self.short_name}')

    @property
    def table_row(self) -> Optional["TableRow"]:
        return self._table_row

    @property
    def is_required(self) -> bool:
        # TABLE-KEY parameters can be implicitly determined from the
        # corresponding TABLE-STRUCT
        return False

    @property
    def is_settable(self) -> bool:
        return True

    def get_coded_value_as_bytes(self, encode_state: EncodeState) -> bytes:
        tr_short_name = encode_state.parameter_values.get(self.short_name)

        if tr_short_name is None:
            # the table key has not been defined explicitly yet, but
            # it is most likely implicitly defined by the associated
            # TABLE-STRUCT parameters. Use all-zeros as a standin for
            # the real data...
            key_dop = self.table.key_dop
            if key_dop is None:
                raise EncodeError(f"Table '{self.table.short_name}' does not define "
                                  f"a KEY-DOP, but is used in TABLE-KEY parameter "
                                  f"'{self.short_name}'")

            byte_len = (odxrequire(key_dop.get_static_bit_length()) + 7) // 8
            if self.bit_position is not None and self.bit_position > 0:
                byte_len += 1

            return bytes([0] * byte_len)

        # the table key is known. We need to encode the associated DOP
        # into the PDU.
        tr_candidates = [x for x in self.table.table_rows if x.short_name == tr_short_name]
        if len(tr_candidates) == 0:
            raise EncodeError(f"No table row with short name '{tr_short_name}' found")
        elif len(tr_candidates) > 1:
            raise EncodeError(f"Multiple rows exhibiting short name '{tr_short_name}'")
        tr = tr_candidates[0]

        key_dop = self.table.key_dop
        if key_dop is None:
            raise EncodeError(f"Table '{self.table.short_name}' does not define "
                              f"a KEY-DOP, but is used in TABLE-KEY parameter "
                              f"'{self.short_name}'")
        bit_position = 0 if self.bit_position is None else self.bit_position
        return key_dop.convert_physical_to_bytes(tr.key, encode_state, bit_position=bit_position)

    def encode_into_pdu(self, encode_state: EncodeState) -> bytes:
        return super().encode_into_pdu(encode_state)

    def decode_from_pdu(self, decode_state: DecodeState) -> Tuple[ParameterValue, int]:
        if self.byte_position is not None and self.byte_position != decode_state.cursor_position:
            cursor_position = self.byte_position

        # update the decode_state's table key
        if self.table_row is not None:
            # the table row to be used is statically specified -> no
            # need to decode anything!
            phys_val = self.table_row.short_name
            cursor_position = decode_state.cursor_position
        else:
            # Use DOP to decode
            key_dop = odxrequire(self.table.key_dop)
            bit_position_int = self.bit_position if self.bit_position is not None else 0
            key_dop_val, cursor_position = key_dop.convert_bytes_to_physical(
                decode_state, bit_position=bit_position_int)

            table_row_candidates = [x for x in self.table.table_rows if x.key == key_dop_val]
            if len(table_row_candidates) == 0:
                raise DecodeError(f"No table row exhibiting the key '{str(key_dop_val)}' found")
            elif len(table_row_candidates) > 1:
                raise DecodeError(
                    f"Multiple rows exhibiting key '{str(key_dop_val)}' found in table")
            phys_val = table_row_candidates[0].short_name

        return phys_val, cursor_position


import os
import sys

import wx
import wx.adv
import pandas as pd
import xlsxwriter
from ObjectListView import ObjectListView, ColumnDefn, FastObjectListView
from threading import Thread
from pubsub import pub

from components.drug_dialog import DrugRegFormDialog

CLOSE_PROGRESS_BAR_SIGNAL = 'close-progressbar'
WRITE_TO_EXCEL_FILE_SIGNAL = 'write-to-excel-file'
ENABLE_BUTTONS = 'enable-buttons'
DISABLE_BUTTONS = 'disable-buttons'

class PulseProgressBarDialog(wx.ProgressDialog):
    def __init__(self, *args, abort_message='abort'):
        super(PulseProgressBarDialog, self)\
            .__init__(*args, style=wx.PD_AUTO_HIDE | wx.PD_APP_MODAL)
        pub.subscribe(self.close, CLOSE_PROGRESS_BAR_SIGNAL)
        while self.GetValue() != self.GetRange():
            self.Pulse()

    def close(self):
        self.Update(self.GetRange())

class ReadExcelThread(Thread):
    def __init__(self, filepath, message):
        super(ReadExcelThread, self).__init__()
        self._filepath = filepath
        self._message = message
        self.start()

    def run(self):
        df = pd.read_excel(self._filepath)
        df = df.dropna(how='all').fillna('')
        wx.CallAfter(pub.sendMessage, self._message, df=df)

class BiogramGeneratorThread(Thread):
    def __init__(self, data, date_col, identifier_col, organism_col, indexes, keys,
                 include_count, include_percent, include_narst, columns, drug_data):
        super(BiogramGeneratorThread, self).__init__()
        self.drug_data = drug_data
        self.data = data
        self.date_col = date_col
        self.identifier_col = identifier_col
        self.organism_col = organism_col
        self.columns = columns
        self.indexes = indexes
        self.keys = keys
        self.include_count = include_count
        self.include_percent = include_percent
        self.include_narst = include_narst
        self.start()

    def run(self):
        # TODO: remove hard-coded organism file
        organism_df = pd.read_excel(os.path.join('appdata', 'organisms2020.xlsx'))

        melted_df = self.data.melt(id_vars=self.keys)
        _melted_df = pd.merge(melted_df, organism_df, how='inner')
        _melted_df = pd.merge(_melted_df, self.drug_data,
                              right_on='abbr', left_on='variable', how='outer')
        indexes = [self.columns[idx] for idx in self.indexes]
        total = _melted_df.pivot_table(index=indexes,
                                       columns=['group', 'variable'], aggfunc='count')
        sens = _melted_df[_melted_df['value'] == 'S'].pivot_table(index=indexes,
                                                                  columns=['group', 'variable'],
                                                                  aggfunc='count')
        resists = _melted_df[(_melted_df['value'] == 'I') | (_melted_df['value'] == 'R')] \
            .pivot_table(index=indexes, columns=['group', 'variable'], aggfunc='count')
        biogram_resists = (resists / total * 100).applymap(lambda x: round(x, 2))
        biogram_sens = (sens / total * 100).applymap(lambda x: round(x, 2))
        formatted_total = total.applymap(lambda x: '' if pd.isna(x) else '{:.0f}'.format(x))
        biogram_narst_s = biogram_sens.fillna('-').applymap(str) + " (" + formatted_total + ")"
        biogram_narst_s = biogram_narst_s.applymap(lambda x: '' if x.startswith('-') else x)
        wx.CallAfter(pub.sendMessage, CLOSE_PROGRESS_BAR_SIGNAL)
        wx.CallAfter(pub.sendMessage,
                     WRITE_TO_EXCEL_FILE_SIGNAL,
                     sens=sens if self.include_count else None,
                     resists=resists if self.include_count else None,
                     biogram_sens=biogram_sens if self.include_percent else None,
                     biogram_resists=biogram_resists if self.include_percent else None,
                     biogram_narst_s=biogram_narst_s if self.include_narst else None,
                     identifier_col=self.identifier_col)

class DataRow(object):
    def __init__(self, id, series) -> None:
        self.id = id
        for k, v in series.items():
            setattr(self, k, v)

    def to_list(self, columns):
        return [getattr(self, c) for c in columns]

    def to_dict(self, columns):
        return {c: getattr(self, c) for c in columns}

class BiogramIndexDialog(wx.Dialog):
    def __init__(self, parent, columns, title='Biogram Indexes', start=None, end=None):
        super().__init__(parent, title=title, style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
        self.indexes = []
        self.choices = columns
        main_sizer = wx.BoxSizer(wx.VERTICAL)
        instruction = wx.StaticText(self, label='Data will be organized in hierarchy based on the order in the list.')
        self.chlbox = wx.CheckListBox(self, choices=columns)
        self.chlbox.Bind(wx.EVT_CHECKLISTBOX, self.on_checked)
        self.index_items_list = wx.ListCtrl(self, wx.ID_ANY, style=wx.LC_REPORT, size=(300, 200))
        self.index_items_list.AppendColumn('Level')
        self.index_items_list.AppendColumn('Attribute')
        dateBoxSizer = wx.StaticBoxSizer(wx.HORIZONTAL, self, label='Date Range')
        self.startDate = wx.adv.DatePickerCtrl(self, dt=start)
        startDateLabel = wx.StaticText(self, label='Start')
        self.endDate = wx.adv.DatePickerCtrl(self, dt=end)
        endDateLabel = wx.StaticText(self, label='End')
        dateBoxSizer.Add(startDateLabel, 0, wx.ALL, 5)
        dateBoxSizer.Add(self.startDate, 0, wx.ALL, 5)
        dateBoxSizer.Add(endDateLabel, 0, wx.ALL, 5)
        dateBoxSizer.Add(self.endDate, 0, wx.ALL, 5)
        outputBoxSizer = wx.StaticBoxSizer(wx.VERTICAL, self, label='Output')
        self.includeCount = wx.CheckBox(self, label='Raw counts')
        self.includePercent = wx.CheckBox(self, label='Percents')
        self.includeNarstStyle = wx.CheckBox(self, label='NARST format')
        self.includeCount.SetValue(True)
        self.includePercent.SetValue(True)
        self.includeNarstStyle.SetValue(True)
        outputBoxSizer.Add(self.includeCount, 0, wx.ALL, 5)
        outputBoxSizer.Add(self.includePercent, 0, wx.ALL, 5)
        outputBoxSizer.Add(self.includeNarstStyle, 0, wx.ALL, 5)
        main_sizer.Add(instruction, 0, wx.ALL, 5)
        main_sizer.Add(self.chlbox, 1, wx.ALL | wx.EXPAND, 10)
        main_sizer.Add(self.index_items_list, 1, wx.ALL | wx.EXPAND, 10)
        main_sizer.Add(dateBoxSizer, 0, wx.ALL | wx.EXPAND, 5)
        main_sizer.Add(outputBoxSizer, 0, wx.ALL | wx.EXPAND, 5)
        btn_sizer = wx.StdDialogButtonSizer()
        ok_btn = wx.Button(self, id=wx.ID_OK, label='Generate')
        ok_btn.SetDefault()
        cancel_btn = wx.Button(self, id=wx.ID_CANCEL, label='Cancel')
        btn_sizer.AddButton(ok_btn)
        btn_sizer.AddButton(cancel_btn)
        btn_sizer.Realize()
        main_sizer.Add(btn_sizer, 0, wx.ALL | wx.ALIGN_CENTER, 10)
        main_sizer.SetSizeHints(self)
        self.SetSizer(main_sizer)
        main_sizer.Fit(self)

    def on_checked(self, event):
        item = event.GetInt()
        if not self.chlbox.IsChecked(item):
            idx = self.indexes.index(item)
            self.index_items_list.DeleteItem(idx)
            self.indexes.remove(item)
        else:
            self.indexes.append(item)
            self.index_items_list.Append([len(self.indexes), self.choices[item]])

class NewColumnDialog(wx.Dialog):
    def __init__(self, parent, data, title='Edit values and save to a new column'):
        super().__init__(parent, title=title, style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
        main_sizer = wx.BoxSizer(wx.VERTICAL)
        colname_label = wx.StaticText(self, label='New column name')
        self.colname_ctrl = wx.TextCtrl(self)
        btn_sizer = wx.StdDialogButtonSizer()
        ok_btn = wx.Button(self, id=wx.ID_OK, label='Create')
        ok_btn.SetDefault()
        cancel_btn = wx.Button(self, id=wx.ID_CANCEL, label='Cancel')
        btn_sizer.AddButton(ok_btn)
        btn_sizer.AddButton(cancel_btn)
        btn_sizer.Realize()
        self.olvData = ObjectListView(self, wx.ID_ANY, style=wx.LC_REPORT | wx.SUNKEN_BORDER)
        self.olvData.oddRowsBackColor = wx.Colour(230, 230, 230, 100)
        self.olvData.evenRowsBackColor = wx.WHITE
        self.olvData.cellEditMode = ObjectListView.CELLEDIT_DOUBLECLICK
        self.data = []
        for dt in data:
            self.data.append({'old': dt, 'new': dt})
        self.olvData.SetColumns([
            ColumnDefn(title='Old Value', align='left', minimumWidth=50, valueGetter='old'),
            ColumnDefn(title='New Value', align='left', minimumWidth=50, valueGetter='new'),
        ])
        self.olvData.SetObjects(self.data)
        main_sizer.Add(self.olvData, 1, wx.ALL | wx.EXPAND, 5)
        main_sizer.Add(colname_label, 0, wx.ALL, 5)
        main_sizer.Add(self.colname_ctrl, 0, wx.ALL, 5)
        main_sizer.Add(btn_sizer, 0, wx.ALL | wx.ALIGN_CENTER, 5)
        self.SetAutoLayout(True)
        self.SetSizer(main_sizer)
        main_sizer.Fit(self)

    def replace(self):
        return self.data

class DrugListCtrl(wx.ListCtrl):
    def __init__(self, parent, cols):
        super(DrugListCtrl, self).__init__(parent, style=wx.LC_REPORT, size=(300, 200))
        self.EnableCheckBoxes(True)
        self.Bind(wx.EVT_LIST_ITEM_CHECKED, self.on_check)
        self.Bind(wx.EVT_LIST_ITEM_UNCHECKED, self.on_uncheck)
        self.cols = cols
        self.drugs = []
        self.AppendColumn('Name')
        for col in cols:
            self.Append([col])

        for d in config.Read('Drugs').split(';'):
            if d in self.cols:
                idx = self.cols.index(d)
                self.CheckItem(idx)

    def on_check(self, event):
        item = event.GetItem()
        idx = item.GetId()
        col = self.cols[idx]
        if self.IsItemChecked(idx):
            self.drugs.append(col)

    def on_uncheck(self, event):
        item = event.GetItem()
        idx = item.GetId()
        col = self.cols[idx]
        if col in self.drugs:
            self.drugs.remove(col)

class ConfigDialog(wx.Dialog):
    def __init__(self, parent, columns, title='Configuration'):
        super().__init__(parent, title=title, style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
        main_sizer = wx.BoxSizer(wx.VERTICAL)
        form_sizer = wx.FlexGridSizer(5, 2, 15, 20)
        form_sizer.Add(wx.StaticText(self, id=wx.ID_ANY, label='Identifier'), 0)
        self.identifier_combo_ctrl = wx.Choice(self, choices=columns)
        _col = config.Read('IdentifierCol', '')
        if _col and _col in columns:
            self.identifier_combo_ctrl.SetSelection(columns.index(_col))
        form_sizer.Add(self.identifier_combo_ctrl, 0)
        form_sizer.Add(wx.StaticText(self, id=wx.ID_ANY, label='Date'))
        self.date_combo_ctrl = wx.Choice(self, choices=columns)
        _col = config.Read('DateCol', '')
        if _col and _col in columns:
            self.date_combo_ctrl.SetSelection(columns.index(_col))
        form_sizer.Add(self.date_combo_ctrl, 0)
        form_sizer.Add(wx.StaticText(self, id=wx.ID_ANY, label='Organism Code'))
        self.organism_combo_ctrl = wx.Choice(self, choices=columns)
        _col = config.Read('OrganismCol', '')
        if _col and _col in columns:
            self.organism_combo_ctrl.SetSelection(columns.index(_col))
        form_sizer.Add(self.organism_combo_ctrl, 0)
        form_sizer.Add(wx.StaticText(self, id=wx.ID_ANY, label='Specimens'))
        self.specimens_combo_ctrl = wx.Choice(self, choices=columns)
        _col = config.Read('SpecimensCol', '')
        if _col and _col in columns:
            self.specimens_combo_ctrl.SetSelection(columns.index(_col))
        form_sizer.Add(self.specimens_combo_ctrl, 0)
        self.drug_listctrl = DrugListCtrl(self, columns)
        form_sizer.Add(wx.StaticText(self, id=wx.ID_ANY, label='Drugs'))
        form_sizer.Add(self.drug_listctrl, 1, wx.EXPAND)

        btn_sizer = wx.StdDialogButtonSizer()
        ok_btn = wx.Button(self, id=wx.ID_OK, label='Ok')
        ok_btn.SetDefault()
        cancel_btn = wx.Button(self, id=wx.ID_CANCEL, label='Cancel')
        btn_sizer.AddButton(ok_btn)
        btn_sizer.AddButton(cancel_btn)
        btn_sizer.Realize()

        main_sizer.Add(form_sizer, 0, wx.ALL | wx.EXPAND, 10)
        main_sizer.Add(btn_sizer, 0, wx.ALL | wx.ALIGN_CENTER, 10)
        main_sizer.SetSizeHints(self)
        self.SetSizer(main_sizer)
        main_sizer.Fit(self)

class DeduplicateIndexDialog(wx.Dialog):
    def __init__(self, parent, columns, title='Deduplication Keys'):
        super().__init__(parent, title=title, style=wx.DEFAULT_DIALOG_STYLE | wx.RESIZE_BORDER)
        self.keys = []
        main_sizer = wx.BoxSizer(wx.VERTICAL)
        instruction = wx.StaticText(self, label='Select columns you want to use for deduplication.')
        self.isSortDate = wx.CheckBox(self, label='Sort by the date column')
        self.isSortDate.SetValue(True)
        self.chlbox = wx.CheckListBox(self, choices=columns)
        self.chlbox.Bind(wx.EVT_CHECKLISTBOX, self.on_checked)
        button_sizer = self.CreateStdDialogButtonSizer(flags=wx.OK | wx.CANCEL)
        main_sizer.Add(instruction, 0, wx.ALL, 5)
        main_sizer.Add(self.chlbox, 1, wx.ALL | wx.EXPAND, 5)
        main_sizer.Add(self.isSortDate, 0, wx.ALL, 5)
        main_sizer.Add(button_sizer, 0, wx.ALL, 5)
        self.SetSizer(main_sizer)
        self.Fit()

    def on_checked(self, event):
        item = event.GetInt()
        if not self.chlbox.IsChecked(item):
            idx = self.keys.index(item)
            self.keys.remove(item)
        else:
            self.keys.append(item)

class MainFrame(wx.Frame):
    def __init__(self):
        wx.Frame.__init__(self, parent=None, id=wx.ID_ANY, title="Mivisor Version 2021.1", size=(800, 600))
        panel = wx.Panel(self)
        # TODO: figure out how to update the statusbar's text from the frame's children
        self.statusbar = self.CreateStatusBar(2)
        self.statusbar.SetStatusText('The app is ready to roll.')
        self.statusbar.SetStatusText('This is for the analytics information', 1)
        menuBar = wx.MenuBar()
        fileMenu = wx.Menu()
        registryMenu = wx.Menu()
        menuBar.Append(fileMenu, '&File')
        menuBar.Append(registryMenu, 'Re&gistry')
        loadItem = fileMenu.Append(wx.ID_ANY, 'Load Data', 'Load Data')
        exportItem = fileMenu.Append(wx.ID_ANY, 'Export Data', 'Export Data')
        fileMenu.AppendSeparator()
        fileItem = fileMenu.Append(wx.ID_EXIT, '&Quit', 'Quit Application')
        drugItem = registryMenu.Append(wx.ID_ANY, 'Drugs', 'Drug Registry')
        self.SetMenuBar(menuBar)
        self.Bind(wx.EVT_MENU, lambda x: self.Close(), fileItem)
        self.Bind(wx.EVT_MENU, self.open_drug_dialog, drugItem)
        self.Bind(wx.EVT_MENU, self.export_data, exportItem)
        self.Bind(wx.EVT_MENU, self.open_load_data_dialog, loadItem)

        self.Bind(wx.EVT_CLOSE, self.OnClose)
        self.Center()
        self.Maximize(True)

        self.load_drug_data()

        self.df = pd.DataFrame()
        self.data = []
        self.colnames = []
        self.organism_col = config.Read('OrganismCol', '')
        self.identifier_col = config.Read('IdentifierCol', '')
        self.date_col = config.Read('DateCol', '')
        self.specimens_col = config.Read('SpecimensCol', '')
        self.drugs_col = config.Read('Drugs', '').split(';') or []
        main_sizer = wx.BoxSizer(wx.VERTICAL)
        btn_sizer = wx.BoxSizer(wx.HORIZONTAL)
        load_button = wx.Button(panel, label="Load")
        self.copy_button = wx.Button(panel, label="Copy Column")
        self.config_btn = wx.Button(panel, label='Config')
        self.generate_btn = wx.Button(panel, label='Generate')
        load_button.Bind(wx.EVT_BUTTON, self.open_load_data_dialog)
        self.copy_button.Bind(wx.EVT_BUTTON, self.copy_column)
        self.config_btn.Bind(wx.EVT_BUTTON, self.configure)
        self.generate_btn.Bind(wx.EVT_BUTTON, self.generate)

        self.dataOlv = FastObjectListView(panel, wx.ID_ANY,
                                          style=wx.LC_REPORT | wx.SUNKEN_BORDER)
        self.dataOlv.oddRowsBackColor = wx.Colour(230, 230, 230, 100)
        self.dataOlv.evenRowsBackColor = wx.WHITE
        self.dataOlv.cellEditMode = ObjectListView.CELLEDIT_DOUBLECLICK
        self.dataOlv.SetEmptyListMsg('Welcome to Mivisor Version 2021.1')
        self.dataOlv.SetObjects([])
        main_sizer.Add(self.dataOlv, 1, wx.ALL | wx.EXPAND, 10)
        btn_sizer.Add(load_button, 0, wx.ALL, 5)
        btn_sizer.Add(self.copy_button, 0, wx.ALL, 5)
        btn_sizer.Add(self.config_btn, 0, wx.ALL, 5)
        btn_sizer.Add(self.generate_btn, 0, wx.ALL, 5)
        main_sizer.Add(btn_sizer, 0, wx.ALL, 5)
        panel.SetSizer(main_sizer)
        panel.Fit()

        self.disable_buttons()

        pub.subscribe(self.disable_buttons, DISABLE_BUTTONS)
        pub.subscribe(self.enable_buttons, ENABLE_BUTTONS)
        pub.subscribe(self.write_output, WRITE_TO_EXCEL_FILE_SIGNAL)

    def OnClose(self, event):
        if event.CanVeto():
            if wx.MessageBox('You want to quit the program?', 'Please confirm', style=wx.YES_NO) != wx.YES:
                event.Veto()
                return
        event.Skip()

    def disable_buttons(self):
        self.generate_btn.Disable()
        self.copy_button.Disable()
        self.config_btn.Disable()

    def enable_buttons(self):
        self.generate_btn.Enable()
        self.copy_button.Enable()
        self.config_btn.Enable()

    def export_data(self, event):
        df = pd.DataFrame([d.to_dict(self.colnames) for d in self.data])
        if df.empty:
            with wx.MessageDialog(self, 'No data to export. Please load data first.',
                                  'Export Data', style=wx.OK) as dlg:
                dlg.ShowModal()
                return
        with wx.FileDialog(self, "Please select the output file for data",
                           wildcard="Excel file (*xlsx)|*xlsx",
                           style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as file_dialog:
            if file_dialog.ShowModal() == wx.ID_CANCEL:
                return
            file_path = file_dialog.GetPath()
            if os.path.splitext(file_path)[1] != '.xlsx':
                file_path = file_path + '.xlsx'
            try:
                df.to_excel(file_path, index=False)
            except:
                with wx.MessageDialog(self, 'Export failed.', 'Export Data', style=wx.OK) as dlg:
                    dlg.ShowModal()
            else:
                with wx.MessageDialog(self, 'Export completed.', 'Export Data', style=wx.OK) as dlg:
                    dlg.ShowModal()

    def load_drug_data(self):
        try:
            drug_df = pd.read_json(os.path.join('appdata', 'drugs.json'))
        except:
            pass
        if drug_df.empty:
            drug_df = pd.DataFrame(columns=['drug', 'abbreviation', 'group'])

        drug_list = []
        drug_df = drug_df.sort_values(['group'])
        for idx, row in drug_df.iterrows():
            if row['abbreviation']:
                abbrs = [a.strip().upper() for a in row['abbreviation'].split(',')]
            else:
                abbrs = []
            for ab in abbrs:
                drug_list.append({'drug': row['drug'], 'abbr': ab, 'group': row['group']})
        self.drug_data = pd.DataFrame(drug_list)

    def set_data_olv(self, df):
        self.df = df
        self.df = self.df.dropna(how='all').fillna('')
        self.setColumns()
        self.data = [DataRow(idx, row) for idx, row in self.df.iterrows()]
        self.dataOlv.SetObjects(self.data)
        pub.sendMessage(CLOSE_PROGRESS_BAR_SIGNAL)
        pub.sendMessage(ENABLE_BUTTONS)

    def read_data_from_file(self):
        with wx.FileDialog(self, "Load data from file",
                           style=wx.FD_OPEN | wx.FD_FILE_MUST_EXIST,
                           wildcard="Excel file (*.xlsx)|*.xlsx") as file_dialog:
            if file_dialog.ShowModal() == wx.ID_CANCEL:
                return
            filepath = file_dialog.GetPath()
            pub.subscribe(self.set_data_olv, 'load_excel_data_finished')
            ReadExcelThread(filepath, 'load_excel_data_finished')
            progress_bar = PulseProgressBarDialog('Loading Data', 'Reading data from {}'.format(filepath))

    def open_load_data_dialog(self, event):
        pub.sendMessage(DISABLE_BUTTONS)
        if not self.df.empty:
            with wx.MessageDialog(self, "Load new dataset?", "Load data", style=wx.YES_NO) as msg_dialog:
                if msg_dialog.ShowModal() == wx.ID_YES:
                    self.read_data_from_file()
        else:
            self.read_data_from_file()

    def open_drug_dialog(self, event):
        with DrugRegFormDialog() as drug_dlg:
            drug_dlg.ShowModal()

    def setColumns(self):
        columns = []
        for c in self.df.columns:
            self.colnames.append(c)
            col_type = str(self.df.dtypes.get(c))
            if col_type.startswith('int') or col_type.startswith('float'):
                formatter = '%.1f'
            elif col_type.startswith('datetime'):
                formatter = '%Y-%m-%d'
            else:
                formatter = '%s'
            columns.append(
                ColumnDefn(title=c.title(), align='left', stringConverter=formatter, valueGetter=c, minimumWidth=50))
        self.dataOlv.SetColumns(columns)

    def copy_column(self, event):
        with wx.SingleChoiceDialog(self, 'Select Source Column', 'Source Column', choices=self.colnames) as dlg:
            if dlg.ShowModal() == wx.ID_OK:
                idx = dlg.GetSelection()
                colname = self.colnames[idx]
                data = set([getattr(row, colname) for row in self.data])
                with NewColumnDialog(self, data) as dlg:
                    if dlg.ShowModal() == wx.ID_OK:
                        new_data = dlg.replace()
                        lookup_dict = {}
                        new_colname = dlg.colname_ctrl.GetValue()
                        for item in new_data:
                            lookup_dict[item['old']] = item['new']
                        for d in self.data:
                            old_value = getattr(d, colname)
                            setattr(d, new_colname, lookup_dict.get(old_value))
                        self.dataOlv.AddColumnDefn(ColumnDefn(
                            title=new_colname,
                            align='left',
                            valueGetter=new_colname,
                            minimumWidth=50,
                        ))
                        self.colnames.append(new_colname)
                        self.dataOlv.RepopulateList()

    def configure(self, event):
        with ConfigDialog(self, self.colnames) as dlg:
            if dlg.ShowModal() == wx.ID_OK:
                if dlg.identifier_combo_ctrl.GetSelection() != -1:
                    self.identifier_col = self.colnames[dlg.identifier_combo_ctrl.GetSelection()]
                if dlg.date_combo_ctrl.GetSelection() != -1:
                    self.date_col = self.colnames[dlg.date_combo_ctrl.GetSelection()]
                if dlg.organism_combo_ctrl.GetSelection() != -1:
                    self.organism_col = self.colnames[dlg.organism_combo_ctrl.GetSelection()]
                if dlg.specimens_combo_ctrl.GetSelection() != -1:
                    self.specimens_col = self.colnames[dlg.specimens_combo_ctrl.GetSelection()]
                self.drugs_col = dlg.drug_listctrl.drugs
                config.Write('IdentifierCol', self.identifier_col)
                config.Write('DateCol', self.date_col)
                config.Write('OrganismCol', self.organism_col)
                config.Write('SpecimensCol', self.specimens_col)
                config.Write('Drugs', ';'.join(self.drugs_col))

    def melt(self, source_data=None):
        if source_data is None:
            source_data = self.data
        keys = []
        for c in self.colnames:
            if c not in self.drugs_col:
                keys.append(c)
        data = [row.to_list(self.colnames) for row in source_data]
        ids = [row.id for row in data]
        df = pd.DataFrame(data=data, index=ids, columns=self.colnames)
        return df.melt(id_vars=keys)

    def generate(self, event):
        if not all([self.date_col, self.identifier_col, self.organism_col]):
            self.configure()
        df = pd.DataFrame([d.to_dict(self.colnames) for d in self.data])
        if df.empty:
            with wx.MessageDialog(self, 'No data provided. Please load data from an Excel file',
                                  'Error', style=wx.OK) as dlg:
                if dlg.ShowModal() == wx.ID_OK:
                    return
        num_rows = len(df)
        with DeduplicateIndexDialog(self, [c for c in self.colnames
                                           if c not in self.drugs_col]) as dlg:
            if dlg.ShowModal() == wx.ID_OK:
                data = df
                if dlg.isSortDate.GetValue():
                    data = data.sort_values(config.Read('DateCol'), ascending=True)
                if dlg.keys:
                    data = data.drop_duplicates(subset=[self.colnames[k] for k in dlg.keys],
                                                keep='first')
                if num_rows == len(data):
                    message = 'No duplicates found.'
                else:
                    message = f'{ num_rows - len(data) } duplicates were removed.'
                with wx.MessageDialog(self, message, 'Deduplication Finished', style=wx.OK) as dlg:
                    dlg.ShowModal()
            else:
                return
        keys = []
        for c in self.colnames:
            if c not in self.drugs_col:
                keys.append(c)
        columns = [c for c in self.colnames if c not in self.drugs_col] + ['GENUS', 'SPECIES', 'GRAM']
        if self.identifier_col in columns:
            columns.remove(self.identifier_col)
        if self.date_col in columns:
            columns.remove(self.date_col)

        with BiogramIndexDialog(self, columns,
                                start=data[self.date_col].min(),
                                end=data[self.date_col].max()) as dlg:
            if dlg.ShowModal() == wx.ID_OK and dlg.indexes:
                # filter data within the date range
                data = data[(data[self.date_col].dt.date >= dlg.startDate.GetValue())
                            & (data[self.date_col].dt.date <= dlg.endDate.GetValue())]
                BiogramGeneratorThread(data,
                                       self.date_col,
                                       self.identifier_col,
                                       self.organism_col,
                                       dlg.indexes,
                                       keys,
                                       dlg.includeCount.GetValue(),
                                       dlg.includePercent.GetValue(),
                                       dlg.includeNarstStyle.GetValue(),
                                       columns,
                                       self.drug_data)
                progress_bar = PulseProgressBarDialog('Generating Antibiogram', 'Calculating...')
            else:
                return

    def write_output(self, sens, resists, biogram_sens, biogram_resists, biogram_narst_s,
                     identifier_col):
        with wx.FileDialog(self, "Please select the output file for your antibiogram",
                           wildcard="Excel file (*xlsx)|*xlsx",
                           style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) as file_dialog:
            if file_dialog.ShowModal() == wx.ID_CANCEL:
                return
            file_path = file_dialog.GetPath()
            if os.path.splitext(file_path)[1] != '.xlsx':
                file_path = file_path + '.xlsx'
        try:
            with pd.ExcelWriter(file_path, engine='xlsxwriter') as writer:
                if sens is not None:
                    sens[identifier_col].to_excel(writer, sheet_name='count_S')
                if resists is not None:
                    resists[identifier_col].to_excel(writer, sheet_name='count_R')
                if biogram_sens is not None:
                    biogram_sens[identifier_col].to_excel(writer, sheet_name='percent_S')
                if biogram_resists is not None:
                    biogram_resists[identifier_col].to_excel(writer, sheet_name='percent_R')
                if biogram_narst_s is not None:
                    biogram_narst_s[identifier_col].to_excel(writer, sheet_name='narst_s')
        except:
            with wx.MessageDialog(self, 'Failed', 'Antibiogram Generator', style=wx.OK) as dlg:
                dlg.ShowModal()
        else:
            with wx.MessageDialog(self, 'Output Saved.', 'Antibiogram Generator', style=wx.OK) as dlg:
                dlg.ShowModal()

class GenApp(wx.App):
    def __init__(self, redirect=False, filename=None):
        wx.App.__init__(self, redirect, filename)

    def OnInit(self):
        # create frame here
        global config
        config = wx.Config('Mivisor')
        frame = MainFrame()
        self.SetTopWindow(frame)
        frame.Show()
        return True