"""Functions for generating multilayer and multiplex networks using various network models.
"""

##### Compatibility for Python 2/3
try:
    xrange
except NameError:
    xrange = range
######

from .net import MultilayerNetwork,MultiplexNetwork
import math,random

def single_layer_conf(net,degs,degstype="distribution"):
    """Generates a realization of configuration model network.

    Parameters
    ----------
    net : MultilayerNetwork with aspects=0
       Empty network object that is to be filled.
    degs : dict
       Degrees of the network. See degstype parameter.
    degstype : string
       If 'distribution', then degs parameter gives the degree distribution. I.e.,
       keys are degrees, and corresponding values are number of nodes with the given degree.
       If 'nodes', then degs parameter gives node degrees. I.e, keys are node names and
       corresponding values are degrees of those nodes.

    Notes
    -----
    The algorithm used here is similar to the one in article:
    B.D McKay, N.C Wormald 'Uniform Generation of Random Regular Graphs of Moderate Degree'
    Journal of Algorithms 11, pages 52-67 (1990)

    The difference between the algorithm presented in the article and the one in this
    function is that the random restarts are not implemented here. This means that the
    sampled networks are not exactly statistically uniform. However, if the degrees
    are small compared to the number of nodes the error is likely to be small.
    """
    stubs=[]
    selfedges={}
    multiedges=set()
    edgetoindex={}

    if degstype=="distribution":
        nstubs=sum(map(lambda x:x[0]*x[1],degs.items()))
        nodes=sum(degs.values())
        shuffled_node_indices=list(range(nodes))
        random.shuffle(shuffled_node_indices)
        node=0
        for k,num in degs.items():
            if k==0:
                for i in range(num):
                    net.add_node(shuffled_node_indices[node])
                    node+=1
            else:
                for i in range(num):
                    for j in range(k):
                        stubs.append(shuffled_node_indices[node])
                    node+=1
    elif degstype=="nodes":
        nstubs=sum(degs.values())
        nodes=len(degs)
        #for node,k in degs.iteritems():
        for node in degs:
            k=degs[node]
            for i in range(k):
                stubs.append(node)
            if k==0:
                net.add_node(node)
    else:
        raise Exception("Invalid degstype: '"+str(degstype)+"'")

    # Here we should do the Erdos-Gallai test
    assert nstubs%2==0
    assert (nodes*(nodes-1)) >= nstubs

    random.shuffle(stubs)

    for s in range(int(len(stubs)/2)):
        node1,node2=sorted([stubs[2*s],stubs[2*s+1]])

        edgetoindex[(node1,node2)]=edgetoindex.get((node1,node2),[])+[2*s]

        if net[node1,node2]!=0:
            multiedges.add((node1,node2))

        if node1==node2:
            selfedges[node1]=selfedges.get(node1,[])+[2*s]
        else:
            net[node1,node2]=1

    for node,sis in selfedges.items():
        for si in sis:
            repeat=True
            while repeat:
                #select two edges at random
                e1i,e2i=map(lambda x:2*x,random.sample(xrange(int(len(stubs)/2)),2))
                c=[node,stubs[e1i],stubs[e1i+1],stubs[e2i],stubs[e2i+1]]
                n2,n3=sorted([c[1],c[2]])
                n4,n5=sorted([c[3],c[4]])
                if len(set(c))==len(c):
                    if (n2,n3) not in multiedges and (n4,n5) not in multiedges:
                        if net[node,n2]==0 and net[node,n4]==0 and net[n3,n5]==0:
                            net[n2,n3]=0
                            net[n4,n5]=0
                            net[node,n2]=1
                            net[node,n4]=1
                            net[n3,n5]=1
                            stubs[si],stubs[si+1]=sorted([n3,n5])
                            stubs[e1i],stubs[e1i+1]=sorted([node,n2])
                            stubs[e2i],stubs[e2i+1]=sorted([node,n4])
                            repeat=False

    # Uncomment to check that everything ok so far:
    #import diagnostics
    #assert sum(map(lambda x:x[0]*x[1],diagnostics.degs(net).items()))/2.+sum(map(lambda x:len(edgetoindex[x[0],x[1]])-1,multiedges))==sum(map(lambda x:x[0]*x[1],degs.items()))/2.

    #for s in range(len(stubs)/2):
    #    n1,n2=stubs[2*s],stubs[2*s+1]
    #    assert net[n1,n2]==1,str(2*s)

    for n1,n2 in multiedges:
        for dummy in range(int(math.floor(len(edgetoindex[(n1,n2)])/2.))):
            repeat=True
            while repeat:
                #select two edges at random
                e1i,e2i=map(lambda x:2*x,random.sample(xrange(int(len(stubs)/2)),2))
                c=[n1,n2,stubs[e1i],stubs[e1i+1],stubs[e2i],stubs[e2i+1]]
                n3,n4=sorted([c[2],c[3]])
                n5,n6=sorted([c[4],c[5]])
                if len(set(c))==len(c):
                    if (n3,n4) not in multiedges and (n5,n6) not in multiedges:
                        if net[n1,n3]==0 and net[n2,n4]==0 and net[n1,n5]==0 and net[n2,n6]==0:
                            if len(edgetoindex[n1,n2])==2:
                                net[n1,n2]=0
                            assert net[n3,n4]==1
                            assert net[n5,n6]==1
                            net[n3,n4]=0
                            net[n5,n6]=0
                            net[n1,n3]=1
                            net[n2,n4]=1
                            net[n1,n5]=1
                            net[n2,n6]=1
                            si1,si2=sorted([edgetoindex[n1,n2].pop(),edgetoindex[n1,n2].pop()])
                            stubs[si1],stubs[si1+1]=sorted([n1,n3])
                            stubs[si2],stubs[si2+1]=sorted([n2,n4]) #stubs[si1],stubs[si1+1]=sorted([n2,n4])
                            stubs[e1i],stubs[e1i+1]=sorted([n1,n5])
                            stubs[e2i],stubs[e2i+1]=sorted([n2,n6])
                            repeat=False

def single_layer_er(net,nodes,p=None,edges=None):
    """Generates a realization of a monoplex Erdos-Renyi network.

    Parameters
    ----------
    net : MultilayerNetwork with aspects=0
       Empty network object that is to be filled.
    nodes : iterable
       Sequence of node labels.
    p : float
       Probability that edges is present.
    edges : int
       Number of edges that are present.

    References
    ----------
    Efficient generation of large random networks. PRE 71, 036113 (2005)
    """

    if (p==None and edges==None) or (p!=None and edges!=None):
        raise Exception("Give one of the parameters: p or edges.")

    n=len(nodes)
    for node in nodes:
        net.add_node(node)

    if p!=None:
        if p==1.0:
            for node1 in nodes:
                for node2 in nodes:
                    if node1!=node2:
                        net[node1,node2]=1
        else:
            v,w=1,-1
            while (v < n):
                r=random.random()
                w=w+1+int(math.floor(math.log(1-r)/math.log(1-p)))
                while ((w >= v) and (v < n)):
                    w = w-v
                    v = v+1
                if (v < n):
                    net[nodes[v],nodes[w]]=1
    else:
        for edge_index in random.sample(xrange(int((n*(n-1))/2)),edges):
            v=int(1+math.floor(-0.5+math.sqrt(0.25+2*edge_index)))
            w=edge_index-int((v*(v-1))/2)
            net[nodes[v],nodes[w]]=1

def conf(degs,degstype="distribution",couplings=("categorical",1.0)):
    """Independent configuration model for multiplex networks.

    Parameters
    ----------
    degs : dict, dict of dicts, list of dicts, MultiplexNetwork, MultilayerNetwork
       Degrees. If dict, a monoplex network is returned. If dict of dicts, a multiplex network with
       keys as layer names is returned. If list of dicts, then a multiplex network with a layer for
       each element in the list is returned. See degstype parameter for the description of the dict
       used for describing intra-layer networks. If MultiplexNetwork (with 1 aspect) or MultilayerNetwork
       (with 0 aspects) object is given then a copy of that network is produced with configuration
       model.

    degstype : string
       If 'distribution', then degs dicts give the degree distributions. I.e.,
       keys are degrees, and corresponding values are number of nodes with the given degree.
       If 'nodes', then degs dicts give node degrees. I.e, keys are node names and
       corresponding values are degrees of those nodes.

    couplings : tuple
       The coupling types of the multiplex network object.

    Returns
    -------
    net : MultiplexNetwork
       The (multiplex) network produced with the configuration model.

    See also
    --------
    single_layer_conf : the function used to generate a network on each layer

    """
    if isinstance(degs,MultiplexNetwork):
        assert degs.aspects==1
        d={}
        for layer in degs.iter_layers():
            dd={}
            d[layer]=dd
            for node in degs.A[layer]:
                dd[node]=degs.A[layer][node].deg()
        return conf(d,degstype="nodes")
    elif isinstance(degs,MultilayerNetwork):
        assert degs.aspects==0
        d={}
        for node in degs:
            d[node]=degs[node].deg()
        return conf(d,degstype="nodes")
    #elif isinstance(degs,dict) and not isinstance(degs.itervalues().next(),dict):
    elif isinstance(degs,dict) and not isinstance(degs[(k for k in degs).send(None)],dict):
        net=MultilayerNetwork(aspects=0)
        single_layer_conf(net,degs,degstype=degstype)
    else:
        #check if the network is going to be node-aligned
        namedlayers=isinstance(degs,dict)
        if namedlayers:
            degslist=degs.values()
        else:
            degslist=degs
        nnodes=None
        nodeAligned=True
        if degstype=="distribution":
            for ldegs in degslist:
                lnnodes=sum(ldegs.values())
                if nnodes!=None and lnnodes!=nnodes:
                    nodeAligned=False
                nnodes=lnnodes
        elif degstype=="nodes":
            for ldegs in degslist:
                lnnodes=set(ldegs.keys())
                if nnodes!=None and lnnodes!=nnodes:
                    nodeAligned=False
                nnodes=lnnodes
        else:
            raise Exception()

        net=MultiplexNetwork(couplings=[couplings],fullyInterconnected=nodeAligned)
        if namedlayers:
            #layers=degs.iteritems()
            layers = ((node, degs[node]) for node in degs )
        else:
            layers=enumerate(degs)
        for l,ldegs in layers:
            net.add_layer(l)
            single_layer_conf(net.A[l],ldegs,degstype=degstype)

    return net

def er(n,p=None,edges=None):
    """Multiplex Erdos-Renyi model.

    Parameters
    ----------
    n : int, list of lists of nodes
       Number of nodes, or lists of nodes in each layer if network is not fully
       interconnected.
    p : float or list of floats
       Connection probability, or list of connection probabilities for each layer.
    edges : int or list of int
       Number of edges, or list of number of edges in each layer.

    Returns
    -------
    net : MultiplexNetwork
       The (multiplex) network produced.

    See also
    --------
    single_layer_er : the function used to generate a network on each layer
    """
    # What kind of network?
    fic = not hasattr(n,'__iter__') #fully interconnected
    monoplex = (not hasattr(p,'__iter__')) and (not hasattr(edges,'__iter__')) and fic

    # Sanity check for parameters
    if (p==None and edges==None) or (p!=None and edges!=None):
        raise Exception("Give one of the parameters: p or edges.")
    if not fic:
        if hasattr(p,'__iter__'):
            assert len(n)==len(p)
        elif hasattr(edges,'__iter__'):
            assert len(n)==len(edges)

    # Create the network
    if monoplex:
        net=MultilayerNetwork(aspects=0)
    else:
        net=MultiplexNetwork(couplings=[('categorical',1.0)],fullyInterconnected=fic)
        if not hasattr(n,'__iter__'):
            if p!=None:
                nodes=list(map(lambda x:xrange(n),p))
                layers=xrange(len(p))
            else:
                nodes=list(map(lambda x:xrange(n),edges))
                layers=xrange(len(edges))
        else:
            nodes=n
            layers=xrange(len(n))
            if p!=None and (not hasattr(p,'__iter__')):
                p=list(map(lambda x:p,layers))
            if edges!=None and (not hasattr(edges,'__iter__')):
                edges=list(map(lambda x:edges,layers))

    # Fill in the edges
    if p!=None:
        if monoplex:
            single_layer_er(net,range(n),p=p)
        else:
            for l,lp,lnodes in zip(layers,p,nodes):
                net.add_layer(l)
                single_layer_er(net.A[l],lnodes,p=lp)
    else:
        if monoplex:
            single_layer_er(net,range(n),edges=edges)
        else:
            for l,ledges,lnodes in zip(layers,edges,nodes):
                net.add_layer(l)
                single_layer_er(net.A[l],lnodes,edges=ledges)

    return net

def er_partially_interconnected(nodes,ps,couplings=('categorical',1.0)):
    """Generate multiplex Erdos-Renyi network which is not fully interconnected.

    The produced multiplex network has a single aspect.

    Parameters
    ----------
    nodes : list of lists
       List of lists of nodes, where each list corresponds to
       nodes in one layer.
    ps : list
       List of edge occupation probabilities for layers
    couplings : tuple
       The coupling types of the multiplex network object.

    Returns
    -------
    net : MultiplexNetwork
       The multiplex network that is produced.
    """
    assert len(nodes)==len(ps)
    net=MultiplexNetwork(couplings=[couplings],fullyInterconnected=False)
    for layer,lnodes in enumerate(nodes):
        net.add_layer(layer)
        single_layer_er(net.A[layer],lnodes,ps[layer])
    return net

def full(nodes,layers,couplings=('categorical',1.0)):
    """Generate a full multiplex network.

    The produced multiplex network has a single aspect and is fully
    interconnected. Can also produce a full monoplex network.

    Parameters
    ----------
    nodes : int
       Number of nodes in the network
    layers : int, sequence or None
       Number of layers in the network, a sequence of layer names, or
       None for monoplex networks.
    couplings : tuple
       The coupling types of the multiplex network object.

    Returns
    -------
    net : MultiplexNetwork or MultilayerNetwork
       The multiplex network that is produced, or the monoplex
       network (which is of type MultilayerNetwork).
    """
    if layers==None:
        n=MultilayerNetwork(aspects=0)
        for node1 in range(nodes):
            for node2 in range(nodes):
                if node1!=node2:
                    n[node1,node2]=1
    elif not hasattr(layers,'__iter__'): #is not sequence
        n=MultiplexNetwork(couplings=[couplings])
        for layer in range(layers):
            for node1 in range(nodes):
                for node2 in range(nodes):
                    if node1!=node2:
                        n[node1,node2,layer,layer]=1
    else: #it's a sequence
        n=MultiplexNetwork(couplings=[couplings])
        for layer in layers:
            for node1 in range(nodes):
                for node2 in range(nodes):
                    if node1!=node2:
                        n[node1,node2,layer,layer]=1

    return n

def full_multilayer(nodes,layers):
    """Generate a full multilayer network.

    The generated network has a single aspect, and all the inter-layer
    and intra-layer edges.

    Parameters
    ----------
    nodes : int
       Number of nodes in the network
    layers : int or sequence
       Number of layers in the network, or a sequence of layer names

    Returns
    -------
    net : MultilayerNetwork
       The multilayer network that is produced.
    """
    if not hasattr(layers,'__iter__'): #is not sequence
        layers=range(layers)

    n=MultilayerNetwork(aspects=1)
    for layer1 in layers:
        for layer2 in layers:
            for node1 in range(nodes):
                for node2 in range(nodes):
                    if node1!=node2 or layer1!=layer2:
                        n[node1,node2,layer1,layer2]=1
    return n

def er_multilayer(nodes,layers,p,randomWeights=False):
    """Generate multilayer Erdos-Renyi network.

    The produced multilayer network has a single aspect.

    Parameters
    ----------
    nodes : int
       Number of nodes in the network
    layers : int or sequence
       Number of layers in the network, or a sequence of layer names
    p : float
       The edge probability
    randomWeights : bool
       If true the weights are uniformly random between (0,1].

    Returns
    -------
    net : MultilayerNetwork
       The multilayer network that is produced.
    """

    if not hasattr(layers,'__iter__'): #is not sequence
        layers=range(layers)

    n=MultilayerNetwork(aspects=1)
    for layer1 in layers:
        for layer2 in layers:
            for node1 in range(nodes):
                for node2 in range(node1+1,nodes):
                    if node1!=node2 or layer1!=layer2:
                        if random.random()<p:
                            if randomWeights:
                                n[node1,node2,layer1,layer2]=random.random()
                            else:
                                n[node1,node2,layer1,layer2]=1

    return n

def conf_overlaps(ol_degs, couplings=None):
    """
    Generate multiplex configuration model network with given overlap degree
    sequences.

    One can specify the 'overlap degree sequences', as defined in overlap_degs.

    Parameters
    ----------
    ol_degs : dict of dicts
        The overlap degrees. Keys are tuples containing layer combinations
        (including the trivial combination of a single layer) and
        values are the overlap degree distributions with nodes as keys. See
        overlap_degs.
    couplings : None or tuple
        The coupling types passed directly to multiplex network object
        constructor.

    Returns
    -------
    net : MultiplexNetwork
       The multiplex network produced with the configuration model.

    Notes
    -----
    The algorithm to produce the multiplex network is a modified version of the
    one implemented in single_layer_conf (McKay et al). The modification tries
    link swapping in cases where edges in another layer combination already
    exists.

    References
    ----------
    Marceau, Vincent, et al. "Modeling the dynamical interaction between
    epidemics on overlay networks." Physical Review E 84.2 (2011): 026105.
    """

    net = MultiplexNetwork(couplings=couplings)
    used_edges = set()

    for layer_comb in ol_degs:
        degs = ol_degs[layer_comb]

        # mostly modified from single_layer_conf
        stubs=[]
        selfedges={}
        multiedges=set()
        takenedges=set()
        edgetoindex={}

        net_temp = MultilayerNetwork()

        nstubs=sum(degs.values())
        n_nodes=len(degs)
        for node in degs:
            k=degs[node]
            for i in range(k):
                stubs.append(node)
            if k==0:
                net_temp.add_node(node)

        # Here we should do the Erdos-Gallai test
        assert nstubs%2==0
        assert (n_nodes*(n_nodes-1)) >= nstubs

        random.shuffle(stubs)

        for s in range(int(len(stubs)/2)):
            node1,node2=sorted([stubs[2*s],stubs[2*s+1]])

            edgetoindex[(node1,node2)]=edgetoindex.get((node1,node2),[])+[2*s]

            if node1==node2:
                selfedges[node1]=selfedges.get(node1,[])+[2*s]
            elif net_temp[node1,node2]!=0:
                multiedges.add((node1,node2))
            elif (node1, node2) in used_edges:
                takenedges.add((node1,node2))
            else:
                net_temp[node1,node2]=1
                used_edges.add((node1, node2))

        for node,sis in selfedges.items():
            for si in sis:
                repeat=True
                while repeat:
                    #select two edges at random
                    e1i,e2i=map(lambda x:2*x,random.sample(xrange(int(len(stubs)/2)),2))
                    c=[node,stubs[e1i],stubs[e1i+1],stubs[e2i],stubs[e2i+1]]
                    n2,n3=sorted([c[1],c[2]])
                    n4,n5=sorted([c[3],c[4]])
                    if len(set(c))==len(c):
                        if (n2,n3) not in multiedges and (n4,n5) not in multiedges and (n2,n3) not in takenedges and (n4,n5) not in takenedges:
                            e1 = tuple(sorted([node, n2]))
                            e2 = tuple(sorted([node, n4]))
                            e3 = tuple(sorted([n3, n5]))
                            if e1 not in used_edges and e2 not in used_edges and e3 not in used_edges:
                                net_temp[n2,n3]=0
                                net_temp[n4,n5]=0
                                net_temp[node,n2]=1
                                net_temp[node,n4]=1
                                net_temp[n3,n5]=1
                                stubs[si],stubs[si+1]=sorted([n3,n5])
                                stubs[e1i],stubs[e1i+1]=sorted([node,n2])
                                stubs[e2i],stubs[e2i+1]=sorted([node,n4])
                                repeat=False

                                used_edges.remove((n2,n3))
                                used_edges.remove((n4,n5))
                                used_edges.add(e1)
                                used_edges.add(e2)
                                used_edges.add(e3)

        for n1,n2 in multiedges:
            for dummy in range(int(math.floor(len(edgetoindex[(n1,n2)])/2.))):
                repeat=True
                while repeat:
                    #select two edges at random
                    e1i,e2i=map(lambda x:2*x,random.sample(xrange(int(len(stubs)/2)),2))
                    c=[n1,n2,stubs[e1i],stubs[e1i+1],stubs[e2i],stubs[e2i+1]]
                    n3,n4=sorted([c[2],c[3]])
                    n5,n6=sorted([c[4],c[5]])
                    if len(set(c))==len(c):
                        if (n3,n4) not in multiedges and (n5,n6) not in multiedges and (n3,n4) not in takenedges and (n5,n6) not in takenedges:
                            e1 = tuple(sorted([n1,n3]))
                            e2 = tuple(sorted([n2,n4]))
                            e3 = tuple(sorted([n1,n5]))
                            e4 = tuple(sorted([n2,n6]))
                            if e1 not in used_edges and e2 not in used_edges and e3 not in used_edges and e4 not in used_edges: #net[n1,n3]==0 and net[n2,n4]==0 and net[n1,n5]==0 and net[n2,n6]==0:
                                if len(edgetoindex[n1,n2])==2:
                                    net_temp[n1,n2]=0
                                    used_edges.remove((n1,n2))
                                net_temp[n3,n4]=0
                                net_temp[n5,n6]=0
                                net_temp[n1,n3]=1
                                net_temp[n2,n4]=1
                                net_temp[n1,n5]=1
                                net_temp[n2,n6]=1
                                si1,si2=sorted([edgetoindex[n1,n2].pop(),edgetoindex[n1,n2].pop()])
                                stubs[si1],stubs[si1+1]=sorted([n1,n3])
                                stubs[si2],stubs[si2+1]=sorted([n2,n4])#stubs[si1],stubs[si1+1]=sorted([n2,n4])
                                stubs[e1i],stubs[e1i+1]=sorted([n1,n5])
                                stubs[e2i],stubs[e2i+1]=sorted([n2,n6])
                                repeat=False

                                used_edges.remove((n3,n4))
                                used_edges.remove((n5,n6))
                                used_edges.add(e1)
                                used_edges.add(e2)
                                used_edges.add(e3)
                                used_edges.add(e4)

        for n1,n2 in takenedges:
            for si1 in edgetoindex[(n1,n2)]:
                repeat=True
                while repeat:
                    #select two edges at random
                    e1i,e2i=map(lambda x:2*x,random.sample(xrange(int(len(stubs)/2)),2))
                    c=[n1,n2,stubs[e1i],stubs[e1i+1],stubs[e2i],stubs[e2i+1]]
                    n3,n4=sorted([c[2],c[3]])
                    n5,n6=sorted([c[4],c[5]])
                    if len(set(c))==len(c):
                        if (n3,n4) not in takenedges and (n5,n6) not in takenedges:
                            e1 = tuple(sorted([n1,n3]))
                            e2 = tuple(sorted([n2,n5]))
                            e3 = tuple(sorted([n4,n6]))
                            if e1 not in used_edges and e2 not in used_edges and e3 not in used_edges:
                                assert net_temp[n3,n4]==1
                                assert net_temp[n5,n6]==1
                                net_temp[n3,n4]=0
                                net_temp[n5,n6]=0
                                net_temp[n1,n3]=1
                                net_temp[n2,n5]=1
                                net_temp[n4,n6]=1

                                stubs[si1],stubs[si1+1]=sorted([n1,n3])
                                stubs[e1i],stubs[e1i+1]=sorted([n2,n5])
                                stubs[e2i],stubs[e2i+1]=sorted([n4,n6])
                                repeat=False

                                used_edges.remove((n3,n4))
                                used_edges.remove((n5,n6))
                                used_edges.add(e1)
                                used_edges.add(e2)
                                used_edges.add(e3)

        for e in net_temp.edges:
            for layer in layer_comb:
                net[e[0], e[1], layer] = 1

    return net

def er_overlaps_match_aggregated(n, edges, ps, couplings=None):
    '''
    Generates a multiplex Erdos-Renyi networks which produces an aggregated
    network with given number of edges. The target it that the aggregated
    network of the resulting network has edges * n_layers edges.

    The algorithm goes through each of the user-given layer combinations with
    2 or more layers. It generates an ER graph which does not include any edges
    that are already in any of the layers of the combination. It then copies
    the edges of that ER graph to all of the layers of the combination.

    Parameters
    ----------
    n : int
        Number of nodes
    edges : int
        Number of edges in each layer
    ps : dict
        Proportions of overlapping edges in each layer combination given as
        keys (note that the sum of these proportions should not exceed 1 for
        any one layer). The trivial combinations including only a single layer
        do not need to be given (and if they are given, the given proportions
        are ignored).
    couplings : None or tuple
        The coupling types of the multiplex network object.

    Returns
    -------
    net : MultiplexNetwork
       The multiplex network produced
    '''

    net = MultiplexNetwork(couplings=couplings)
    used_edges = set()

    e_left = {}
    for layer_comb in ps:
        if len(layer_comb) == 1:
            e_left[layer_comb[0]] = e_left.get(layer_comb[0], edges)
            continue
        p = ps[layer_comb]
        m = int(round(p*edges))
        k = len(layer_comb)
        if k > 1:
            for layer in layer_comb:
                e_left[layer] = e_left.get(layer, edges) - m
            i = 0
            while i < (k * m):
                edge_index = random.choice(xrange(int((n*(n-1))/2)))
                v=int(1+math.floor(-0.5+math.sqrt(0.25+2*edge_index)))
                w=edge_index-int((v*(v-1))/2)
                edge = (w, v)
                if edge not in used_edges:
                    used_edges.add(edge)
                    i += 1
                    for layer in layer_comb:
                        net[edge[0], edge[1], layer] = 1

    for layer in e_left:
        m = e_left[layer]
        i = 0
        while i < m:
            edge_index = random.choice(xrange(int((n*(n-1))/2)))
            v=int(1+math.floor(-0.5+math.sqrt(0.25+2*edge_index)))
            w=edge_index-int((v*(v-1))/2)
            edge = (w, v)
            if edge not in used_edges:
                used_edges.add(edge)
                i += 1
                net[edge[0], edge[1], layer] = 1

    return net

def ba_total_degree(n, ms, couplings=None):
    """
    Generates a Barabasi-Albert multiplex network, where the preferential
    attachment process is run for each layer separately and concurrently
    in a way that the total degree of the node is used for the preferential
    attachment. That is, the new nodes attach preferentially to nodes that
    have high total degree (sum of degree in all layers).

    Parameters
    ----------
    n : int
        number of nodes
    ms : list of ints
        the numbers of links added to each new node for each layer
    couplings : None or tuple
        The coupling types of the multiplex network object.

    Returns
    -------
    net : MultiplexNetwork
       The multiplex network produced

    References
    ----------
    Kim, Jung Yeol, and K-I. Goh. "Coevolution and correlated multiplexity in
    multiplex networks." Physical review letters 111.5 (2013): 058702.
    """

    net = MultiplexNetwork(couplings=couplings)
    links = []

    for i in range(n):
        net.add_node(i)
        link_sets = []
        for layer, m in enumerate(ms):
            if i == m:
                link_set = set(range(i))

            elif i > m:
                link_set = set()
                while len(link_set) < m:
                    link_set = link_set | set(random.sample(links, m - len(link_set)))

            else:
                link_set = set()

            link_sets.append(link_set)

            for j in link_set:
                net[i,j,layer] = 1

        for link_set in link_sets:
            links += list(link_set)
            links += [i]*(len(link_set))

    return net

try:
    from . import nxwrap as nx
    import networkx

    def ws(n, edges, p=0.3, couplings=None):
        """
        Generates a multiplex network where each layer is generated
        using the same Watts-Strogatz model.

        Parameters
        ----------
        n : int
            Number of nodes
        edges : list of ints
            Number of edges in each layer
        p : float
            Probability of rewiring an edge
        couplings : None or tuple
            The coupling types of the multiplex network object.

        Returns
        -------
        net : MultiplexNetwork
           The multiplex network produced
        """

        net = MultiplexNetwork(couplings=couplings)
        for layer, m in enumerate(edges):
            net.add_layer(layer)
            net.A[layer] = nx.watts_strogatz_graph(n, int(math.ceil(2*m / float(n))), p=p)

        return net

    def geo(n, edges, couplings=None):
        """
        Generates a multiplex network where each layer is generated using the
        same soft geometric network model.

        The intra-layer networks are generated using the networkx function
        soft_random_geometeric_graph.

        Parameters
        ----------
        n : int
            Number of nodes
        edges : list of ints
            Approximate number of edges in each layer
        couplings : None or tuple
            The coupling types of the multiplex network object.

        Returns
        -------
        net : MultiplexNetwork
           The multiplex network produced

        Notes
        -----
        Works only with networkX2
        """

        net = MultiplexNetwork(couplings=couplings)
        pos = None
        for layer, m in enumerate(edges):
            net.add_layer(layer)
            r = math.sqrt(2.2 * float(m) / ((n - 1.0) * n) / math.pi)
            netX = networkx.soft_random_geometric_graph(n, r, pos=pos)
            pos = networkx.get_node_attributes(netX, 'pos')
            for node in netX.nodes:
                net.add_node(node)

            for e in netX.edges:
                net[e[0], e[1], layer] = 1

        return net
except ImportError:
    pass


# -*- coding: utf-8 -*-

from django.core import exceptions
from django.conf import settings
from django.db.models import fields

class BigAutoField(fields.AutoField):
    def db_type(self, connection=None):
        for item in settings.DATABASES:
            if settings.DATABASES[item]['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
                return "bigserial"
            else:
                raise NotImplemented

    def get_internal_type(self):
        return "BigAutoField"

    def to_python(self, value):
        if value is None:
            return value
        try:
            return long(value)
        except (TypeError, ValueError):
            raise exceptions.ValidationError(
                _("This value must be a long integer."))


import os
import numpy as np
import pandas as pd

from loguru import logger
from sklearn.model_selection import train_test_split

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier

from data_quality.config import *

def tokenize(sentences_train, sentences_test, vocab_size, col_count):
    tokenizer = Tokenizer(num_words=vocab_size + 1)
    tokenizer.fit_on_texts(sentences_train)

    X_train = tokenizer.texts_to_sequences(sentences_train)
    X_test = tokenizer.texts_to_sequences(sentences_test)

    vocab_size = len(tokenizer.word_index) + 1

    maxlen = col_count + 1

    X_train = pad_sequences(X_train, padding="post", maxlen=maxlen)
    X_test = pad_sequences(X_test, padding="post", maxlen=maxlen)

    return X_train, X_test

def classifier(vocab_size, col_count):
    embedding_dim = EMBEDDING_DIM
    opt = tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE)

    model = Sequential()
    model.add(
        layers.Embedding(
            input_dim=vocab_size, output_dim=embedding_dim, input_length=col_count + 1
        )
    )

    model.add(layers.Conv1D(embedding_dim, 5, activation="relu"))
    model.add(layers.GlobalMaxPooling1D())
    model.add(layers.Dropout(DROPOUT_RATE))
    model.add(layers.BatchNormalization())
    model.add(layers.Dense(HIDDEN_DIM, activation="relu"))
    model.add(layers.Dropout(DROPOUT_RATE))
    model.add(layers.Dense(col_count + 1, activation="sigmoid"))
    model.compile(optimizer=opt, loss="binary_crossentropy")
    model.summary()

def prepare_data_for_classsifier(df, vocab_size, col_count):
    sentences = df["predicted_sentence"].values
    masked_indices = df["masked_index"].values

    b = np.zeros((masked_indices.size, masked_indices.max() + 1))
    b[np.arange(masked_indices.size), masked_indices] = 1

    y = df["label"].values
    y = np.concatenate((y.reshape(-1, 1), b), axis=1)

    sentences_train, sentences_test, y_train, y_test = train_test_split(
        sentences, y, test_size=0.2, random_state=1000
    )

    x_train, x_test = tokenize(sentences_train, sentences_test, vocab_size, col_count)

    return x_train, x_test, y_train, y_test

def train_classifier(classifier_data_path, vocab_size, col_count):
    df = pd.read_csv(classifier_data_path, vocab_size, col_count)

    x_train, x_test, y_train, t_test = prepare_data_for_classsifier(
        df, vocab_size, col_count
    )
    _ = model.fit(
        x_train,
        y_train,
        epochs=CLF_TRAIN_EPOCHS,
        verbose=True,
        validation_data=(x_test, y_test),
        batch_size=CLF_TRAIN_BATCH_SIZE,
    )


from typing import Dict

import sys

import os

from org.pyut.PyutConstants import PyutConstants
from org.pyut.general.exceptions.PreferencesLocationNotSet import PreferencesLocationNotSet
from org.pyut.preferences.BaseSubPreference import BaseSubPreference

PREFS_NAME_VALUES = Dict[str, str]

class PreferencesCommon(BaseSubPreference):

    preferencesFileLocationAndName: str = None

    def init(self, *args, **kwds):

        BaseSubPreference.init(self, *args, **kwds)

    @staticmethod
    def determinePreferencesLocation():
        """
        This method MUST (I repeat MUST) be called before attempting to instantiate the preferences Singleton
        """
        if sys.platform == "linux2" or sys.platform == "linux" or sys.platform == PyutConstants.THE_GREAT_MAC_PLATFORM:
            PreferencesCommon.preferencesFileLocationAndName = os.getenv("HOME") + "/.PyutPrefs.dat"
        else:
            PreferencesCommon.preferencesFileLocationAndName = "PyutPrefs.dat"

    @staticmethod
    def getPreferencesLocation():
        if PreferencesCommon.preferencesFileLocationAndName is None:
            raise PreferencesLocationNotSet()
        else:
            return PreferencesCommon.preferencesFileLocationAndName

    def addMissingPreference(self, sectionName: str, preferenceName: str, value: str):
        self._config.set(sectionName, preferenceName, value)
        self.saveConfig()

    def saveConfig(self):
        """
        Save data to the preferences file
        """
        f = open(PreferencesCommon.getPreferencesLocation(), "w")
        self._config.write(f)
        f.close()


# importing required libraries
import json
from urllib.request import urlopen
name=input("Please Enter your Name : ")
# create an free account in www.gender-api.com
# account activation will take some time...
# after activation get the authentication key and use it in mykey
mykey="paste your key here"
#url to open and get the response
url="https://gender-api.com/get?name="
#concatinating the name you entered and your api key..
try:
   url=url+f"{name}&key={mykey}"
#opening the final url...
   respose=urlopen(url)
#decoding the data to json format
   decode=respose.read().decode("utf-8")
   data=json.loads(decode)
# printing the gender of the name you entered...
   print("Gender : "+data["gender"])
except(Exception):
    print("please enter the valid name..")


import random
import numpy as np
import matplotlib.pyplot as plt
import sys

#參數
learn_rate = 0.2
epoch = 100
accuracy = 0.9

#input data
data = []
sol = []
sol_class = []

#store final solution
best_ac = 0.0
best_w = []
final_ac = 0.0

def data_input(path):
     #read file
     file_c = False
     while(file_c == False):
          try:
               f = open(path)
               file_c = True
          except:
               print("can't find file")
               return False

     for line in f:
          #repace '\n' to ''
          line = line.replace('\n', '')
          #split and string to int
          sp = line.split(" ")
          data_x = [-1]
          data_y = 0
          count = 0
          for item in sp:
               if(count == (len(sp)-1)):
                    data_y = int(item)
               else:
                    data_x.append(float(item))
               count = count + 1

          data.append(data_x)
          sol.append(data_y)

          #紀錄class label
          if len(sol_class) == 0:
               sol_class.append(data_y)
          else:
               ch = True
               for item in sol_class:
                    if(item == data_y):
                         ch = False
                         break
               if(ch == True):
                    sol_class.append(data_y)
     f.close()

def paint(data, sol, status ):
     x = []
     y = []
     x2 = []
     y2 = []
     for i  in range(len(data)):
          for j in range(len(data[i])):

               if(sol[i] == 1):
                    x.append(data[i][1])
                    y.append(data[i][2])
               else:
                    x2.append(data[i][1])
                    y2.append(data[i][2])
     max_x = 0
     min_x = 0
     max_y = 0
     min_y = 0
     # 座標最大 最小
     if(len(x) != 0 and len(x2)!= 0):
          max_x = max([max(x), max(x2)])
          min_x = min([min(x), min(x2)])
     elif(len(x) != 0):
          max_x = max(x)
          min_x = min(x)
     elif(len(x2)!= 0):
          max_x = max(x2)
          min_x = min(x2)

     if(len(y) != 0 and len(y2)!= 0):
          max_y = max([max(y), max(y2)])
          min_y = min([min(y), min(y2)])
     elif(len(y) != 0):
          max_y = max(y)
          min_y = min(y)
     elif(len(y2)!= 0):
          max_y = max(y2)
          min_y = min(y2)

     plt.plot(x, y ,'r^')
     plt.plot(x2, y2 ,'gs')
     if(best_w[2] != 0):
          plt.plot([max_x, min_x], [(best_w[0] - best_w[1] * max_x)/best_w[2], (best_w[0] - best_w[1] * min_x)/best_w[2]] ,'y--')
     else:
          plt.plot([max_x, min_x], [max_y, min_y] ,'y--')

     #存檔的path
     path_spilt = path.split('\\')
     path_spilt = path.split('/')
     name = (path_spilt[len(path_spilt)-1].split('.'))[0]
     try:
          if(status == 0):
               plt.savefig('dataset/image/' + name + '_train_data.jpg')
          else:
               plt.savefig('dataset/image/' + name + '_all_data.jpg')
     except:
          print("--------*******************************************  -----")
          print("--------   can't find path, path is dataset/image/   -----")
          print("--------*******************************************  -----")
     plt.show()

def data_split():
     data_train = []
     sol_train = []
     data_test = []
     sol_test = []
     #use random()
     for j in range(len(data)):
          if(random.random() >= 0.33):
               data_train.append(data[j])
               sol_train.append(sol[j])
          else:
               data_test.append(data[j])
               sol_test.append(sol[j])

          #防止data_test沒有資料
          if(len(data_test) == 0):
               ran = random.randint(0, len(data_train)-1)
               data_test.append(data_train[ran])
               sol_test.append(sol[j])
               del data_train[ran]
               del sol_train[ran]

     return data_train, sol_train, data_test, sol_test

def train():
     global best_ac
     global best_w
     global accuracy
     global final_ac

     #初始weight

     w = [-1]
     #初始weight
     if(len(sol_class) > 2):
          for i in range(len(sol_class)):
               w.append(random.random())
     else:
          w = [-1,1, 0]

     data_train = []
     data_test = []
     sol_train = []
     sol_test = []
     predict = 0

     pre_ac = 0
     pre_w = []
     data_train, sol_train, data_test, sol_test = data_split()

     #start train
     for i in range(epoch):
          predict = 0
          for j in range(len(data_train)):
               sum = 0
               for k in range(len(data_train[j])):
                    sum = sum + data_train[j][k] * w[k]

               if(sum >= 0):
                    predict = sol_class[0]
                    if(predict != sol_train[j]):
                         for k in range(len(w)):
                              w[k] = w[k] - (learn_rate)/(1+i/10) * data_train[j][k]
               if(sum < 0):
                    predict = sol_class[1]
                    if(predict != sol_train[j]):
                         for k in range(len(w)):
                              w[k] = w[k] + (learn_rate)/(1+i/10) * data_train[j][k]

          #evalute
          count = 0
          for j in range(len(data_train)):
               sum = 0
               for k in range(len(data_train[j])):
                    sum = sum + data_train[j][k] * w[k]

               if(sum >= 0):
                    predict = sol_class[0]
               if(sum < 0):
                    predict = sol_class[1]
               if(predict == sol[j]):
                    count = count + 1
          print("epoch: " + str(i+1) + "\ntrain accuracy:", count/len(data_train))

          count = 0
          for j in range(len(data_test)):
               sum = 0
               for k in range(len(data_test[j])):
                    sum = sum + data_test[j][k] * w[k]

               if(sum >= 0):
                    predict = sol_class[0]
               if(sum < 0):
                    predict = sol_class[1]
               if(predict == sol_test[j]):
                    count = count + 1
          ac = count/len(data_test)

          if(pre_ac > ac):
               w = pre_w
               ac = pre_ac
          else:
               pre_ac = ac
               pre_w = w

          #record best accuracy
          if(best_ac < ac):
               best_ac = ac
               best_w = w

          print("test accuracy:", str(ac))

          count = 0
          for j in range(len(data)):
               sum = 0
               for k in range(len(data[j])):
                    sum = sum + data[j][k] * w[k]

               if(sum >= 0):
                    predict = sol_class[0]
               if(sum < 0):
                    predict = sol_class[1]
               if(predict == sol[j]):
                    count = count + 1

          final_ac = count/len(data)
          print("total_accuracy", final_ac)
          if(best_ac >= accuracy):
               paint(data_train, sol_train, 0)
               return

     if(len(sol_class) <= 2):
          paint(data_train, sol_train, 0)

if __name__ == '__main__':
     while(True):
          data = []
          sol = []
          sol_class = []
          best_ac = 0.0
          best_w = []
          final_ac = 0.0
          path = str(input("檔案路徑: "))
          c1 = False
          c2 = False
          c3 = False
          while(True):
               if(c1 == False):
                    try:
                         learn_rate = float(input("type:float learning_rate: "))
                         break
                    except:
                         print("learning_rate error")
          while(True):
               if(c2 == False):
                    try:
                         epoch = int(input("type:int epoch: "))
                         break
                    except:
                         print("epoch error")
          while(True):
               if(c3 == False):
                    try:
                         accuracy = float(input("type:float accuracy: "))
                         break
                    except:
                         print("accuracy error")

          if(data_input(path) != False):
               train()
               print("test data  accuracy: " + str(best_ac),)
               print("wieght ", end = "")
               print(best_w)

               print("all data accuracy: ", final_ac)
               if(len(sol_class) <= 2):
                    paint(data, sol, 1)
          print("")

          status = int(input("input->1 continue, input->2 stop: "))
          if(status == 2):
               sys.exit()


#!/usr/bin/env python3

from importlib import import_module

import time
import attr

from labgrid.driver import Driver
from labgrid.factory import target_factory
from labgrid.protocol import PowerProtocol
from labgrid.step import step

@target_factory.reg_driver
@attr.s(eq=False)
class ModbusRTUPowerDriver(Driver, PowerProtocol):
    bindings = {"resource": "ModbusRTU", }

    downtime = attr.ib(default=1.0, validator=attr.validators.instance_of(float))
    delay = attr.ib(default=0.5, validator=attr.validators.instance_of(float))
    coil = attr.ib(default=0, validator=attr.validators.instance_of(int))

    def __attrs_post_init__(self):
        super().__attrs_post_init__()
        self._modbus = import_module('minimalmodbus')
        self.instrument = None

    def on_activate(self):
        self.instrument = self._modbus.Instrument(
            self.resource.port,
            self.resource.address,
            debug=False)

        self.instrument.serial.baudrate = self.resource.speed
        self.instrument.serial.timeout = self.resource.timeout
        self.instrument.mode = self._modbus.MODE_RTU
        self.instrument.clear_buffers_before_each_transaction = True

    def on_deactivate(self):
        self.instrument = None

    @Driver.check_active
    @step()
    def on(self):
        self.instrument.write_bit(self.coil, 1, 5)
        time.sleep(self.delay)

    @Driver.check_active
    @step()
    def off(self):
        self.instrument.write_bit(self.coil, 0, 5)
        time.sleep(self.delay)

    @Driver.check_active
    @step()
    def cycle(self):
        self.off()
        time.sleep(self.downtime)
        self.on()

    @Driver.check_active
    @step()
    def get(self):
        return self.instrument.read_bit(self.coil, 1)


import wget
import os

str1 = "https://www.ncei.noaa.gov/data/sea-surface-temperature-optimum-interpolation/v2.1/access/avhrr/"
str2 = "oisst-avhrr-v02r01."
str3 = ".nc"
for year in range(1981, 1982):
    for month in range(9, 11):
        for day in range(15, 20):
            url = str1 + "{}".format(year) + "{:02d}".format(month) + "/" + str2 + "{}".format(
                year)+"{:02d}".format(month) + "{:02d}".format(day) + str3
            path1 = '../avhrr-only'
            file_path = os.path.join(
                path1, "{}".format(year), "{:02d}".format(month))
            if not os.path.exists(file_path):
                os.makedirs(file_path)

            try:
                wget.download(url, out=file_path)
            except:
                print("File:" + str2 + "{0}{1:02d}{2:02d}".format(year,
                      month, day) + str3 + " does not exist")


import argparse
import codecs
import json
import struct
import xml.etree.ElementTree as ElementTree
import os

import zlib

import dicttoxml
import yaml

class NodeType:
    Node = 0x00
    Boolean = 0x00
    Float = 0x01
    Int = 0x02
    Vector2 = 0x03
    Vector3 = 0x04
    Vector4 = 0x06
    String = 0x07
    Actor = 0x08
    UnknownString = 0x0f
    UnknownUnsignedInt = 0x11
    String2 = 0x14

    Values = [
        0x00,
        0x01,
        0x02,
        0x07,
        0x08,
        0x0f,
        0x11,
        0x14
    ]

    Reference = [
    ]

class AAMP:
    data_object = {}
    hash_table = {}

    def __init__(self, path):
        print("Parsing AAMP file...")

        filename = os.path.basename(path)
        print("Reading {0}...".format(filename))

        file = open(path, 'rb')
        self.data = file.read()

        signature = self.data[0x00:0x04]

        if signature != b'AAMP':
            print('\033[31mQuitting: {0} is not a AAMP file\033[0m'.format(filename))
            print('\033[31mExpected b\'AAMP\' but saw {0}\033[0m'.format(signature))
            exit(0)

        version = struct.unpack('<I', self.data[0x04:0x08])[0]
        if version != 2:
            print('\033[31mQuitting: {0} is not the correct AAMP version\033[0m'.format(filename))
            print('\033[31mExpected 2 but saw {0}\033[0m'.format(version))
            exit(0)

        # Get hashed names
        self.get_hash_table()

        root_nodes_length = struct.unpack('<I', self.data[0x18:0x1c])[0]
        pos = 0x34

        for index in range(0, root_nodes_length):
            children = {}

            node_id, unknown, offset, child_count = \
                struct.unpack('<IIHH', self.data[pos:pos + 0x0c])

            if node_id in self.hash_table:
                node_id = self.hash_table[node_id]

            node_id = str(node_id)

            self.data_object[node_id] = {}

            child_pos = offset * 4 + pos
            for child_index in range(0, child_count):
                child_node_id = struct.unpack('<I', self.data[child_pos:child_pos + 0x04])[0]
                if child_node_id in self.hash_table:
                    child_node_id = self.hash_table[child_node_id]

                child_node_id = str(child_node_id)

                children[child_node_id] = self.get_node(child_pos)
                child_pos += 0x08

            self.data_object[node_id] = children
            pos += 0x0c

    def get_hash_table(self):
        file = open('C:\\botw-data\\src\\extractors\\hashed_names.txt', 'r')
        data = file.read()
        data = data.split('\n')

        for index in range(0, len(data)):
            self.hash_table[zlib.crc32(bytearray(data[index], 'utf-8'))] = data[index]

        file = open('C:\\botw-data\\src\\extractors\\hash-number-appendix.txt', 'r')
        data = file.read()
        data = data.split('\n')

        for index in range(0, len(data)):
            self.hash_table[zlib.crc32(bytearray(data[index], 'utf-8'))] = data[index]

        file.close()

    def get_node(self, pos):
        node = {}

        node_id, offset, child_count, child_node_type \
            = struct.unpack('<IHBB', self.data[pos:pos + 0x08])

        if node_id in self.hash_table:
            node_id = self.hash_table[node_id]

        node_id = str(node_id)

        offset = offset * 4 + pos

        # print("Node id: {0}, Offset: {1}, Child Count: {2}, Child Node Type: {3}"
        #       .format(node_id, hex(offset), child_count, hex(child_node_type)))

        if child_node_type == NodeType.Node and child_count > 0:
            children = []
            for index in range(0, child_count):
                child = self.get_node(offset)
                node[child[0]] = child[1]
                offset += 0x08
            return node

        # Node = 0x00
        # Boolean = 0x00
        # Float = 0x01
        # Int = 0x02
        # Vector2 = 0x03
        # Vector3 = 0x04
        # Vector4 = 0x06
        # String = 0x07
        # Actor = 0x08
        # UnknownString = 0x0f
        # UnknownUnsignedInt = 0x11
        # String2 = 0x14

        elif child_node_type == NodeType.Boolean:
            value = struct.unpack('<I', self.data[offset:offset + 0x04])[0]
            value = True if value == 1 else False
            node[node_id] = value

        elif child_node_type == NodeType.Float:
            value = struct.unpack('<f', self.data[offset:offset + 0x04])[0]
            node[node_id] = value

        elif child_node_type == NodeType.Int:
            value = struct.unpack('<I', self.data[offset:offset + 0x04])[0]
            node[node_id] = value

        elif child_node_type == NodeType.String:
            value = self.data[offset:].decode('utf-8')
            value = value.split('\x00')
            value = value[0]
            node[node_id] = value

        elif child_node_type == NodeType.Actor:
            value = self.data[offset:].decode('utf-8')
            value = value.split('\x00')
            value = value[0]
            node[node_id] = value

        elif child_node_type == NodeType.String2:
            value = self.data[offset:].decode('utf-8')
            value = value.split('\x00')
            value = value[0]
            node[node_id] = value

        else:
            value = self.data[offset:offset + 0x04]

        return node_id, value

def main():
    parser = argparse.ArgumentParser(description="Parse the Legend of Zelda: Breath of the Wild aamp files to xml")
    parser.add_argument("filename", type=str, help="File to be parsed.")
    parser.add_argument("-x", "--xml",
                        help="Exports data as a xml file (default)",
                        action="store_true")
    parser.add_argument("-y", "--yaml",
                        help="Exports data as a yaml file",
                        action="store_true")
    parser.add_argument("-j", "--json",
                        help="Exports data as a json file",
                        action="store_true")
    parser.add_argument("-a", "--all",
                        help="Exports data as a xml, yaml and json file",
                        action="store_true")

    args = parser.parse_args()

    aamp = AAMP(args.filename)

    if args.all:
        args.yaml = True
        args.json = True
        args.xml = True

    if args.yaml:
        save_as_yaml(args, aamp)

    if args.json:
        save_as_json(args, aamp)

    if args.xml:
        save_as_xml(args, aamp)

    if not args.yaml and not args.json and not args.xml:
        save_as_xml(args, aamp)

def save_as_yaml(args, byml):
    filename = os.path.basename(args.filename)
    print('Saving {0}.yaml...'.format(filename))
    file = codecs.open(args.filename + '.yaml', 'w', 'utf-8')
    yaml.dump(byml.data_object, file, allow_unicode=True)
    file.close()

def save_as_json(args, byml):
    filename = os.path.basename(args.filename)
    print('Saving {0}.json...'.format(filename))
    file = codecs.open(args.filename + '.json', 'w', 'utf-8')
    json.dump(byml.data_object, file, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ': '))
    file.close()

def save_as_xml(args, byml):
    from xml.dom.minidom import parseString
    filename = os.path.basename(args.filename)
    path = os.path.dirname(os.path.abspath(args.filename))
    base_filename = os.path.splitext(filename)[0]
    print('Saving {0}...'.format(path + '\\' + base_filename + '.xml'))
    file = codecs.open(path + '\\' + base_filename + '.xml', 'w', 'utf-8')
    dom = dicttoxml.dicttoxml(byml.data_object).decode('utf-8')
    file.write(parseString(dom).toprettyxml())
    file.close()

if __name__ == "__main__":
    main()


"""
Extract fields at a number of sections which may be used later for TEF analysis
of transport and other properties, making use of multiple subprocesses
to speed up operation.  It also runs the process_sections.py and bulk_calc.py jobs
unless you use -test True.  You need to run for at least three days to get any
results at the end, because of the tidal averaging.

All input parameters are specified at the command line, so this can be run in the background.
This is essential for year-long extractions.

PERFORMANCE: 6 hours per year, perigee, cas6_v3_lo8b, 39 sections, all variables.  About 6.5 hours
including the processing and bulk_calc steps.  2.25 hours when just extracting salt.

To test on mac (default is to just get salt on section ai1):
run extract_sections -gtx cas6_v3_lo8b -ro 2 -0 2019.07.04 -1 2019.07.06

To get all sections and all variables use these flags:
-get_bio True -sect_name all

On perigee use -Nproc 20

"""

from lo_tools import Lfun, zrfun
from lo_tools import extract_argfun as exfun
Ldir = exfun.intro() # this handles the argument passing
import tef_fun
if Ldir['testing']:
    from importlib import reload
    reload(tef_fun)

import sys
from time import time
import numpy as np
import netCDF4 as nc
import pickle
from subprocess import Popen as Po
from subprocess import PIPE as Pi

# set list of variables to extract
if Ldir['get_bio']:
    vn_list = tef_fun.vn_list
else:
    vn_list = ['salt']

ds0 = Ldir['ds0']
ds1 = Ldir['ds1']

tt00 = time()
print(' Doing TEF extraction for '.center(60,'='))
print(' gtagex = ' + Ldir['gtagex'])
outname = 'extractions_' + ds0 + '_' + ds1
print(' outname = ' + outname)

# make sure the output directory exists
out_dir = Ldir['LOo'] / 'extract' / Ldir['gtagex'] / 'tef' / outname
Lfun.make_dir(out_dir, clean=True)

# make the scratch directory for holding temporary files
temp_dir = Ldir['LOo'] / 'extract' / Ldir['gtagex'] / ('tef_temp_' + ds0 + '_' + ds1)
Lfun.make_dir(temp_dir, clean=True)

# get the DataFrame of all sections
gridname=Ldir['gtagex'].split('_')[0]
sect_df = tef_fun.get_sect_df(gridname)

# initialize a dictionary of info for each section
sect_info = dict()
# select which sections to extract
if Ldir['sect_name'] == 'all':
    # full list
    sect_list = [item for item in sect_df.index]
else: # single item
    if Ldir['sect_name'] in sect_df.index:
        sect_list = [Ldir['sect_name']]
    else:
        print('That section is not available')
        sys.exit()

# get list of history files to process
fn_list = Lfun.get_fn_list('hourly', Ldir, ds0, ds1)
NT = len(fn_list)

# get grid info
fn = fn_list[0]
G = zrfun.get_basic_info(fn, only_G=True)
S = zrfun.get_basic_info(fn, only_S=True)
NZ = S['N']

# Create and save the sect_info dict
# - make a dictionary of info for each section
sect_info = dict()
for sect_name in sect_list:
    x0, x1, y0, y1 = sect_df.loc[sect_name,:]
    # - get indices for this section
    ii0, ii1, jj0, jj1, sdir, Lon, Lat, Mask = tef_fun.get_inds(x0, x1, y0, y1, G)
    NX = len(Mask)
    # - save some things for later use
    sect_info[sect_name] = (ii0, ii1, jj0, jj1, sdir, NX, Lon, Lat)
info_fn = temp_dir / 'sect_info.p'
pickle.dump(sect_info, open(info_fn, 'wb'))

# Initialize NetCDF output files
sl = sect_list.copy()
while len(sl) > 0:
    if len(sl) >= 10:
        print(sl[:10])
        sl = sl[10:]
    else:
        print(sl)
        sl = []
for sect_name in sect_list:
    out_fn = out_dir / (sect_name + '.nc')
    ii0, ii1, jj0, jj1, sdir, NX, Lon, Lat = sect_info[sect_name]
    tef_fun.start_netcdf(fn, out_fn, NT, NX, NZ, Lon, Lat, Ldir, vn_list)

print('Doing initial data extraction:')
# We do extractions one hour at a time, as separate subprocess jobs.
# Running Nproc (e.g. 20) of these in parallel makes the code much faster.
# Files are saved to temp_dir.
tt000 = time()
proc_list = []
N = len(fn_list)
for ii in range(N):
    fn = fn_list[ii]
    d = fn.parent.name.replace('f','')
    nhis = int(fn.name.split('.')[0].split('_')[-1])
    cmd_list = ['python3', 'extract_section_one_time.py',
            '-pth', str(Ldir['roms_out']),
            '-out_dir',str(temp_dir),
            '-gtagex', Ldir['gtagex'],
            '-d', d, '-nhis', str(nhis),
            '-get_bio', str(Ldir['get_bio'])]
    proc = Po(cmd_list, stdout=Pi, stderr=Pi)
    proc_list.append(proc)

    Nproc = Ldir['Nproc']
    if ((np.mod(ii,Nproc) == 0) and (ii > 0)) or (ii == N-1):
        tt0 = time()
        for proc in proc_list:
            proc.communicate()
        print(' - %d out of %d: %d took %0.2f sec' % (ii, N, Nproc, time()-tt0))
        sys.stdout.flush()
        proc_list = []
print('Elapsed time = %0.2f sec' % (time()-tt000))

# Extract and save time-dependent fields
for sect_name in sect_list:
    out_fn = out_dir / (sect_name + '.nc')
    tef_fun.add_fields(out_fn, temp_dir, sect_name, vn_list, S, NT)

# Clean up
Lfun.make_dir(temp_dir, clean=True)
temp_dir.rmdir()

# Then do the processing and bulk calculation
if Ldir['testing'] == False:

    # processing
    tt0 = time()
    print(' process_sections '.center(60,'='))
    cmd_list = ['python3', 'process_sections.py',
            '-gtagex', Ldir['gtagex'],
            '-0', ds0, '-1', ds1]
    proc = Po(cmd_list, stdout=Pi, stderr=Pi)
    stdout, stderr = proc.communicate()
    #print(stdout.decode())
    if len(stderr) > 0:
        print(stderr.decode())
    print('Elapsed time = %0.2f sec' % (time()-tt0))

    # bulk_calc
    tt0 = time()
    print(' bulk_calc '.center(60,'='))
    cmd_list = ['python3', 'bulk_calc.py',
            '-gtagex', Ldir['gtagex'],
            '-0', ds0, '-1', ds1]
    proc = Po(cmd_list, stdout=Pi, stderr=Pi)
    stdout, stderr = proc.communicate()
    #print(stdout.decode())
    if len(stderr) > 0:
        print(stderr.decode())
    print('Elapsed time = %0.2f sec' % (time()-tt0))

print(' Total elapsed time = %d sec '.center(60,'-') % (time()-tt00))
print(' DONE '.center(60,'='))


import os
import re
import shutil
import signal
import sys
import textract
from tempfile import mkstemp
from pathlib import Path
from prompt_toolkit import prompt
from prompt_toolkit.validation import Validator, ValidationError
from prompt_toolkit.completion import WordCompleter

TestCompleter = WordCompleter(['aa', 'zz', 'hush little baby'])

srcpath = Path('/home/gbirke/SynologyDrive/scans')
dest_path = Path('/home/gbirke/SynologyDrive/Dokumente')
preview_link = Path('/tmp/sort_scan_preview.pdf')

destinations = [d.stem for d in dest_path.glob('*') if d.is_dir()]

def create_preview_copy(preview_file):
    _, tmp_name = mkstemp()
    shutil.copy(preview_file, tmp_name)
    os.rename(tmp_name, preview_link)

class DestinationValidator(Validator):
    def validate(self, document):
        text = document.text.strip()
        if text in destinations:
            return
        raise ValidationError(message='Destination not valid')

DestinationList = WordCompleter(destinations)

def signal_handler(sig, frame):
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)

date_match = re.compile(r'^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})')
normalize_whitespace = re.compile(r'\s+')

for srcfile in srcpath.glob('*.pdf'):
    print('{0}'.format(srcfile))
    create_preview_copy(srcfile)
    text = textract.process(srcfile)
    raw_wordlist = normalize_whitespace.sub(' ', str(text, 'utf-8')).split(' ')
    unique_words=set([w for w in raw_wordlist if len(w) > 2])
    NameCompleter = WordCompleter(list(unique_words))
    newname = prompt('File name: ', completer=NameCompleter)

    newname = newname.strip()
    split_date_format = date_match.sub(r"\1_\2_\3_\4_\5_\6", srcfile.stem)
    full_name = "{0} {1}{2}".format(split_date_format, newname, srcfile.suffix)

    destination = prompt('Destination: ',
        validator=DestinationValidator(),
        completer=DestinationList)

    final_destination = dest_path.joinpath(destination, full_name)
    print('Moving {0} to {1}'.format(srcfile, final_destination))
    os.rename(srcfile, final_destination)


from unittest.mock import Mock

import pytest

from libpythonprodaanrod.spam.main import EnviadorDeSpam
from libpythonprodaanrod.spam.modelos import Usuario

@pytest.mark.parametrize(
    'usuarios',
    [
        [
            Usuario(nome='Danilo', email='daanrod93@gmail.com'),
            Usuario(nome='Joyce', email='daanrod93@gmail.com')
        ],
        [
            Usuario(nome='Danilo', email='daanrod93@gmail.com')
        ]
    ]
)
def test_qde_de_spam(sessao, usuarios):
    for usuario in usuarios:
        sessao.salvar(usuario)
    enviador = Mock()
    enviador_de_spam = EnviadorDeSpam(sessao, enviador)
    enviador_de_spam.enviar_emails(
        'daanrod93@gmail.com',
        'Vambora!',
        'Conseguimos primo'
    )
    assert len(usuarios) == enviador.enviar.call_count

def test_parametros_de_spam(sessao):
    usuario = Usuario(nome='Danilo', email='daanrod93@gmail.com')
    sessao.salvar(usuario)
    enviador = Mock()
    enviador_de_spam = EnviadorDeSpam(sessao, enviador)
    enviador_de_spam.enviar_emails(
        'sharp.aedes@gmail.com',
        'Vambora!',
        'Conseguimos primo'
    )
    enviador.enviar.assert_called_once_with(
        'sharp.aedes@gmail.com',
        'daanrod93@gmail.com',
        'Vambora!',
        'Conseguimos primo'
    )


import dash
import dash_core_components as dcc
import dash_html_components as html
import dataset as myds

KPI1 = myds.KPI1
KPI2 = myds.KPI2

KPI31 = myds.KPI31 # Product top 1 - name - 2019
KPI32 = myds.KPI32 # Product top 1 - units - 2019

KPI41 = myds.KPI41 # Product top 1 - name - 2019
KPI42 = myds.KPI42 # Product top 1 - units - 2019

# data graphOne - bar -
xg1 = myds.xg1 # return product names
yg1 = myds.yg1 # return product units

# data graphTwo
xg2 = myds.xg2 # return months 2019 jan to dec
yg2 = myds.yg2 # return

# data to line sales person
# data to card best sales person
nomeV = myds.nomeV
pmvStr = f'Produto mais vendido: {myds.pmv.index[0]} - {myds.pmv.QTDE[0]} units'
mfStr = f'Maior faturamento: US${myds.mf.TOTAL.values}. Ref.: {myds.mf.index[0]}/2019 '
# data to sales numbers
xg3 = myds.tvpmpv.index
yg3 = myds.tvpmpv.values

#         success     primary    info     warning     danger
cores = ['#00bc8c', '#28415b', '#217dbb','#c87f0a', '#dc2c1a']

app = dash.Dash(__name__, external_stylesheets=['https://bootswatch.com/3/darkly/bootstrap.css'])

app.layout = html.Div([
    # row 1 - linha 1
    html.Div([
        html.H1('DASHBOARD - VENDAS - 2019')
    ], className='col-xs-12'),

    # row 2 - linha 2 - creating filter
    html.Div([
        html.Nav([
            html.Div([
                html.P('Filtro: ', className='navbar-brand')
            ], className='navbar-header'),
            html.Div([
                html.Label('Produto'),
                html.Select([
                    html.Option('A', value='A'),
                    html.Option('B', value='B')
                ], className='form-control')
            ], className='col-xs-2'),
            html.Div([
                html.Label('Cat. Produto'),
                html.Select([
                    html.Option('A', value='A'),
                    html.Option('B', value='B', selected='selected')
                ], className='form-control')
            ], className='col-xs-2'),
            html.Div([
                html.Label('Vendedor'),
                html.Select([
                    html.Option('Ana', value='A'),
                    html.Option('Beatriz', value='B', selected='selected'),
                    html.Option('Carlos', value='C', selected='selected')
                ], className='form-control')
            ], className='col-xs-2'),
            html.Div([
                html.Br(),
                html.Button('Filtrar', type='submit', className='btn btn-default')
            ], className='col-xs-2'),

        ], className='nav navbar-default'),
    ], className='col-xs-12 well'),

    # row 3 - linha 3 - showing 4 kpis
    html.Div([
        # indicador 1 - kpi 1
        html.Div([
            html.Div([
                html.Div([
                    html.H4('Qtde. Vendida', className='panel-title text-center')
                ], className='panel-heading'),
                html.Div([
                    html.H1(KPI1, className='text-center')
                ])
            ], className='panel panel-warning')
        ], className='col-xs-3'),
        # indicador 2 - kpi 2
        html.Div([
            html.Div([
                html.Div([
                    html.H4('Total  Vendido', className='panel-title text-center')
                ], className='panel-heading'),
                html.Div([
                    html.H1(KPI2, className='text-center')
                ])
            ], className='panel panel-success')
        ], className='col-xs-3'),
        # indicador 3 - kpi 3
        html.Div([
            html.Div([
                html.Div([
                    html.H4('Melhor Produto', className='panel-title text-center')
                ], className='panel-heading'),
                html.Div([
                    html.H1(f' {KPI31} - {KPI32} units', className='text-center')
                ])
            ], className='panel panel-info')
        ], className='col-xs-3'),
        # indicador 4 - kpi 4
        html.Div([
            html.Div([
                html.Div([
                    html.H4('Produto em alerta', className='panel-title text-center')
                ], className='panel-heading'),
                html.Div([
                    html.H1(f'{KPI41} - {KPI42} units', className='text-center')
                ])
            ], className='panel panel-info')
        ], className='col-xs-3'),

    ], className='col-xs-12 well'),

    # row 4 - linha 4 - graphs top 5 and sales year 2019
    html.Div([
        # graph Bar
        html.Div([
            dcc.Graph(
                id='graphOne',
                figure={
                    'data': [{
                        'x': xg1, 'y': yg1, 'type': 'bar',
                        'marker': {'color': cores},
                        'name': 'Produtos'
                    }],
                    'layout': {
                        'title': 'Top 5 Products - 2019',
                        'showlegend': False,
                        'showgrid': False,
                        # background transparent
                        'plot_bgcolor': 'rgba(0,0,0,0)',
                        'paper_bgcolor': 'rgba(0,0,0,0)',
                        'font': {'color': 'white'}
                    }
                }
            )
        ], className='col-xs-4'),

        # graph Lines
        html.Div([
            dcc.Graph(
                id='graphTwo',
                figure={
                    'data': [{
                        'x': xg2, 'y': yg2, 'type': 'line',
                        'marker': {'color': cores[3]},
                        'name': 'Vendas'
                    }],
                    'layout': {
                        'title': 'Vendas - Faturamento 2019',
                        'showlegend': True,
                        'xaxis': {'showgrid': False},
                        'yaxis': {'showgrid': True},
                        # background transparent
                        'plot_bgcolor': 'rgba(0,0,0,0)',
                        'paper_bgcolor': 'rgba(0,0,0,0)',
                        'font': {'color': 'white'}
                    }
                }
            )
        ], className='col-xs-8'),

    ], className='col-xs-12 well'),

    # row 5 - linha 5 - best sales person
    html.Div([
        # card best salesperson - melhor vendedor
        html.Div([
            html.H4('DESTAQUE 2019', className='text-center'),
            html.Br(),
            html.Img(src='https://www.fiscalti.com.br/wp-content/uploads/2020/05/default-user-image-365x365.png',
                     width='70px', height='70px'),
            html.H2(f'Vendedor: {nomeV}'),
            html.H5(pmvStr),
            html.H5(mfStr)

        ], className='col-xs-4 jumbotron'),
        html.Div([
            html.Div([
                dcc.Graph(
                    id='graphThree',
                    figure={
                        'data': [{
                            'x': xg3, 'y': yg3, 'type': 'line',
                            'marker': {'color': cores[0]},
                            'name': 'Vendas'
                        }],
                        'layout': {
                            'title': f'Vendedor {nomeV} - Faturamento 2019 ',
                            'showlegend': True,
                            'xaxis': {'showgrid': False},
                            'yaxis': {'showgrid': True},
                            # background transparent
                            'plot_bgcolor': 'rgba(0,0,0,0)',
                            'paper_bgcolor': 'rgba(0,0,0,0)',
                            'font': {'color': 'white'}
                        }
                    }
                )
            ], className='col-xs-8'),
        ])
    ], className='col-xs-12 well'),

], className='col-xs-12')

# this code will be in my github (description this video)
# thankyou!!!!

if __name__ == '__main__':
    app.run_server(debug=True)


import itertools
import numpy as np
import math

class Permutation:
    def __init__(self,
                 time_series: np.ndarray,
                 embedding_order: int = 3,
                 time_delay: int = 1
                 ):
        self.x = time_series
        self.order = embedding_order
        self.delay = time_delay
        self.embed = self._gen_embed()
        self.all_perms = list(itertools.permutations(np.arange(0, embedding_order), embedding_order))
        self.frqs = self._gen_frq()

    def _gen_embed(self):

        if self.x.ndim == 1:
            # pass 1D array

            y = np.zeros((self.order, self.x.shape[-1] - (self.order - 1) * self.delay))
            for i in range(self.order):
                y[i] = self.x[(i * self.delay):(i * self.delay + y.shape[1])]
            return y.T

        else:

            y = []

            embed_signal_length = self.x.shape[-1] - (self.order - 1) * self.delay

            indices = [[(i * self.delay), (i * self.delay + embed_signal_length)] for i in range(self.order)]

            for i in range(self.order):
                # loop with the order
                temp = self.x[:, indices[i][0]: indices[i][1]].reshape(-1, embed_signal_length, 1)
                # slicing the signal with the indices of each order (vectorized operation)

                y.append(temp)
                # append the sliced signal to list

            y = np.concatenate(y, axis=-1)
            # print(np.argpartition(y, kth=3, axis=1))
            # y = np.apply_along_axis(np.argsort, 2, y )
            y = np.argpartition(y, kth=1, axis=2)
            return y

    def _gen_frq(self):
        num_per_permutation = np.zeros((self.x.shape[0], len(self.all_perms)))
        for i in range(self.x.shape[0]):
            for j, x_i in enumerate(self.embed[i, ...]):
                for k, perm_k in enumerate(self.all_perms):
                    if tuple(x_i) == perm_k:
                        num_per_permutation[i, k] += 1
        return num_per_permutation

    def gen_prob(self):
        return self.frqs / np.sum(self.frqs, axis=1)[0]

    def entropy(self, x: np.ndarray, base=2):
        """Returns x log_b x if x is positive, 0 if x == 0, and np.nan
        otherwise. This handles the case when the power spectrum density
        takes any zero value.
        """
        xlogx = np.zeros(x.shape)
        xlogx[x < 0] = np.nan
        valid = x > 0
        xlogx[valid] = x[valid] * np.log(x[valid]) / np.log(base)

        return -1 * xlogx.sum(axis=1) / np.log2(math.factorial(self.order))

# a = np.arange(1, 2100)
# np.random.shuffle(a)
# np.random.shuffle(a)
#
# b = np.arange(1, 2100)
# b = b[::-1]
# # np.random.shuffle(b)
# c = np.arange(1, 2100)
# np.random.shuffle(c)
# x = np.array([a, b[::-1],b, c, c])
#
# order = 3
# perm = Permutation(x, order, 1)
# print(x)
# print(perm.embed)
# print(perm.all_perms)
# print(perm._gen_frq())
# print(perm.gen_prob())
# print(perm.entropy(perm.gen_prob()))
#
# print(np.partition([4,3,2,1],0))
# print(np.partition([4,3,2,1],1))
# print(np.partition([4,3,2,1],2))
# print(np.partition([4,3,2,1],3))

# print(np.array([4,3,2,1])[np.argpartition([4,3,2,1], kth=order - 1)])

#
#
# order = 3
# perms = list(itertools.permutations(np.arange(0, order), order))
# for k, prm_i in enumerate(perms):
#     for j, prm_j in enumerate(perms):
#         if prm_i[0] == prm_j[0] == 0:
#             # for p in range(1,len(prm_j)):
#             if all(prm_j[i] <= prm_j[i + 1] for i in range(len(prm_j) - 1)):
#                 # if prm_i[-1] == order-1:
#                 fig, axs = plt.subplots(2)
#                 plt.figure(j + k + 1)
#                 axs[0].plot(prm_i, '.-', c='r')
#                 axs[1].plot(prm_j, '.-', c='b')
#
#         elif prm_i[0] == prm_j[0] == order - 1:
#             # for p in range(1,len(prm_j)):
#             if all(prm_i[i] >= prm_i[i + 1] for i in range(len(prm_j) - 1)):
#                 fig, axs = plt.subplots(2)
#                 plt.figure(j + k + 1000)
#                 axs[0].plot(prm_i, '.-', c='r')
#                 axs[1].plot(prm_j, '.-', c='b')
#
# plt.show()

def micro_channels(order=3):
    channels = []
    perms = list(itertools.permutations(np.arange(0, order), order))
    for k, prm_i in enumerate(perms):
        for j, prm_j in enumerate(perms):
            if prm_i[0] == prm_j[0] == 0:
                if all(prm_j[i] <= prm_j[i + 1] for i in range(len(prm_j) - 1)):
                    channels.append([prm_i, prm_j])
            elif prm_i[0] == prm_j[0] == order - 1:
                if all(prm_i[i] >= prm_i[i + 1] for i in range(len(prm_j) - 1)):
                    channels.append([prm_i, prm_j])
    return channels

import matplotlib.pyplot as plt

chans = micro_channels(4)
for i in range(1,7):
    print(micro_channels(i))


import torch
import sklearn
import numpy as np
from elc_metric.ELC import elc

# Metrics
def getPerformance(y_true, y_pred, calc_evt):
    # Return all relevant performance metrics
    # Do not consider elements with -1
    loc = y_true != -1
    y_pred = y_pred[loc]
    y_true = y_true[loc]

    Perf = dict()
    Perf['kappa'] = sklearn.metrics.cohen_kappa_score(y_true, y_pred, labels=[0,1,2])
    Perf['mpca'] = sklearn.metrics.accuracy_score(y_true, y_pred)
    Perf['conf'] = sklearn.metrics.confusion_matrix(y_true, y_pred, labels=[0,1,2])
    Perf['prec'] = sklearn.metrics.precision_score(y_true, y_pred, average=None, labels=[0,1,2])
    Perf['mpcp'] = sklearn.metrics.precision_score(y_true, y_pred, average='macro', labels=[0,1,2])
    Perf['recall'] = sklearn.metrics.recall_score(y_true, y_pred, average=None, labels=[0,1,2])
    Perf['mpcr'] = sklearn.metrics.recall_score(y_true, y_pred, average='macro', labels=[0,1,2])
    Perf['iou'] = sklearn.metrics.jaccard_score(y_true, y_pred, average='macro', labels=[0,1,2])
    Perf['iou_class'] = sklearn.metrics.jaccard_score(y_true, y_pred, average=None, labels=[0,1,2])

    if calc_evt:
        cmat_list = [elc(y_true[i, ...].squeeze(), y_pred[i, ...].squeeze(), 15)[2] for i in range(0, y_true.shape[0])]
        Perf['kappa_evt'] = np.mean([kappa_confusion(cmat) for cmat in cmat_list])
        Perf['kappa_class_evt'] = np.mean(np.stack([kappa_perClass(cmat) for cmat in cmat_list], axis=0), axis=0)
    else:
        Perf['kappa_evt'] = 0
        Perf['kappa_class_evt'] = np.array([0, 0, 0])

    return Perf

def kappa_perClass(cmat):
    '''
    Computes the kappa score per class for a given confusion matrix
    '''
    kappa_class = []
    C = cmat.shape[0]
    classes_present = np.arange(0, C)
    for k in range(0, C):
        x = classes_present[classes_present!=k]
        y = k
        a = cmat[k, k]
        b = np.sum(cmat[x, y])
        c = np.sum(cmat[y, x])
        d = np.sum(cmat[x, x])
        assert np.sum([a,b,c,d]) == np.sum(cmat), "Incorrect calculation"
        cmat_c = np.array([[a, c], [b, d]])
        kappa_class.append(kappa_confusion(cmat_c))
    return np.array(kappa_class)

def kappa_confusion(cmat):
    '''
    Computes the kappa score from a given confusion matrix
    '''
    w = np.eye(cmat.shape[0])
    n = np.sum(cmat) # Total elements
    x = cmat/n
    r = np.sum(x, axis=1) # Sum across rows
    s = np.sum(x, axis=0) # Sum across columns
    Ex = r.reshape(3,1)*s.reshape(1,3) # Expected proportion for random agreement
    #pom = np.sum(np.min(np.stack([r, s], axis=0), axis=0))
    po = np.sum(x.dot(w))
    pe = np.sum(Ex.dot(w))
    kappa = (po - pe)/(1 - pe)
    return kappa

# Loss functions
def loss_giw(ip, target, weight, ignore_index):
    loss1 = torch.mean(loss_ce(ip, target, weight, ignore_index))
    GD = GeneralizedDiceLoss(ignore_index=ignore_index).cuda().to(torch.float64)
    loss2 = GD(ip, target)
    loss = loss1.to(torch.float64) + loss2.to(torch.float64)
    return loss, loss1.detach().cpu().item(), loss2.detach().cpu()

def loss_giw_dual(ip, target, weight, ignore_index, task_2):
    # Note that is important to add an eps value.
    loss1 = torch.mean(loss_ce(ip[:,:3,:], target, weight, ignore_index))
    GD = GeneralizedDiceLoss(ignore_index=ignore_index).cuda().to(torch.float64)
    reg_loss = torch.nn.SmoothL1Loss().to(torch.float64)
    loss2 = GD(ip[:,:3,:], target)
    loss3 = reg_loss(ip[:,3:,:], task_2)
    loss = loss1.to(torch.float64) + loss2.to(torch.float64) + loss3.to(torch.float64)
    return loss, loss1.detach().cpu().item(), loss2.detach().cpu().item()

def loss_ce(ip, target, weight, ignore_index):
    # Computes the loss based on given input and target.
    # Input: batch, sequence, features
    # Target: batch, sequence, class
    '''
    cE = torch.nn.CrossEntropyLoss(ignore_index=-1, weight=torch.tensor([0.3, 0.9, 0.5]).cuda().to(torch.float64))
    loss_ce = cE(ip, target.to(torch.long))
    '''

    cE = torch.nn.CrossEntropyLoss(reduction='none',
                                   ignore_index=-1,
                                   weight=torch.tensor([0.3, 0.9, 0.5]).cuda().to(torch.float64))
    sampleWeight = weight/torch.sum(weight, dim=1, keepdim=True)
    loss_ce = sampleWeight*cE(ip, target.to(torch.long))
    loss_ce = torch.sum(loss_ce, dim=1)

    return loss_ce

class GeneralizedDiceLoss(torch.nn.Module):
    # Author: Rakshit Kothari
    # Input: (B, C, ...)
    # Target: (B, C, ...)
    def __init__(self, epsilon=1e-5, weight=None, softmax=True, reduction=True, ignore_index=-1):
        super(GeneralizedDiceLoss, self).__init__()
        self.epsilon = epsilon
        self.weight = []
        self.reduction = reduction
        self.ignore_index = ignore_index
        if softmax:
            self.norm = torch.nn.Softmax(dim=1)
        else:
            self.norm = torch.nn.Sigmoid()

    def forward(self, ip, target):
        mask = target.clone().ne_(self.ignore_index)
        mask = [mask for i in range(0, ip.shape[1])]
        mask = torch.stack(mask, dim=1)
        mask.requires_grad = False

        ip = ip*mask.to(torch.float64)

        # Rapid way to convert to one-hot. For future version, use functional
        Label = (np.arange(3) == target.cpu().numpy()[..., None]).astype(np.uint8)
        target = torch.from_numpy(np.rollaxis(Label, 2, start=1)).cuda().to(torch.float64)

        if not (ip.shape == target.shape):
            print('Shapes do not match')
            print('Input shape: {}'.format(ip.shape))
            print('Target shape: {}'.format(target.shape))
        ip = self.norm(ip)

        # Flatten for multidimensional data
        ip = torch.flatten(ip, start_dim=2, end_dim=-1).cuda().to(torch.float32)
        target = torch.flatten(target, start_dim=2, end_dim=-1).cuda().to(torch.float32)

        numerator = ip*target
        denominator = ip + target

        class_weights = 1./(torch.sum(target, dim=2)**2).clamp(min=self.epsilon)

        A = class_weights*torch.sum(numerator, dim=2)
        B = class_weights*torch.sum(denominator, dim=2)

        dice_metric = 2.*torch.sum(A, dim=1)/torch.sum(B, dim=1)
        if self.reduction:
            return torch.mean(1. - dice_metric.clamp(min=self.epsilon))
        else:
            return 1. - dice_metric.clamp(min=self.epsilon)


from django.db import models
from rest_framework import serializers
from django.contrib.sites.models import Site
from media.serializers import CustomImageFieldSerializer

from media.models import Photo, Video, Media

class VaultPhotoSerializer(serializers.ModelSerializer):
    """Serializer for photos in ChatVaultView"""

    media = CustomImageFieldSerializer(sizes="photo_upload")

    class Meta:
        model = Photo
        fields = ["public_id", "upload_time", "media"]

class VaultVideoThumbnailSerializer(serializers.FileField):
    """Return a dictionary of urls corresponding to video and its thumbnail"""

    read_only = True

    def to_representation(self, value):
        video_id = value.instance.public_id
        video_rel_url = f"chat/vault/video-stream/{video_id}/"
        domain = Site.objects.get_current().domain

        ret = {
            "video": f"http://{domain}/{video_rel_url}",
            "thumbnail": f"http://{domain}/{value.url_300x300}/",
        }
        return ret

class VaultVideoSerializer(serializers.ModelSerializer):
    """Serializer for videos in ChatVaultView"""

    media = VaultVideoThumbnailSerializer()

    class Meta:
        model = Video
        fields = ["public_id", "upload_time", "media"]

class VaultListSerializer(serializers.ListSerializer):
    """Custom ListSerializer to implement custom to_representation for each media item in vault"""

    def to_representation(self, data):
        iterable = data.all() if isinstance(data, models.Manager) else data

        to_rep = []
        for item in iterable:
            to_rep += [self.get_to_rep(item)]
        return to_rep

    def get_to_rep(self, instance):
        model = instance._meta.model

        if model == Media:
            instance = instance.media_item
            model = instance._meta.model

        self.child.Meta.model = model
        instance.type = model.__name__.lower()

        def get_serializer():
            if model == Photo:
                return VaultPhotoSerializer
            else:
                return VaultVideoSerializer

        serializer = get_serializer()

        ret = serializer(instance).to_representation(instance)

        ret["type"] = instance.type

        return ret

class VaultSerializer(serializers.ModelSerializer):
    """Common serializer for all media items in vault"""

    class Meta:
        list_serializer_class = VaultListSerializer
        model = None
        fields = "__all__"


from rest_framework import serializers
from ghostpost_app.models import BoastsRoastsModel

# Worked with Sohail and Albina too on the planning/beginning stage before finishing separately
class BoastRoastSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = BoastsRoastsModel
        fields = [
            'id',
            'is_boast',
            'post_content',
            'upvotes',
            'downvotes',
            'post_datetime',
            'privatesecret_key',
            'vote_score'
        ]


import os

import pygame
from pygame.locals import *

def load_image(file, transparent=True):
    # print("Loading " + file + " ..")
    fullname = os.path.join("../media", file)
    image = pygame.image.load(fullname)
    if transparent == True:
        image = image.convert()
        colorkey = image.get_at((0, 0))
        image.set_colorkey(colorkey, RLEACCEL)
    else:
        image = image.convert_alpha()
    return image


import json
import random

def progress(str, i, end):
    barLen = 30
    print(
        f'{str} {i}/{end}\t[{"="*(i*barLen//end)}>{" "*(barLen-i*barLen//end)}]', end='\r')

print('Getting data...')
print('Loading json...')

with open('moves.json', 'r') as f:
    json_moves = json.load(f)

print('Creating conv_tables...')

board_size = 8

moves_conv_tbl = {(fromx, fromy, tox, toy): (fromx * board_size**3 + fromy *
                  board_size**2 + tox * board_size + toy) for fromx in range(
    board_size) for fromy in range(board_size) for tox in range(board_size) for toy in range(board_size)}

pieces_types_num = 6

def get_pieces_conv_tbl_list(i):
    conv_tbl_list = [0] * (pieces_types_num * 2 + 1)
    conv_tbl_list[i] = 1
    return conv_tbl_list

pieces_conv_tbl = [get_pieces_conv_tbl_list(
    i) for i in range(pieces_types_num * 2 + 1)]

def get_boards_moves(winner_index, moves):
    _boards = []
    _moves = []
    board = [
        [8, 9, 10, 11, 12, 10, 9, 8],
        [7, 7, 7, 7, 7, 7, 7, 7],
        [0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0],
        [0, 0, 0, 0, 0, 0, 0, 0],
        [1, 1, 1, 1, 1, 1, 1, 1],
        [2, 3, 4, 5, 6, 4, 3, 2]
    ]
    for row in board:
        for i, cell in enumerate(row):
            row[i] = pieces_conv_tbl[cell]
    for i, m in enumerate(moves):
        if i % 2 == winner_index:
            _boards.append(board)
            _moves.append(moves_conv_tbl[tuple(m)])
        board[m[2]][m[3]] = board[m[0]][m[1]]
        board[m[0]][m[1]] = pieces_conv_tbl[0]
    return _boards, _moves

def get_train_data(train_len=None, test_len=None):
    temp = []
    for i, round in enumerate(json_moves):
        progress('Creating data', i, len(json_moves))
        winner = round[0]
        moves = round[1]
        if winner == '1-0' or winner == '1/2-1/2':
            temp.append(get_boards_moves(0, moves))
        if winner == '0-1' or winner == '1/2-1/2':
            temp.append(get_boards_moves(1, moves))

    print('\nSampling data...')
    if (train_len is not None) and (test_len is not None):
        train = random.sample(temp, train_len)
        test = random.sample(temp, test_len)
    else:
        train = temp
        test = random.sample(temp, len(temp) / 10)
    print('Converting data...')
    train_boards = []
    train_moves = []
    for b, m in train:
        train_boards.extend(b)
        train_moves.extend(m)
    test_boards = []
    test_moves = []
    for b, m in test:
        test_boards.extend(b)
        test_moves.extend(m)

    return train_boards, train_moves, test_boards, test_moves