text:
import collections
import math
import unittest

import numpy as np
from srl.rl.memories.proportional_memory import ProportionalMemory
from srl.rl.memories.rankbase_memory import RankBaseMemory
from srl.rl.memories.replay_memory import ReplayMemory

class TestMemory(unittest.TestCase):
    def test_memory(self):
        capacity = 10

        memories = [
            (ReplayMemory(capacity), False),
            (ProportionalMemory(capacity, 0.8, 1, 10), False),
            (RankBaseMemory(capacity, 0.8, 1, 10), False),
        ]
        for memory, use_priority in memories:
            with self.subTest(memory.__class__.__name__):
                self._test_memory(memory, use_priority)

    def _test_memory(self, memory, use_priority):
        capacity = memory.capacity
        assert capacity == 10

        # add
        for i in range(100):
            memory.add((i, i, i, i), 0)
        assert len(memory) == capacity

        # 中身を1～10にする
        for i in range(10):
            i += 1
            memory.add((i, i, i, i), i)
        assert len(memory) == capacity

        # --- 複数回やって比率をだす
        counter = []
        for i in range(10000):
            (indices, batchs, weights) = memory.sample(5, 1)
            assert len(indices) == 5
            assert len(batchs) == 5
            assert len(weights) == 5

            # 重複がないこと
            assert len(list(set(batchs))) == 5, list(set(batchs))

            # batchの中身をカウント
            for batch in batchs:
                counter.append(batch[0])

            # update priority
            memory.update(indices, batchs, [b[3] for b in batchs])
            assert len(memory) == capacity

            # save/load
            d = memory.backup()
            memory.restore(d)
            d2 = memory.backup()
            assert d == d2

        counter = collections.Counter(counter)

        # keyは1～10まであること
        keys = sorted(counter.keys())
        assert keys == [i + 1 for i in range(capacity)]

        if use_priority:
            # priorityが高いほど数が増えている
            vals = [counter[key] for key in keys]
            for i in range(capacity - 1):
                assert vals[i] < vals[i + 1]

    # -------------------------------
    # test IS
    # -------------------------------
    def test_IS_Proportional(self):
        capacity = 10
        for alpha in [0, 0.5, 0.8, 1.0]:
            with self.subTest(alpha=alpha):
                config = {
                    "capacity": capacity,
                    "alpha": alpha,
                    "beta_initial": 1,
                }
                memory = ProportionalMemory(**config)
                priorities = [
                    1**alpha,
                    2**alpha,
                    3**alpha,
                ]
                sum_priority = sum(priorities)
                probs = [
                    priorities[0] / sum_priority,
                    priorities[1] / sum_priority,
                    priorities[2] / sum_priority,
                ]
                self._test_IS(memory, priorities, probs)

    def test_IS_RankBase(self):
        capacity = 10
        for alpha in [0, 0.5, 0.8, 1.0]:
            with self.subTest(alpha=alpha):
                config = {
                    "capacity": capacity,
                    "alpha": alpha,
                    "beta_initial": 1,
                }
                memory = RankBaseMemory(**config)
                priorities = [
                    1 + 0 * alpha,  # 3位
                    1 + 1 * alpha,  # 2位
                    1 + 2 * alpha,  # 1位
                ]
                sum_priority = sum(priorities)
                probs = [
                    priorities[0] / sum_priority,
                    priorities[1] / sum_priority,
                    priorities[2] / sum_priority,
                ]
                self._test_IS(memory, priorities, probs)

    def _test_IS(self, memory, priorities, probs):
        for i in range(len(priorities)):
            memory.add((i, i, i, i + 1), i + 1)  # 最後がpriority

        N = len(priorities)
        true_weights = np.array(
            [
                (N * probs[0]) ** (-1),
                (N * probs[1]) ** (-1),
                (N * probs[2]) ** (-1),
            ]
        )
        maxw = np.max(true_weights)
        true_weights /= maxw

        (indices, batchs, weights) = memory.sample(3, 1)

        for i, b in enumerate(batchs):
            idx = b[0]
            self.assertTrue(
                math.isclose(weights[i], true_weights[idx], rel_tol=1e-7),
                f"{weights[i]} != {true_weights[idx]}",
            )

if __name__ == "__main__":
    unittest.main(module=__name__, defaultTest="TestMemory.test_memory", verbosity=2)


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from keras.layers import Input, Lambda, Subtract
from keras.models import Model, model_from_json
from keras.optimizers import SGD, Adam
from keras import backend as KBack
import time
import numpy as np

from dynamicgem.embedding.dynamic_graph_embedding import DynamicGraphEmbedding
from dynamicgem.utils import graph_util, plot_util
from dynamicgem.visualization import plot_dynamic_sbm_embedding
from dynamicgem.utils.sdne_utils import *

MAX_CHUNK_SIZE = 50000

class SDNE(DynamicGraphEmbedding):
    """Structural Deep Network Embedding

    SDNE perform the network embedding
    while utilizing Structural Deep Network Embedding (SDNE).

    Args:
        d (int): dimension of the embedding
        beta (float): penalty parameter in matrix B of 2nd order objective
        n_prev_graphs (int): Lookback (number of previous graphs to be considered) for the dynamic graph embedding
        nu1 (float): L1-reg hyperparameter
        nu2 (float): L2-reg hyperparameter
        K (float): number of hidden layers in encoder/decoder
        rho (float): bounding ratio for number of units in consecutive layers (< 1)
        n_aeunits  (list) = List of embedding dimension for auto encoder layers
        n_lstmunits= List of embedding dimension for lstm layers
        n_iter (int): number of sgd iterations for first embedding (const)
        xeta (float): sgd step size parameter
        n_batch (int): minibatch size for SGD
        modelfile (str): Files containing previous encoder and decoder models
        weightfile (str): Files containing previous encoder and decoder weights
        node_frac (float): Fraction of nodes to use for random walk
        n_walks_per_node (int): Number of random walks to do for each selected nodes
        len_rw (int): Length of every random walk

    Examples:
        >>> from dynamicgem.embedding.sdne_dynamic import SDNE
        >>> from dynamicgem.graph_generation import dynamic_SBM_graph
        >>> node_num = 100
        >>> community_num = 2
        >>> node_change_num = 2
        >>> length = 2
        >>> dynamic_sbm_series = dynamic_SBM_graph.get_community_diminish_series(node_num, community_num, length, 1, node_change_num)
        >>> dynamic_embedding = SDNE(d=16, beta=5, alpha=1e-5, nu1=1e-6, nu2=1e-6, K=3, n_units=[500, 300,], rho=0.3, n_iter=2, n_iter_subs=5, xeta=0.01, n_batch=500, modelfile=['./intermediate/enc_model.json', './intermediate/dec_model.json'], weightfile=['./intermediate/enc_weights.hdf5', './intermediate/dec_weights.hdf5'], node_frac=1, n_walks_per_node=10, len_rw=2)
        >>> dynamic_embedding.learn_embeddings([g[0] for g in dynamic_sbm_series], False, subsample=False)
        >>> plot_dynamic_sbm_embedding.plot_dynamic_sbm_embedding(dynamic_embedding.get_embeddings(), dynamic_sbm_series)
        >>> plt.savefig('result/visualization_sdne_cd.png')
        >>> plt.show()
    """
    def __init__(self, d, beta, alpha, nu1, nu2, K, n_units, rho, n_iter, n_iter_subs, xeta, n_batch, modelfile=None,
                 weightfile=None, node_frac=1, n_walks_per_node=5, len_rw=2):
        ''' Initialize the SDNE class'''
        super().__init__(d)
        self._method_name = 'sdne' # embedding method name
        self._d = d
        self._Y = None # embedding
        self._Ys = None # embeddings
        self._beta = beta
        self._alpha = alpha
        self._nu1 = nu1
        self._nu2 = nu2
        self._K = K
        self._n_units = n_units
        self._actfn = 'relu' # We use relu instead of sigmoid from the paper, to avoid vanishing gradients and allow correct layer deepening
        self._rho = rho
        self._n_iter = n_iter
        self._n_iter_subs = n_iter_subs
        self._xeta = xeta
        self._n_batch = n_batch
        self._modelfile = modelfile
        self._weightfile = weightfile
        self._node_frac = node_frac
        self._n_walks_per_node = n_walks_per_node
        self._len_rw = len_rw
        self._num_iter = n_iter # max number of iterations during sgd (variable)
        # self._node_num is number of nodes: initialized later in learn_embedding()
        # self._encoder is the vertex->embedding model
        # self._decoder is the embedding->vertex model
        # self._autocoder is the vertex->(vertex,embedding) model
        # self._model is the SDNE model to be trained (uses self._autoencoder internally)

    def get_method_name(self):
        """Function to return the method name.

           Returns:
                String: Name of the method.
        """
        return self._method_name

    def get_method_summary(self):
        """Function to return the summary of the algorithm.

           Returns:
                String: Method summary
        """
        return '%s_%d' % (self._method_name, self._d)

    def learn_embedding(self, graph, prevStepInfo=True, loadfilesuffix=None, savefilesuffix=None, subsample=False):
        """Learns the embedding of the nodes.

           Attributes:
               graph (Object): Networkx Graph Object
               prevStepInfo (bool): Incorporate previous step info
               loadfilesuffix (str): file suffix for loading the previos data
               savefilesuffix (str): file suffix to be used for saving the data

            Returns:
                List: Node embeddings and time taken by the algorithm
        """
        if subsample:
            S = graph_util.randwalk_DiGraph_to_adj(graph, node_frac=self._node_frac, n_walks_per_node=self._n_walks_per_node, len_rw=self._len_rw)
        else:
            S = graph_util.transform_DiGraph_to_adj(graph)
        if not np.allclose(S.T, S):
            print("SDNE only works for symmetric graphs! Making the graph symmetric")
        S = (S + S.T)/2					# enforce S is symmetric
        S -= np.diag(np.diag(S))		# enforce diagonal = 0
        self._node_num = S.shape[0]
        n_edges = np.count_nonzero(S)	# Double counting symmetric edges deliberately to maintain autoencoder symmetry
        # Create matrix B
        B = np.ones(S.shape)
        B[S != 0] = self._beta

        # compute degree of each node
        deg = np.sum(S!=0, 1)

        # Generate encoder, decoder and autoencoder
        if(prevStepInfo):
            self._num_iter = self._n_iter_subs
            # Load previous encoder, decoder and weights
            if loadfilesuffix is None:
                encoder_prev = loadmodel(self._modelfile[0])
                decoder_prev = loadmodel(self._modelfile[1])
                loadweights(encoder_prev, self._weightfile[0])
                loadweights(decoder_prev, self._weightfile[1])
            else:
                encoder_prev = loadmodel('encoder_model_'+loadfilesuffix+'.json')
                decoder_prev = loadmodel('decoder_model_'+loadfilesuffix+'.json')
                loadweights(encoder_prev, 'encoder_weights_'+loadfilesuffix+'.hdf5')
                loadweights(decoder_prev, 'decoder_weights_'+loadfilesuffix+'.hdf5')
            # Size consistency is being assumed for encoder and decoder, and won't be checked for!

            # Check graph for size changes from previous step
            prev_node_num = encoder_prev.layers[0].input_shape[1]
            # Note: We only assume node addition. No nodes should be deleted from the graph at any time step.
            if(self._node_num > prev_node_num):
                n_units_prev = [encoder_prev.layers[i].output_shape[1] for i in range(len(encoder_prev.layers))]
                # Get new number of units in each layer
                n_units = get_new_model_shape(n_units_prev, self._node_num, self._rho, deepen=False)
                self._K = len(n_units) - 1
                self._n_units = n_units[1:-1]
                self._encoder = get_encoder(self._node_num, n_units[-1], self._K, self._n_units, self._nu1, self._nu2, self._actfn)
                self._decoder = get_decoder(self._node_num, n_units[-1], self._K, self._n_units, self._nu1, self._nu2, self._actfn)
                self._autoencoder = get_autoencoder(self._encoder, self._decoder)
                # First do widening for encoder layers
                [w1, b1] = [None, None]
                for i in range(0, len(n_units_prev) - 1):
                    if(n_units[i] > n_units_prev[i]):
                        if i == 0:
                            w2, b2 = encoder_prev.layers[i+1].get_weights()
                            final_w1, final_b1, w1 = widernet(None, None, w2, n_units[i], n_units_prev[i])
                            b1 = b2
                        else:
                            w2, b2 = encoder_prev.layers[i+1].get_weights()
                            final_w1, final_b1, w1 = widernet(w1, b1, w2, n_units[i], n_units_prev[i])
                            b1 = b2
                            self._encoder.layers[i].set_weights([final_w1, final_b1])
                    else:
                        if i > 0:
                            self._encoder.layers[i].set_weights([w1, b1])
                        [w1, b1] = encoder_prev.layers[i+1].get_weights()
                # Next do deepening for encoder layers
                last_layer = len(n_units_prev) - 2
                extra_depth = len(n_units) - len(n_units_prev)
                [temp_w, temp_b] = [w1, b1]
                while(extra_depth > 0):
                    w1, temp_w, b1, temp_b = deepernet(n_units[last_layer], n_units[last_layer+1], n_units_prev[-1], temp_w, temp_b)
                    self._encoder.layers[last_layer+1].set_weights([w1, b1])
                    last_layer += 1
                    extra_depth -= 1
                self._encoder.layers[last_layer+1].set_weights([temp_w, temp_b])
                # Next do deepening for decoder layers
                n_units.reverse()
                n_units_prev.reverse()
                last_layer = 0
                extra_depth = len(n_units) - len(n_units_prev)
                [temp_w, temp_b] = decoder_prev.layers[last_layer+1].get_weights()
                while(extra_depth > 0):
                    w1, temp_w, b1, temp_b = deepernet(n_units[last_layer], n_units[last_layer+1], n_units_prev[1], temp_w, temp_b)
                    self._decoder.layers[last_layer+1].set_weights([w1, b1])
                    last_layer += 1
                    extra_depth -= 1
                # Next do widening for decoder layers
                extra_depth = len(n_units) - len(n_units_prev)
                [w1, b1] = [temp_w, temp_b]
                for i in range(last_layer + 1, len(n_units) - 1):
                    if(n_units[i] > n_units_prev[i - extra_depth]):
                        w2, b2 = decoder_prev.layers[i+1 - extra_depth].get_weights()
                        final_w1, final_b1, w1 = widernet(w1, b1, w2, n_units[i], n_units_prev[i - extra_depth])
                        b1 = b2
                        self._decoder.layers[i].set_weights([final_w1, final_b1])
                    else:
                        self._decoder.layers[i].set_weights([w1, b1])
                        w1, b1 = decoder_prev.layers[i+1 - extra_depth].get_weights()
                if(n_units[len(n_units) - 1] > n_units_prev[len(n_units) - 1 - extra_depth]):
                    final_w1, final_b1, w1 = widernet(w1, b1, None, n_units[len(n_units) - 1], n_units_prev[len(n_units) - 1 - extra_depth])
                    self._decoder.layers[len(n_units) - 1].set_weights([final_w1, final_b1])
                else:
                    self._decoder.layers[len(n_units) - 1].set_weights([w1, b1])
            else:
                # If no new nodes, then just initialize with previous models
                self._encoder = encoder_prev
                self._decoder = decoder_prev
                self._autoencoder = get_autoencoder(self._encoder, self._decoder)
        else:
            self._num_iter = self._n_iter
            # If cannot use previous step information, initialize new models
            self._encoder = get_encoder(self._node_num, self._d,  self._n_units, self._nu1, self._nu2, self._actfn)
            self._decoder = get_decoder(self._node_num, self._d, self._n_units, self._nu1, self._nu2, self._actfn)
            self._autoencoder = get_autoencoder(self._encoder, self._decoder)

        # Initialize self._model
        # Input
        x_in = Input(shape=(2*self._node_num,), name='x_in')
        x1 = Lambda(lambda x: x[:,0:self._node_num], output_shape=(self._node_num,))(x_in)
        x2 = Lambda(lambda x: x[:,self._node_num:2*self._node_num], output_shape=(self._node_num,))(x_in)
        # Process inputs
        [x_hat1, y1] = self._autoencoder(x1)
        [x_hat2, y2] = self._autoencoder(x2)
        # Outputs
        x_diff1 = Subtract()([x_hat1, x1])
        x_diff2 = Subtract()([x_hat2, x2])
        y_diff = Subtract()([y2, y1])

        # Objectives
        def weighted_mse_x(y_true, y_pred):
            ''' Hack: This fn doesn't accept additional arguments. We use y_true to pass them.
                y_pred: Contains x_hat - x
                y_true: Contains [b, deg]
            '''
            return KBack.sum(KBack.square(y_pred * y_true[:,0:self._node_num]), axis=-1)/y_true[:,self._node_num]
        def weighted_mse_y(y_true, y_pred):
            ''' Hack: This fn doesn't accept additional arguments. We use y_true to pass them.
                y_pred: Contains y2 - y1
                y_true: Contains s12
            '''
            min_batch_size = KBack.shape(y_true)[0]
            return KBack.reshape(
                KBack.sum(KBack.square(y_pred), axis=-1),
                [min_batch_size, 1]
            ) * y_true

        # Model
        self._model = Model(input=x_in, output=[x_diff1, x_diff2, y_diff])
        sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True)
        # adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
        self._model.compile(optimizer=sgd, loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y], loss_weights=[1, 1, self._alpha])
        # self._model.compile(optimizer=adam, loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y], loss_weights=[1, 1, self._alpha])

        # Structure data in the correct format for the SDNE model
        # InData format: [x1, x2]
        # OutData format: [b1, b2, s12, deg1, deg2]
        data_chunk_size = MAX_CHUNK_SIZE
        InData = np.zeros((data_chunk_size, 2*self._node_num))
        OutData = np.zeros((data_chunk_size, 2*self._node_num + 3))
        for epoch_num in range(self._num_iter):
            print('EPOCH %d/%d' % (epoch_num, self._num_iter))
            e = 0
            k = 0
            for i in range(self._node_num):
                for j in range(self._node_num):
                    if(S[i,j] != 0):
                        temp = np.append(S[i,:], S[j,:])
                        InData[k,:] = temp
                        temp = np.append(np.append(np.append(np.append(B[i,:], B[j,:]), S[i,j]), deg[i]), deg[j])
                        OutData[k,:] = temp
                        e += 1
                        k += 1
                        if k == data_chunk_size:
                            self._model.fit(InData, [ np.append(OutData[:,0:self._node_num], np.reshape(OutData[:,2*self._node_num+1], [data_chunk_size, 1]), axis=1),
                                np.append(OutData[:,self._node_num:2*self._node_num], np.reshape(OutData[:,2*self._node_num+2], [data_chunk_size, 1]), axis=1),
                                OutData[:,2*self._node_num] ], nb_epoch=1, batch_size=self._n_batch, shuffle=True, verbose=1)
                            k = 0
            if k > 0:
                self._model.fit(InData[:k, :], [ np.append(OutData[:k,0:self._node_num], np.reshape(OutData[:k,2*self._node_num+1], [k, 1]), axis=1),
                    np.append(OutData[:k,self._node_num:2*self._node_num], np.reshape(OutData[:k,2*self._node_num+2], [k, 1]), axis=1),
                    OutData[:k,2*self._node_num] ], nb_epoch=1, batch_size=self._n_batch, shuffle=True, verbose=1)

        # Get embedding for all points
        _, self._Y = self._autoencoder.predict(S, batch_size=self._n_batch)

        # Save the autoencoder and its weights
        if(self._weightfile is not None):
            saveweights(self._encoder, self._weightfile[0])
            saveweights(self._decoder, self._weightfile[1])
        if(self._modelfile is not None):
            savemodel(self._encoder, self._modelfile[0])
            savemodel(self._decoder, self._modelfile[1])
        if(savefilesuffix is not None):
            saveweights(self._encoder, 'encoder_weights_'+savefilesuffix+'.hdf5')
            saveweights(self._decoder, 'decoder_weights_'+savefilesuffix+'.hdf5')
            savemodel(self._encoder, 'encoder_model_'+savefilesuffix+'.json')
            savemodel(self._decoder, 'decoder_model_'+savefilesuffix+'.json')
            # Save the embedding
            np.savetxt('embedding_'+savefilesuffix+'.txt', self._Y)

        return self._Y

    def learn_embeddings(self, graphs, prevStepInfo = False, loadsuffixinfo=None, savesuffixinfo=None, subsample=False):
        """Learns the embedding of the nodes.

           Attributes:
               graph (Object): Networkx Graph Object
               prevStepInfo (bool): Incorporate previous step info
               loadfilesuffix (str): file suffix for loading the previos data
               savefilesuffix (str): file suffix to be used for saving the data

            Returns:
                List: Node embeddings and time taken by the algorithm
        """
        if loadsuffixinfo is None:
            if savesuffixinfo is None:
                Y = self.learn_embedding(graphs[0], prevStepInfo, subsample=subsample)
            else:
                Y = self.learn_embedding(graphs[0], prevStepInfo, savefilesuffix=savesuffixinfo+'_'+str(1), subsample=subsample)
        else:
            if savesuffixinfo is None:
                Y = self.learn_embedding(graphs[0], prevStepInfo, loadfilesuffix=loadsuffixinfo+'_'+str(1), subsample=subsample)
            else:
                Y = self.learn_embedding(graphs[0], prevStepInfo, loadfilesuffix=loadsuffixinfo+'_'+str(1), savefilesuffix=savesuffixinfo+'_'+str(1), subsample=subsample)
        self._Ys = [np.copy(Y)]
        for i in range(1, len(graphs)):
            print('Processing graph {0}:'.format(i+1))
            if loadsuffixinfo is None:
                if savesuffixinfo is None:
                    Y = self.learn_embedding(graphs[i], True, subsample=subsample)
                else:
                    Y = self.learn_embedding(graphs[i], True, savefilesuffix=savesuffixinfo+'_'+str(i+1), subsample=subsample)
            else:
                if savesuffixinfo is None:
                    Y = self.learn_embedding(graphs[i], True, loadfilesuffix=loadsuffixinfo+'_'+str(i+1), subsample=subsample)
                else:
                    Y = self.learn_embedding(graphs[i], True, loadfilesuffix=loadsuffixinfo+'_'+str(i+1), savefilesuffix=savesuffixinfo+'_'+str(i+1), subsample=subsample)
            self._Ys.append(np.copy(Y))
        return self._Ys

    def get_embedding(self, filesuffix):
        """Function to return sinfle embedding"""
        return self._Y if filesuffix is None else np.loadtxt('embedding_'+filesuffix+'.txt')

    def get_embeddings(self):
        """Function to return all the  embeddings"""
        return self._Ys

    def get_edge_weight(self, i, j, embed=None, filesuffix=None):
        """Function to get edge weight.

            Attributes:
              i (int): source node for the edge.
              j (int): target node for the edge.
              embed (Matrix): Embedding values of all the nodes.
              filesuffix (str): File suffix to be used to load the embedding.

            Returns:
                Float: Weight of the given edge.
        """
        if embed is None:
            if filesuffix is None:
                embed = self._Y
            else:
                embed = np.loadtxt('embedding_'+filesuffix+'.txt')
        if i == j:
            return 0
        else:
            S_hat = self.get_reconst_from_embed(embed[(i,j),:], filesuffix)
            return (S_hat[i,j] + S_hat[j,i])/2

    def get_reconstructed_adj(self, embed=None, filesuffix=None):
        """Function to reconstruct the adjacency list for the given node.

            Attributes:
              node_l (int): node for which the adjacency list will be created.
              embed (Matrix): Embedding values of all the nodes.
              filesuffix (str): File suffix to be used to load the embedding.

            Returns:
                List : Adjacency list of the given node.
        """
        if embed is None:
            if filesuffix is None:
                embed = self._Y
            else:
                embed = np.loadtxt('embedding_'+filesuffix+'.txt')
        S_hat = self.get_reconst_from_embed(embed, filesuffix)
        return graphify(S_hat)

    def get_reconst_from_embed(self, embed, filesuffix=None):
        """Function to reconstruct the graph from the embedding.

            Attributes:
              node_l (int): node for which the adjacency list will be created.
              embed (Matrix): Embedding values of all the nodes.
              filesuffix (str): File suffix to be used to load the embedding.

            Returns:
                List: REconstructed graph for the given nodes.
        """
        if filesuffix is None:
            return self._decoder.predict(embed, batch_size=self._n_batch)
        else:
            try:
                decoder = model_from_json(open('decoder_model_'+filesuffix+'.json').read())
            except:
                print('Error reading file: {0}. Cannot load previous model'.format('decoder_model_'+filesuffix+'.json'))
                exit()
            try:
                decoder.load_weights('decoder_weights_'+filesuffix+'.hdf5')
            except:
                print('Error reading file: {0}. Cannot load previous weights'.format('decoder_weights_'+filesuffix+'.hdf5'))
                exit()
            return decoder.predict(embed, batch_size=self._n_batch)

ext: py

max_stars_repo_name: Sujit-O/dyngem

max_issues_repo_name: Sujit-O/dyngem

max_forks_repo_name: Sujit-O/dyngem

reason: import numpy


text:
#!/usr/bin/env python
"""
    Run dynamic programming on similarities object from `segment_job.py`
"""
import torch
import numpy as np

def nan_masking(tensor):
    mask = (1 - nan_mask(tensor))
    print(tensor.size())
    print(mask.size())
    return tensor[mask]

def nan_mask(tensor):
    return (tensor != tensor)

def replace_nan(tensor, value=0):
    mask = nan_mask(tensor)
    tensor[mask] = value
    return tensor

def ranged_reduce(reduce_op, tensor, range_y, range_x,
                  empty_val=0.0, masking_fct=lambda x: x):
    tensor_slice = tensor[range_y, range_x]
    tensor_masked = masking_fct(tensor_slice)

    if sum(tensor_masked.size()) == 0:
        return torch.tensor([empty_val])

    return reduce_op(tensor_masked)

def reduce_scores(ctm_slices, docx_slices, scores, reduce_op,
                  empty_val=0.0, masking_fct=lambda x: x):
    all_scores = []
    for i, ctm_slice in enumerate(ctm_slices):
        print("%d: " % i, str(ctm_slice))
        ctm_scores = []
        for j, docx_slice in enumerate(docx_slices):
            res = ranged_reduce(reduce_op, scores, ctm_slice,
                                docx_slice, masking_fct=masking_fct)
            score = float(res)
            print("\t%f" % score, "%d: " % j, str(docx_slice))
            ctm_scores += [score]
        all_scores += [ctm_scores]

    return torch.Tensor(all_scores)

def avg_scores(ctm_slices, docx_slices, scores):
    return reduce_scores(ctm_slices, docx_slices, scores, op=tensor_mean)

def dp(scores, score_scale_fct=lambda x: x,
       score_pre_scale_fct=lambda x: x, DIAGDP=False, dhw=0.9999, dgw=0.9999):
    # softmax = torch.nn.functional.softmax
    scores = score_pre_scale_fct(scores)

    # soft-normalization over CTM has some (but not much) effect
    # scores = softmax(scores, 1)

    # this soft-normalize over docx sentences which makes the alignment
    # less noise resistant i.e. docx side noise like participant list
    # will then be aligned with content.
    # scores = softmax(scores, 0)

    # haut weights (& delta)
    hw = 1
    # dhw = 1
    # dhw = 0.9999
    # # dhw = 1

    # # gauche weight (& delta)
    # # dgw = 0.9999
    # dgw = 0.9999
    # dgw = 1
    gw = 1

    # diag weight (& delta)
    diagw = 0.8

    # DIAGDP = False

    y, x = scores.size()
    t = torch.zeros([y + 1, x + 1])
    hist = torch.zeros([y + 1, x + 1])
    W = torch.ones([y + 1, x + 1])
    for j in range(1, x + 1):
        for i in range(1, y + 1):
            # if DIAGDP:
            #     s = None
            #     diag = t[i - 1, j - 1] + scores[i - 1, j - 1] * diagw
            #     g = t[i, j - 1] * W[i, j - 1]
            #     h = t[i - 1, j] * W[i - 1, j]

            #     if diag > g and diag > h:
            #         hist[i, j] = 2
            #         W[i, j] = 1
            #         s = diag
            #     elif g < h and hist[i - 1, j] != -1:
            #         hist[i, j] = 1
            #         if hist[i, j] == hist[i - 1, j]:
            #             W[i, j] = W[i - 1, j] * dhw
            #         else:
            #             W[i, j] = 1
            #         s = h
            #     elif g > h and hist[i, j - 1] != 1:
            #         hist[i, j] = -1
            #         if hist[i, j] == hist[i, j - 1]:
            #             W[i, j] = W[i, j - 1] * dgw
            #         else:
            #             W[i, j] = 1
            #         s = g
            #     else:
            #         hist[i, j] = 2
            #         W[i, j] = 1
            #         s = diag
            #     t[i, j] = (score_scale_fct(scores[i - 1, j - 1])) + s
            # else:
            if True:
                g = t[i, j - 1] * W[i, j - 1]
                h = t[i - 1, j] * W[i - 1, j]

                if g < h:
                    hist[i, j] = 1
                    if hist[i, j] == hist[i - 1, j]:
                        W[i, j] = W[i - 1, j] * dhw
                    else:
                        W[i, j] = 1
                    s = h
                elif g > h:
                    hist[i, j] = -1
                    if hist[i, j] == hist[i, j - 1]:
                        W[i, j] = W[i, j - 1] * dgw
                    else:
                        W[i, j] = 1
                    s = g
                else:
                    hist[i, j] = 0

                try:
                    t[i, j] = (score_scale_fct(
                        scores[i - 1, j - 1])) + max(g, h)
                except Exception as e:
                    print(score_scale_fct.__name__)
                    print(score_scale_fct(scores[i - 1, j - 1]))
                    print(max(g, h))
                    raise e

                # t[i, j] = (scores[i - 1, j - 1] ** 4) + max(g, h)

            # NOTE(09/18): rlly not sure why i used to **4
            # t[i, j] = (scores[i - 1, j - 1] ** 4) + max(g, h)
    return t, hist

def analyse_dp(hist):
    path = []
    size_y, size_x = hist.size()
    yy, xx = size_y - 1, size_x - 1
    x_count = torch.zeros(size_x)
    while yy > 1 or xx > 1:
        path += [[xx - 1, yy - 1]]
        v = hist[yy, xx]
        x_count[xx] += 1

        if v == 2 and xx > 1 and yy > 1:
            xx -= 1
            yy -= 1

        elif v < 1 and xx > 1:
            xx -= 1
            print("decay to xx: %d (yy: %d)" % (xx, yy))
        elif v > 0 and yy > 1:
            yy -= 1
            print("continue to yy: %d" % yy)
        else:
            if xx > 1:
                xx -= 1
                print("EGALITE (decay to xx: %d)" % xx)
            elif yy > 1:
                yy -= 1
                print("EGALITE (decay to yy: %d)" % yy)

    return x_count, path

def slice_len(_slice):
    stop = _slice.stop
    start = _slice.start if _slice.start is not None else 0
    step = _slice.step if _slice.step is not None else 1
    return np.ceil((stop - start) / step)

def viz_dp(cumul_scores, scores, dp_scores, dp_hist, docx_slices,
           ctm_slices, path, output_path="similarities.html"):
    path = [";".join([str(_) for _ in coords]) for coords in path]
    margin_left = 10
    margin_top = 10

    width = 10
    height = 10
    x_space = 2
    y_space = 2

    docx_shift = height
    ctm_shift = width

    blocks = []

    i_docx = 0
    end_docx = docx_slices[i_docx].stop
    pos_y = margin_top
    cumul_max = cumul_scores.max().item()

    for y in range(scores.size(1) - 1):

        pos_x = margin_left
        if y == end_docx:
            pos_y += docx_shift
            i_docx += 1
            if i_docx < len(docx_slices):
                end_docx = docx_slices[i_docx].stop

        i_ctm = 0
        end_ctm = ctm_slices[i_ctm].stop
        for x in range(scores.size(0) - 1):
            if x == end_ctm:
                pos_x += ctm_shift
                i_ctm += 1
                if i_ctm < len(ctm_slices):
                    end_ctm = ctm_slices[i_ctm].stop
            score = scores[x, y]
            coord = "%d;%d" % (y, x)

            _class = ""
            if coord in path:
                _class = " class='path' "

            if score != 0.0 or coord in path:
                # print(score)
                gray = 255 * (1 - score)
                olor = "background-color: rgb(%d, %d, %d);" % (gray,
                                                               gray, gray)
                position = "top: %dpx;" % pos_y
                position += "left: %dpx;" % pos_x

                block = "

" % (
                    _class, " ".join([color, position]))
                blocks += [block]

            pos_x += width + x_space
        blocks += ["\n"]
        pos_y += height + y_space

    style = """
            """ % (width, height)

    head = "\n" + style + "\n\n"
    body = "\n" + "".join(blocks) + ""
    html = "" + head + body + ""
    with open(output_path, 'w') as f:
        print(html, file=f)
    print("Viz_dp output: %s" % output_path)

def align_slices(dp_path, docx_slices, ctm_slices,
                 one_doc_per_ctm=False, verbose=False):
    def log(*args, **kwargs):
        if verbose:
            print(*args, **kwargs)

    def inslice(val, s):
        start = 0 if s.start is None else s.start
        step = 1
        stop = s.stop
        return (val >= start and val < stop)

    print("Alignment")

    i_docx_slice = len(docx_slices) - 1
    i_ctm_slice = len(ctm_slices) - 1
    i_coord = 0

    # feature meant not to replicate CTM over 2 docx slice by
    # giving it to the best.
    prev_count = 0
    cur_count = 0
    check_count = False

    cur_slices = []
    processed_slices = []
    aligned_slices = [cur_slices]
    log("First slices:")
    log("* docx[#%d]: " % i_docx_slice, docx_slices[i_docx_slice])
    log("* ctm[#%d]: " % i_ctm_slice, ctm_slices[i_ctm_slice])

    def find_prev_aligned():
        for i in range(len(aligned_slices) - 2, 1, -1):
            if len(aligned_slices[i]) > 0:
                return aligned_slices[i][-1]

    while i_coord < len(dp_path):
        y, x = dp_path[i_coord]
        log("x, y: ", x, y)

        if i_docx_slice > 0 and not inslice(y, docx_slices[i_docx_slice]):
            # new doc slice on position
            log("[NEW DOCX SLICE] %d is over w/ %s" %
                (i_docx_slice, str(cur_slices)))

            if one_doc_per_ctm and len(
                    cur_slices) == 1 and len(aligned_slices) >= 2:
                # only 1 CTM means no CTM border, therefore we must check_count
                # a bit
                log("CHECK-COUNT: new docx slice after a len=1 one")
                i_prev_aligned_doc = max(len(aligned_slices) - 2, 0)
                while i_prev_aligned_doc > 0:
                    if len(aligned_slices[i_prev_aligned_doc]) >= 1:
                        break
                    i_prev_aligned_doc -= 1

                if cur_count > prev_count:
                    log("\t!! [%d|%d] higher cur_count" %
                        (cur_count, prev_count))
                    log("\t removing from align[%d|%d](%s), keeping in cur" % (
                        i_prev_aligned_doc, i_prev_aligned_doc - len(aligned_slices), str(aligned_slices[-2])))
                    _ = aligned_slices[i_prev_aligned_doc].pop()
                    if not _ in cur_slices:
                        cur_slices.insert(0, _)

                    log("\tswapping prev_count <- cur_count")
                    prev_count = cur_count
                else:
                    log("\t!! [%d|%d] lower cur_count" %
                        (cur_count, prev_count))
                    log("\t removing from cur_slices %s" % str(cur_slices))
                    cur_slices.pop()
                    log("\t keeping prev_count=%d" % prev_count)
            log("\t (resetting cur_count")
            cur_count = 0

            log("\t (resetting/appending cur_slices)")
            fpa = find_prev_aligned()
            if fpa is not None:
                try:
                    if len(cur_slices) > 0:
                        if one_doc_per_ctm:
                            assert cur_slices[0] + \
                                1 == fpa, "%d != %d" % (cur_slices[0] + 1, fpa)
                        else:
                            assert fpa in [cur_slices[0], cur_slices[0] + 1]
                except AssertionError as e:
                    log("aligned_slices[-1]: '%s'" % str(aligned_slices[-1]))
                    log("aligned_slices[-2]: '%s'" % str(aligned_slices[-2]))
                    log("aligned_slices[-3]: '%s'" % str(aligned_slices[-3]))
                    raise e
            cur_slices = []
            aligned_slices.append(cur_slices)
            i_docx_slice -= 1
            log("\t (#aligned_slices: %d)" % len(aligned_slices))

        elif i_ctm_slice >= 0 and (not inslice(x, ctm_slices[i_ctm_slice]) or i_coord + 1 == len(dp_path)):
            # new ctm slice on position
            log("[NEW CTM SLICE] (#%d is over)" % (i_ctm_slice))

            if one_doc_per_ctm:
                if len(cur_slices) == 1:
                    # Only check for 1rst one (not mandatory condition)
                    log("[CHECK-COUNT: first CTM")
                    # find relevant previous doc index
                    i_prev_aligned_doc = max(len(aligned_slices) - 2, 0)
                    while i_prev_aligned_doc > 0:
                        if len(aligned_slices[i_prev_aligned_doc]) >= 1:
                            break
                        i_prev_aligned_doc -= 1
                    log("\t(using i_prev_aligned_doc=%d (i.e. %d)" % (
                        i_prev_aligned_doc, i_prev_aligned_doc - len(aligned_slices)))
                    if len(aligned_slices[i_prev_aligned_doc]) >= 1:
                        log("\t(this prev_align has >= 1 ctm, continuing)")
                        if cur_slices[-1] == aligned_slices[i_prev_aligned_doc][-1]:
                            log(
                                "\t(last i_ctm match for cur/prev_align, continuing)")
                            if cur_count > prev_count:
                                log("\t!! [%d|%d] higher cur_count" %
                                    (cur_count, prev_count))
                                log("\t removing from prev_align %s, keeping in cur" % str(
                                    aligned_slices[i_prev_aligned_doc]))
                                _ = aligned_slices[i_prev_aligned_doc].pop()
                                if not _ in cur_slices:
                                    cur_slices.insert(0, _)
                                # log("\t also reseting prev_count")
                                # prev_count = 0
                            else:
                                log("\t!! [%d|%d] lower cur_count" %
                                    (cur_count, prev_count))
                                log("\t removing from cur, keeping in prev_align")
                                _ = cur_slices.pop()
                                assert _ in aligned_slices[i_prev_aligned_doc]
                                # log("\t keeping prev_count=%d" % prev_count)
                        else:
                            log("! aborting check_count: i_ctm does not match")
                    else:
                        log("! aborting check_count: prev_align has no ctm")
            log("\t resetting both prev/cur count")
            prev_count = 0
            cur_count = 0
            i_ctm_slice -= 1
            log("\t i_ctm_slice=%d" % i_ctm_slice)
        else:
            # continue on path
            i_coord += 1
            log("[CONTINUE] on path w/ i_coord:%d [%d]" %
                (i_coord, len(dp_path)))

        if i_ctm_slice not in cur_slices and i_ctm_slice >= 0:
            log("\t(add i_ctm_slice: %d un cur_slices)" % i_ctm_slice)
            cur_slices.append(i_ctm_slice)

        cur_count += 1

    for last_i_docx_slice in range(i_docx_slice, 0, -1):
        # add list docx slices that have been ignored?
        aligned_slices.append([])

    log("last slices: %s" % str(cur_slices))
    aligned_slices = [slices[::-1] for slices in aligned_slices[::-1]]

    log(str(one_doc_per_ctm), aligned_slices)
    log(len(aligned_slices), len(docx_slices))
    log(len(sum(aligned_slices, [])), len(ctm_slices))
    log(len(set(sum(aligned_slices, []))))
    if one_doc_per_ctm:
        assert_msgs = []
        assert_msgs += ["#aligned_slices: %d" % len(aligned_slices)]
        assert_msgs += ["#ctm_slices: %d" % len(ctm_slices)]
        assert_msgs += ["#docx_slices: %d" % len(docx_slices)]
        assert_msgs += ["#sum(aligned_slices): %d" %
                        len(sum(aligned_slices, []))]
        assert_msgs += ["#set(sum(aligned_slices)): %d" %
                        len(set(sum(aligned_slices, [])))]
        assert_msgs += ["i_docx_slice: %d, i_ctm_slice: %d" %
                        (i_docx_slice, i_ctm_slice)]
        assert_msgs += [str(docx_slices)]
        assert_msgs += [str(aligned_slices)]
        assert len(aligned_slices) == len(docx_slices) and len(
            sum(aligned_slices, [])) == len(ctm_slices), "\n".join(assert_msgs)

    return aligned_slices

def viz_alignement(scores, slices_align, docx_slices, ctm_slices, docx_sentences,
                   ctm_sentences, output_html="alignment.html", output_pt="alignment.pt",
                   extra_data={}):
    """
    Args:
        slices_align(list[list[int]]): alignment of docx to ctm.
            slices_align[i] contains id j such that
            ctm_slices[j] is aligned with docx_slices[i]
            extra_data: dict to save in output_pt in the first record
                        i.e. for all (k, v) in extra_data there will be
                        align[0][k] = v, align being saved to `output_pt`
    """
    import html

    def sentences2scores(sentences):
        return [s for sentence in sentences for w, s in sentence]

    def sentences2text(sentences, with_scores=False):
        if with_scores:
            text = " ".join([" ".join([w for w, s in sentence])
                             for sentence in sentences])
        else:
            text = " ".join([" ".join(sentence)
                             for sentence in sentences])
        text = " ".join(text.split())
        return text

    blocks = []
    align = []
    for i_docx_slice, i_ctm_slices in enumerate(slices_align):
        _docx_text = html.escape(sentences2text(
            docx_sentences[docx_slices[i_docx_slice]]))

        _ctm_slices_sentences = [ctm_sentences[ctm_slices[i_ctm_slice]]
                                 for i_ctm_slice in i_ctm_slices]
        _ctm_text = "

".join(
            [html.escape(sentences2text(_ctm_sentences, with_scores=True))
             for _ctm_sentences in _ctm_slices_sentences])

        _docx_block = '\n
\n' + _docx_text + '\n
'
        _ctm_block = '\n
\n' + _ctm_text + '\n
'
        block = '
' + _ctm_block + _docx_block + '
'
        blocks += [block]
        align += [{"ctm": [sentences2text(_ctm_sentences, with_scores=True)
                           for _ctm_sentences in _ctm_slices_sentences],
                   "doc": sentences2text(docx_sentences[docx_slices[i_docx_slice]]),
                   "ctm_scores": [sentences2scores(_ctm_sentences)
                                  for _ctm_sentences in _ctm_slices_sentences]
                   }]

    script = ""
    css = {
        ".block": ["width: 100%", "text-align: justify;", "background-color: white",
                   "display: inline-block", "margin-top: 10px"],
        "p": ["width: 49%"],
        "p.ctm": ["float: right"]
    }
    css_code = "\n".join(
        ["%s{\n\t%s\n}" % (k, "\n\t".join(["%s;" % instr
                                           for instr in v]))
         for k, v in css.items()])
    style = ""

    meta = '\n<meta charset="utf-8" />'
    title = "\n<title>Aligned segments</title>"
    head = "" + title + meta + script + style + "\n"
    body = "\n\n" + "\n
".join(blocks) + "\n"
    page = "\n" + head + body + ""

    if output_html is not None:
        with open(output_html, 'w') as out:
            out.write(page)
        print("Wrote html to: '%s'" % output_html)

    if output_pt is not None:
        for k, v in extra_data.items():
            align[0][k] = v
        with open(output_pt, 'wb') as out:
            torch.save(align, out)
        print("Wrote alignment to: '%s'" % output_pt)

max_stars_repo_name: pltrdy/autoalign

max_issues_repo_name: pltrdy/autoalign

max_forks_repo_name: pltrdy/autoalign


text:
from scipy.io import loadmat
import matplotlib.pyplot as plt
from hsi_toolkit.signature_detectors import *
from hsi_toolkit.util import get_RGB

"""
Demo script that runs all signature detectors in hsi_toolkit_py

Inputs:
	hsi_sub - n_row x n_col x n_band hyperspectral image
	tgt_spectra - n_band x 1 target signature vector
	wavelengths - n_band x 1 vector listing wavelength values for hsi_sub in nm
	gt_img_sub - n_row x n_col ground truths
	mask - binary image limiting detector operation to pixels where mask is true
	 	if not present or empty, no mask restrictions are used
Outputs:
	det_out - dictionary of RGB image, ground truth image, and detector outputs

6/2/2018 - Alina Zare
10/12/2018 - Python Implementation by Yutai Zhou
"""
# Load data
an_hsi_img_for_tgt_det_demo = loadmat('an_hsi_img_for_tgt_det_demo.mat')
hsi_sub = an_hsi_img_for_tgt_det_demo['hsi_sub']
tgt_spectra = an_hsi_img_for_tgt_det_demo['tgt_spectra']
tgt_spectra = tgt_spectra.squeeze()
wavelengths = an_hsi_img_for_tgt_det_demo['wavelengths']
gt_img_sub = an_hsi_img_for_tgt_det_demo['gtImg_sub']

det_out = {}
det_out['RGB'] = get_RGB(hsi_sub, wavelengths)
det_out['Ground Truth'] = gt_img_sub

# init detector args
guard_win = 1; bg_win = 3; beta = 0.001; n_dim_ss = 10
ems = hsi_sub[:3,1,:].T # need to provide background endmembers (can get them using SPICE unmixing)

# call detectors
abd_out = abd_detector(hsi_sub, tgt_spectra, ems)
det_out['ABD'] = abd_out
ace_out, _, _ = ace_detector(hsi_sub, tgt_spectra)
det_out['ACE Squared'] = ace_out
ace_local_out, _ = ace_local_detector(hsi_sub, tgt_spectra, guard_win = guard_win, bg_win = bg_win, beta = beta)
det_out['ACE Local Squared'] = ace_local_out
ace_ss_out = ace_ss_detector(hsi_sub, tgt_spectra)
det_out['ACE SS'] = ace_ss_out
ace_rt_out, _, _ = ace_rt_detector(hsi_sub, tgt_spectra)
det_out['ACE RT'] = ace_rt_out
ace_rt_max_out, _, _ = ace_rt_max_detector(hsi_sub, tgt_spectra)
det_out['ACE RT Max'] = ace_rt_max_out
amsd_out= amsd_detector(hsi_sub, tgt_spectra, n_dim_tgt = 1, n_dim_bg = 3)
det_out['AMSD'] = amsd_out
ccmf_out, _ = ccmf_detector(hsi_sub, tgt_spectra, n_comp = 2)
det_out['CCMF'] = ccmf_out
cem_out, w = cem_detector(hsi_sub, tgt_spectra)
det_out['CEM'] = cem_out
ctmf_out, _ = ctmf_detector(hsi_sub, tgt_spectra, n_cluster = 2)
det_out['CTMF'] = ctmf_out
#vftmf_out = ftmf_detector(hsi_sub, tgt_spectra, gamma = 1) <--- moved into dev
# det_out['FTMF'] = ftmf_out
ha_out = ha_detector(hsi_sub, tgt_spectra, ems, n_comp = 2)
det_out['HA'] = ha_out
hsd_out, _ = hsd_detector(hsi_sub, tgt_spectra, ems)
det_out['HSD'] = hsd_out
hsd_local_out = hsd_local_detector(hsi_sub, tgt_spectra, ems, guard_win = guard_win, bg_win = bg_win, beta = beta)
det_out['HSD Local'] = hsd_local_out
hua_out = hua_detector(hsi_sub, tgt_spectra, ems, n_comp = 2)
det_out['HUA'] = hua_out
# mtmf_out,_ = mtmf_statistic(hsi_sub, tgt_spectra) <--- moved into dev
# det_out['MTMF'] = mtmf_out
smf_out, _, _ = smf_detector(hsi_sub, tgt_spectra)
det_out['SMF'] = smf_out
smf_local_out = smf_local_detector(hsi_sub, tgt_spectra, guard_win = guard_win, bg_win = bg_win)
det_out['SMF Local'] = smf_local_out
smf_max_out = smf_max_detector(hsi_sub, tgt_spectra)
det_out['SMF Max'] = smf_max_out
fam_statistic_out = fam_statistic(hsi_sub, tgt_spectra)
det_out['FAM Statistic'] = fam_statistic_out
osp_out = osp_detector(hsi_sub, tgt_spectra, n_dim_ss = 10)
det_out['OSP'] = osp_out
# qmf_out = qmf_detector(hsi_sub, tgt_spectra, 0.1 * np.eye(hsi_sub.shape[2])) <--- moved into dev
# det_out['QMF'] = qmf_out
sam_out = sam_detector(hsi_sub, tgt_spectra)
det_out['SAM'] = sam_out
# spsmf_out = spsmf_detector(hsi_sub, tgt_spectra) <--- moved into dev
# det_out['SPSMF'] = spsmf_out
palm_out = palm_detector(hsi_sub, tgt_spectra, n_comp = 5)
det_out['PALM'] = palm_out

# Segmented Detector Examples

# # # get Segments (using K-means here, but better ways to do this in general, see context-dependent methods for detection)
# n_cluster = 3
# n_row, n_col, n_band = hsi_sub.shape
# idx = KMeans(n_clusters = n_cluster, n_init = 1).fit(hsi_sub.reshape((n_row * n_col, n_band), order='F')).labels_
# idx_img = idx.reshape((n_row,n_col), order='F')

# segments = np.zeros((n_cluster,n_row,n_col))
# for i in range(n_cluster):
# 	segments[i,:,:] = idx_img == i

# # Segmented Spectral Angle Mapper
# seg_sam_out = img_seg(sam_detector,hsi_sub, tgt_spectra, segments)
# det_out['Seg SAM'] = seg_sam_out

# # Segmented Spectral Angle Mapper
# seg_ace_out,_,_ = img_seg(ace_detector,hsi_sub, tgt_spectra, segments)
# det_out['Seg ACE'] = seg_ace_out
# plt.imshow(seg_ace_out)
# plt.show()

# # visualization
plt.figure(figsize=(10, 15))
plt.subplots_adjust(hspace=.5)
n_row = 5; n_col = 7
#
i = 1
for key, value in det_out.items():
	plt.subplot(n_row, n_col, i);
	plt.imshow(value); plt.title(key)
	i += 1
plt.show()

reason: from scipy


text:
import tensorflow as tf
import tensorlayer as tl
import numpy as np
from tensorlayer.cost import cross_entropy_seq, cross_entropy_seq_with_mask
from tqdm import tqdm
from sklearn.utils import shuffle
from data.twitter import data
from tensorlayer.models.seq2seq import Seq2seq
from tensorlayer.models.seq2seq_with_attention import Seq2seqLuongAttention
import os
import argparse

def initial_setup(data_corpus):
    metadata, idx_q, idx_a = data.load_data(PATH='data/{}/'.format(data_corpus))
    (trainX, trainY), (testX, testY), (validX, validY) = data.split_dataset(idx_q, idx_a)
    trainX = tl.prepro.remove_pad_sequences(trainX.tolist())
    trainY = tl.prepro.remove_pad_sequences(trainY.tolist())
    testX = tl.prepro.remove_pad_sequences(testX.tolist())
    testY = tl.prepro.remove_pad_sequences(testY.tolist())
    validX = tl.prepro.remove_pad_sequences(validX.tolist())
    validY = tl.prepro.remove_pad_sequences(validY.tolist())
    return metadata, trainX, trainY, testX, testY, validX, validY

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Train Seq2Seq Chatbot')
    parser.add_argument('--dataset',type=str,required=True,help='Dataset to be used twitter or cornell')
    parser.add_argument('--epochs',type=str,default=50,help='Number of training epochs')
    args = parser.parse_args()
    data_corpus = args.dataset

    #data preprocessing
    metadata, trainX, trainY, testX, testY, validX, validY = initial_setup(data_corpus)

    # Parameters
    src_len = len(trainX)
    tgt_len = len(trainY)

    assert src_len == tgt_len

    batch_size = 32
    n_step = src_len // batch_size
    src_vocab_size = len(metadata['idx2w']) # 8002 (0~8001)
    emb_dim = 1024

    word2idx = metadata['w2idx']   # dict  word 2 index
    idx2word = metadata['idx2w']   # list index 2 word

    unk_id = word2idx['unk']   # 1
    pad_id = word2idx['_']     # 0

    start_id = src_vocab_size  # 8002
    end_id = src_vocab_size + 1  # 8003

    word2idx.update({'start_id': start_id})
    word2idx.update({'end_id': end_id})
    idx2word = idx2word + ['start_id', 'end_id']

    src_vocab_size = tgt_vocab_size = src_vocab_size + 2

    num_epochs = args.epochs
    vocabulary_size = src_vocab_size

    def inference(seed, top_n):
        model_.eval()
        seed_id = [word2idx.get(w, unk_id) for w in seed.split(" ")]
        sentence_id = model_(inputs=[[seed_id]], seq_length=20, start_token=start_id, top_n = top_n)
        sentence = []
        for w_id in sentence_id[0]:
            w = idx2word[w_id]
            if w == 'end_id':
                break
            sentence = sentence + [w]
        return sentence

    decoder_seq_length = 20
    model_ = Seq2seq(
        decoder_seq_length = decoder_seq_length,
        cell_enc=tf.keras.layers.GRUCell,
        cell_dec=tf.keras.layers.GRUCell,
        n_layer=3,
        n_units=256,
        embedding_layer=tl.layers.Embedding(vocabulary_size=vocabulary_size, embedding_size=emb_dim),
        )

    # Uncomment below statements if you have already saved the model

    # load_weights = tl.files.load_npz(name='model.npz')
    # tl.files.assign_weights(load_weights, model_)

    if os.path.exists('model_'+data_corpus+'/') and len(os.listdir('model_'+data_corpus+'/'))!=0:
        files = []
        for each in os.listdir('model_'+data_corpus+'/'):
            files.append(os.path.join('model_'+data_corpus+'/',each))
        files.sort(key=lambda x:float(x[-10:-4]))
        load_weights = tl.files.load_npz(name=files[0])
        tl.files.assign_weights(load_weights, model_)
        print("Weight ", files[0]," Loaded...............")
    elif not os.path.exists('model_'+data_corpus+'/'):
            os.mkdir('model_'+data_corpus+'/')

    optimizer = tf.optimizers.Adam(learning_rate=0.001)
    model_.train()

    seeds = ["happy birthday have a nice day",
                 "how are you ?"]

    for epoch in range(num_epochs):
        model_.train()
        trainX, trainY = shuffle(trainX, trainY, random_state=0)
        total_loss, n_iter = 0, 0
        for X, Y in tqdm(tl.iterate.minibatches(inputs=trainX, targets=trainY, batch_size=batch_size, shuffle=False),
                        total=n_step, desc='Epoch[{}/{}]'.format(epoch + 1, num_epochs), leave=False):

            X = tl.prepro.pad_sequences(X)
            _target_seqs = tl.prepro.sequences_add_end_id(Y, end_id=end_id)
            _target_seqs = tl.prepro.pad_sequences(_target_seqs, maxlen=decoder_seq_length)
            _decode_seqs = tl.prepro.sequences_add_start_id(Y, start_id=start_id, remove_last=False)
            _decode_seqs = tl.prepro.pad_sequences(_decode_seqs, maxlen=decoder_seq_length)
            _target_mask = tl.prepro.sequences_get_mask(_target_seqs)

            with tf.GradientTape() as tape:
                ## compute outputs
                output = model_(inputs = [X, _decode_seqs])

                output = tf.reshape(output, [-1, vocabulary_size])
                ## compute loss and update model
                loss = cross_entropy_seq_with_mask(logits=output, target_seqs=_target_seqs, input_mask=_target_mask)

                grad = tape.gradient(loss, model_.all_weights)
                optimizer.apply_gradients(zip(grad, model_.all_weights))

            total_loss += loss
            n_iter += 1

        # printing average loss after every epoch
        print('Epoch [{}/{}]: loss {:.4f}'.format(epoch + 1, num_epochs, total_loss / n_iter))

        for seed in seeds:
            print("Query >", seed)
            top_n = 3
            for i in range(top_n):
                sentence = inference(seed, top_n)
                print(" >", ' '.join(sentence))

        tl.files.save_npz(model_.all_weights, name='model_{}/model_{}_{:.4f}.npz'.format(data_corpus, epoch+1, total_loss / n_iter))


text:
import torch
import cv2
import numpy as np
from models.cspnet import CSPNet_p3p4p5
from utils.keras_weights_loader import load_keras_weights
from utils.utils import *

device = 'cuda:1'

def detect_face(model, img, scale=1, flip=False):
    img_h, img_w = img.shape[:2]
    img_h_new, img_w_new = int(np.ceil(scale * img_h / 16) * 16), int(np.ceil(scale * img_w / 16) * 16)
    scale_h, scale_w = img_h_new / img_h, img_w_new / img_w

    img_s = cv2.resize(img, None, None, fx=scale_w, fy=scale_h, interpolation=cv2.INTER_LINEAR)
    # img_h, img_w = img_s.shape[:2]
    # print frame_number
    input_dim = [img_h_new, img_w_new]

    if flip:
        img_sf = cv2.flip(img_s, 1)
        # x_rcnn = format_img_pad(img_sf, C)
        x_rcnn = format_img(img_sf)
    else:
        # x_rcnn = format_img_pad(img_s, C)
        x_rcnn = format_img(img_s)
    x = torch.from_numpy(x_rcnn).to(device)
    x = x.permute(0, 3, 1, 2)
    x_cls, x_reg, x_off = model(x)
    print('x reg shape ', x_reg.shape)
    Y = [x_cls.detach().cpu().numpy(), x_reg.detach().cpu().numpy(), x_off.detach().cpu().numpy()]
    boxes = parse_wider_offset(Y, input_dim, score=0.3, nmsthre=0.4)
    if len(boxes) > 0:
        keep_index = np.where(np.minimum(boxes[:, 2] - boxes[:, 0], boxes[:, 3] - boxes[:, 1]) >= 12)[0]
        boxes = boxes[keep_index, :]
    if len(boxes) > 0:
        if flip:
            boxes[:, [0, 2]] = img_s.shape[1] - boxes[:, [2, 0]]
        boxes[:, 0:4:2] = boxes[:, 0:4:2] / scale_w
        boxes[:, 1:4:2] = boxes[:, 1:4:2] / scale_h
    else:
        boxes = np.empty(shape=[0, 5], dtype=np.float32)
    return boxes

if __name__ == '__main__':

    model = CSPNet_p3p4p5(num_scale=2)
    load_keras_weights(model, 'net_e382_l0.hdf5')
    model.to(device).eval()

    img = cv2.imread('worlds-largest-selfie.jpg')
    bboxes = detect_face(model, img)

    print('bbox----', bboxes.shape)
    for b in bboxes:
        cv2.rectangle(img, (int(b[0]), int(b[1])), (int(b[2]), int(b[3])), (0, 0, 255), 2)
    cv2.imwrite('out.jpg', img)


text:
# Copyright 2019-2021 ETH Zurich and the DaCe authors. All rights reserved.
# Original application code: NPBench - https://github.com/spcl/npbench

import dace.dtypes
import numpy as np
import dace as dc
import pytest
from dace.fpga_testing import fpga_test
from dace.transformation.interstate import FPGATransformSDFG, InlineSDFG
from dace.transformation.dataflow import StreamingMemory, StreamingComposition, MapFusion
from dace.transformation.auto.auto_optimize import auto_optimize
import argparse

TMAX, NX, NY = (dc.symbol(s, dtype=dc.int32) for s in ('TMAX', 'NX', 'NY'))

@dc.program
def kernel(ex: dc.float32[NX, NY], ey: dc.float32[NX, NY], hz: dc.float32[NX, NY], _fict_: dc.float32[TMAX]):
    for t in range(TMAX):
        ey[0, :] = _fict_[t]
        ey[1:, :] -= 0.5 * (hz[1:, :] - hz[:-1, :])
        ex[:, 1:] -= 0.5 * (hz[:, 1:] - hz[:, :-1])
        hz[:-1, :-1] -= 0.7 * (ex[:-1, 1:] - ex[:-1, :-1] + ey[1:, :-1] - ey[:-1, :-1])

def init_data(TMAX, NX, NY):

    ex = np.empty((NX, NY), dtype=np.float32)
    ey = np.empty((NX, NY), dtype=np.float32)
    hz = np.empty((NX, NY), dtype=np.float32)
    _fict_ = np.empty((TMAX, ), dtype=np.float32)
    for i in range(TMAX):
        _fict_[i] = i
    for i in range(NX):
        for j in range(NY):
            ex[i, j] = (i * (j + 1)) / NX
            ey[i, j] = (i * (j + 2)) / NY
            hz[i, j] = (i * (j + 3)) / NX

    return ex, ey, hz, _fict_

def ground_truth(TMAX, NX, NY, ex, ey, hz, _fict_):

    for t in range(TMAX):
        ey[0, :] = _fict_[t]
        ey[1:, :] -= 0.5 * (hz[1:, :] - hz[:NX - 1, :])
        ex[:, 1:] -= 0.5 * (hz[:, 1:] - hz[:, :NY - 1])
        hz[:NX - 1, :NY - 1] -= 0.7 * (ex[:NX - 1, 1:] - ex[:NX - 1, :NY - 1] + ey[1:, :NY - 1] - ey[:NX - 1, :NY - 1])

def run_fdtd_2d(device_type: dace.dtypes.DeviceType):
    '''
    Runs FDTD-2D for the given device
    :return: the SDFG
    '''

    # Initialize data (polybench medium size)
    TMAX, NX, NY = (100, 200, 240)

    ex, ey, hz, _fict_ = init_data(TMAX, NX, NY)
    gt_ex, gt_ey, gt_hz = np.copy(ex), np.copy(ey), np.copy(hz)

    if device_type in {dace.dtypes.DeviceType.CPU, dace.dtypes.DeviceType.GPU}:
        # Parse the SDFG and apply autopot
        sdfg = kernel.to_sdfg()
        sdfg = auto_optimize(sdfg, device_type)
        sdfg(ex=ex, ey=ey, hz=hz, _fict_=_fict_, TMAX=TMAX, NX=NX, NY=NY)
    elif device_type == dace.dtypes.DeviceType.FPGA:
        # Parse SDFG and apply FPGA friendly optimization
        sdfg = kernel.to_sdfg(simplify=True)
        sdfg.apply_transformations_repeated([MapFusion])
        applied = sdfg.apply_transformations([FPGATransformSDFG])
        assert applied == 1

        sm_applied = sdfg.apply_transformations_repeated([InlineSDFG, StreamingMemory],
                                                         [{}, {
                                                             'storage': dace.StorageType.FPGA_Local
                                                         }],
                                                         print_report=True)

        assert sm_applied == 2

        sdfg.apply_transformations_repeated([InlineSDFG])
        # In this case, we want to generate the top-level state as an host-based state,
        # not an FPGA kernel. We need to explicitly indicate that
        sdfg.states()[0].location["is_FPGA_kernel"] = False

        sdfg(ex=ex, ey=ey, hz=hz, _fict_=_fict_, TMAX=TMAX, NX=NX, NY=NY)

    # Compute ground truth and validate result
    ground_truth(TMAX, NX, NY, gt_ex, gt_ey, gt_hz, _fict_=_fict_)
    diff_ex = np.linalg.norm(gt_ex - ex) / np.linalg.norm(gt_ex)
    diff_ey = np.linalg.norm(gt_ex - ex) / np.linalg.norm(gt_ex)
    diff_hz = np.linalg.norm(gt_ex - ex) / np.linalg.norm(gt_ex)
    tol = 1e-6

    assert diff_ex < tol
    assert diff_ey < tol
    assert diff_hz < tol

    return sdfg

def test_cpu():
    run_fdtd_2d(dace.dtypes.DeviceType.CPU)

@pytest.mark.gpu
def test_gpu():
    run_fdtd_2d(dace.dtypes.DeviceType.GPU)

@fpga_test(assert_ii_1=False)
def test_fpga():
    return run_fdtd_2d(dace.dtypes.DeviceType.FPGA)

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("-t", "--target", default='cpu', choices=['cpu', 'gpu', 'fpga'], help='Target platform')

    args = vars(parser.parse_args())
    target = args["target"]

    if target == "cpu":
        run_fdtd_2d(dace.dtypes.DeviceType.CPU)
    elif target == "gpu":
        run_fdtd_2d(dace.dtypes.DeviceType.GPU)
    elif target == "fpga":
        run_fdtd_2d(dace.dtypes.DeviceType.FPGA)


text:
import torch as T
import torch.nn as nn

import pandas as pd
import numpy as np

class Policy(nn.Module):
    def __init__(self, obs_dim, act_dim):
        super(Policy, self).__init__()

        # Layers
        self.linear1 = nn.Linear(obs_dim, 50)
        self.linear2 = nn.Linear(50, 30)
        self.linear3 = nn.Linear(30, act_dim)

    def forward(self, x):
        x = self.linear1(x)
        x = self.linear2(x)
        x = self.linear3(x)

        return x

max_stars_repo_name: mohdabdin/Evolving-RL-Agents

max_issues_repo_name: mohdabdin/Evolving-RL-Agents

max_forks_repo_name: mohdabdin/Evolving-RL-Agents


text:
#!/bin/python
# -*- UTF-8 -*-

import numpy as np
from matplotlib.pyplot import plot
from matplotlib.pyplot import show
N = 8
weights = np.hanning(N)
print("weights", weights)
bhp = np.loadtxt("BHP.csv", delimiter=",", usecols=(6,), unpack=True)
print(bhp, len(bhp))
bhp_returns = np.diff(bhp) / bhp[: -1] # (x(i+1) - x(i) / x(i)
# convolve(weight, input)
smooth_bhp = np.convolve(weights / weights.sum(), bhp_returns)[N - 1: -N + 1]
vale = np.loadtxt("VALE.csv", delimiter=",", usecols=(6,), unpack=True)
vale_returns = np.diff(vale) / vale[: -1]
smooth_vale = np.convolve(weights, vale_returns)[N - 1: -N + 1]
t = np.arange(N - 1, len(bhp_returns))

plot(t, bhp_returns[N - 1: ], lw=1.0)
plot(t, smooth_bhp, lw=2.0)
plot(t, vale_returns[N - 1: ], lw=1.0)
plot(t, smooth_vale, lw=2.0)
show()

K = 3

t = np.arange(N - 1, len(bhp_returns))
poly_bhp = np.polyfit(t, smooth_bhp, K)
poly_vale = np.polyfit(t, smooth_vale, K)
poly_sub = np.polysub(poly_bhp, poly_vale)
x_points = np.roots(poly_sub)
print("Intersection points", x_points)
reals = np.isreal(x_points)
print("Real number?", reals)
print("x_points before select", x_points)
x_points = np.select([reals], [x_points])
print("x_points after select", x_points)
x_points= x_points.real
print("Real Intersection points", x_points)
print("San 0s", np.trim_zeros(x_points))


text:
from unittest.mock import patch

import numpy as np
import pytest

from sm.engine.postprocessing.ion_thumbnail import generate_ion_thumbnail, ALGORITHMS
from sm.engine.db import DB
from .utils import create_test_molecular_db, create_test_ds

OLD_IMG_ID = 'old-ion-thumb-id'
IMG_ID = 'new-ion-thumb-id'
IMG_URL = 'ion-thumb-url'

def _make_fake_ds(db, metadata, ds_config):
    ds = create_test_ds(metadata=metadata, config=ds_config)

    moldb = create_test_molecular_db()
    (job_id,) = db.insert_return(
        "INSERT INTO job (moldb_id, ds_id) VALUES (%s, %s) RETURNING id", rows=[(moldb.id, ds.id)]
    )
    db.insert(
        (
            "INSERT INTO annotation ("
            "   job_id, formula, chem_mod, neutral_loss, adduct, msm, fdr, stats, iso_image_ids"
            ") "
            "VALUES (%s, %s, '', '', %s, 1, 0, '{}', %s)"
        ),
        rows=[(job_id, f'H{i+1}O', '+H', [str(i), str(1000 + i)]) for i in range(200)],
    )

    return ds

def _mock_get_ion_images_for_analysis(ds_id, img_ids, **kwargs):
    images = np.unpackbits(np.arange(len(img_ids), dtype=np.uint8)).reshape((len(img_ids), 8))
    mask = np.ones((4, 2))
    return images, mask, (4, 2)

@pytest.mark.parametrize('algorithm', [alg for alg in ALGORITHMS.keys()])
def test_creates_ion_thumbnail(test_db, algorithm, metadata, ds_config):
    db = DB()
    ds = _make_fake_ds(db, metadata, ds_config)

    with patch('sm.engine.postprocessing.ion_thumbnail.image_storage') as image_storage_mock:
        image_storage_mock.post_image.return_value = IMG_ID
        image_storage_mock.get_image_url.return_value = IMG_URL
        image_storage_mock.get_ion_images_for_analysis.side_effect = (
            _mock_get_ion_images_for_analysis
        )

        generate_ion_thumbnail(db, ds, algorithm=algorithm)

        ion_thumbnail, ion_thumbnail_url = db.select_one(
            "SELECT ion_thumbnail, ion_thumbnail_url FROM dataset WHERE id = %s", [ds.id]
        )
        assert ion_thumbnail == IMG_ID
        assert ion_thumbnail_url == IMG_URL
        assert image_storage_mock.post_image.called


text:
from gensim.models import Word2Vec
from time import time
from tqdm import tqdm
from type4py import logger, AVAILABLE_TYPES_NUMBER, TOKEN_SEQ_LEN
from type4py.utils import mk_dir_not_exist
import os
import multiprocessing
import numpy as np
import pandas as pd

logger.name = __name__
tqdm.pandas()

W2V_VEC_LENGTH = 100

class TokenIterator:
    def __init__(self, param_df: pd.DataFrame, return_df: pd.DataFrame,
                 var_df: pd.DataFrame) -> None:
        self.param_df = param_df
        self.return_df = return_df
        self.var_df = var_df

    def __iter__(self):
        for return_expr_sentences in self.return_df['return_expr_str'][self.return_df['return_expr_str'] != '']:
            yield return_expr_sentences.split()

        for code_occur_sentences in self.param_df['arg_occur'][self.param_df['arg_occur'] != '']:
            yield code_occur_sentences.split()

        for func_name_sentences in self.param_df['func_name'][~self.param_df['func_name'].isnull()]:
            yield func_name_sentences.split()

        for arg_names_sentences in self.return_df['arg_names_str'][self.return_df['arg_names_str'] != '']:
            yield arg_names_sentences.split()

        for var_names_sentences in self.var_df['var_name'][self.var_df['var_name'].notnull()]:
            yield var_names_sentences.split()

        for var_occur_sentences in self.var_df['var_occur'][self.var_df['var_occur'] != '']:
            yield var_occur_sentences.split()

class W2VEmbedding:
    """
    Word2Vec embeddings for code tokens and identifiers
    """
    def __init__(self, param_df: pd.DataFrame, return_df: pd.DataFrame,
                 var_df: pd.DataFrame, w2v_model_tk_path) -> None:
        self.param_df = param_df
        self.return_df = return_df
        self.var_df = var_df
        self.w2v_model_tk_path = w2v_model_tk_path

    def train_model(self, corpus_iterator: TokenIterator, model_path_name: str) -> None:
        """
        Train a Word2Vec model and save the output to a file.
        :param corpus_iterator: class that can provide an iterator that goes through the corpus
        :param model_path_name: path name of the output file
        """

        w2v_model = Word2Vec(min_count=5,
                             window=5,
                             vector_size=W2V_VEC_LENGTH,
                             workers=multiprocessing.cpu_count())

        t = time()
        w2v_model.build_vocab(corpus_iterable=corpus_iterator)
        logger.info('Built W2V vocab in {} mins'.format(round((time() - t) / 60, 2)))
        logger.info(f"W2V model's vocab size: {len(w2v_model.wv):,}")

        t = time()
        w2v_model.train(corpus_iterable=corpus_iterator,
                        total_examples=w2v_model.corpus_count,
                        epochs=20,
                        report_delay=1)

        logger.info('Built W2V model in {} mins'.format(round((time() - t) / 60, 2)))
        w2v_model.save(model_path_name)

    def train_token_model(self):
        """
        Trains a W2V model for tokens.
        """

        self.train_model(TokenIterator(self.param_df, self.return_df, self.var_df), self.w2v_model_tk_path)

def vectorize_sequence(sequence: str, feature_length: int, w2v_model: Word2Vec) -> np.ndarray:
    """
    Vectorize a sequence to a multi-dimensial NumPy array
    """

    vector = np.zeros((feature_length, W2V_VEC_LENGTH))

    for i, word in enumerate(sequence.split()):
        if i >= feature_length:
            break
        try:
            vector[i] = w2v_model.wv[word]
        except KeyError:
            pass

    return vector

class IdentifierSequence:
    """
    Vector representation of identifiers
    """
    def __init__(self, identifiers_embd, arg_name, args_name, func_name, var_name):

        self.identifiers_embd = identifiers_embd
        self.arg_name = arg_name
        self.args_name = args_name
        self.func_name = func_name
        self.var_name = var_name

    def seq_length_param(self):

        return {
            "arg_name": 10,
            "sep": 1,
            "func_name": 10,
            "args_name": 10
        }

    def seq_length_return(self):

        return {
            "func_name": 10,
            "sep": 1,
            "args_name": 10,
            "padding": 10
        }

    def seq_length_var(self):
        return {
            'var_name': 10,
            "padding": 21
        }

    def __gen_datapoint(self, seq_length):
        datapoint = np.zeros((sum(seq_length.values()), W2V_VEC_LENGTH))
        separator = np.ones(W2V_VEC_LENGTH)

        p = 0
        for seq, length in seq_length.items():
            if seq == "sep":
                datapoint[p] = separator
                p += 1
            elif seq == 'padding':
                for i in range(0, length):
                    datapoint[p] = np.zeros(W2V_VEC_LENGTH)
                    p += 1
            else:
                try:
                    for w in vectorize_sequence(self.__dict__[seq], length, self.identifiers_embd):
                        datapoint[p] = w
                        p += 1
                except AttributeError:
                    pass

        return datapoint

    # def param_datapoint(self):
    #     return self.__gen_datapoint(self.seq_length_param())

    # def return_datapoint(self):
    #     return self.__gen_datapoint(self.seq_length_return())

    # def var_datapoint(self):
    #     return self.__gen_datapoint(self.seq_length_var())

    def generate_datapoint(self):
        if self.arg_name is not None and self.args_name is not None and self.func_name is None:
            return self.__gen_datapoint(self.seq_length_param())
        elif self.args_name is not None and self.func_name is not None:
            return self.__gen_datapoint(self.seq_length_return())
        elif self.var_name is not None:
            return self.__gen_datapoint(self.seq_length_var())

class TokenSequence:
    """
    Vector representation of code tokens
    """
    def __init__(self, token_model, len_tk_seq, num_tokens_seq, args_usage,
                 return_expr, var_usage):
        self.token_model = token_model
        self.len_tk_seq = len_tk_seq
        self.num_tokens_seq = num_tokens_seq
        self.args_usage = args_usage
        self.return_expr = return_expr
        self.var_usage = var_usage

    def param_datapoint(self):
        datapoint = np.zeros((self.num_tokens_seq*self.len_tk_seq, W2V_VEC_LENGTH))
        for i, w in enumerate(vectorize_sequence(self.args_usage, self.num_tokens_seq*self.len_tk_seq, self.token_model)):
            datapoint[i] = w
        return datapoint

    def return_datapoint(self):
        datapoint = np.zeros((self.num_tokens_seq*self.len_tk_seq, W2V_VEC_LENGTH))

        if isinstance(self.return_expr, str):
            p = 0
            for w in vectorize_sequence(self.return_expr, self.len_tk_seq, self.token_model):
                datapoint[p] = w
                p += 1
        return datapoint

    def var_datapoint(self):
        datapoint = np.zeros((self.num_tokens_seq*self.len_tk_seq, W2V_VEC_LENGTH))
        for i, w in enumerate(vectorize_sequence(self.var_usage, self.num_tokens_seq*self.len_tk_seq, self.token_model)):
            datapoint[i] = w
        return datapoint

    def generate_datapoint(self):
        if self.args_usage is not None:
            return self.param_datapoint()
        elif self.return_expr is not None:
            return self.return_datapoint()
        elif self.var_usage is not None:
            return self.var_datapoint()

def process_datapoints(df, output_path, embedding_type, type, trans_func, cached_file: bool=True):

    if not os.path.exists(os.path.join(output_path, embedding_type + type + '_datapoints_x.npy')) or not cached_file:
        datapoints = df.apply(trans_func, axis=1)

        datapoints_X = np.stack(datapoints.progress_apply(lambda x: x.generate_datapoint()),
                                axis=0)
        np.save(os.path.join(output_path, embedding_type + type + '_datapoints_x'), datapoints_X)

        return datapoints_X
    else:
        logger.warn(f"file {embedding_type + type + '_datapoints_x'} exists!")
        return

def type_vector(size, index):
    v = np.zeros(size)
    v[index] = 1
    return v

def gen_aval_types_datapoints(df_params, df_ret, df_var, set_type, output_path, cached_file: bool=False):
    """
    It generates data points for available types.
    """

    if not (os.path.exists(os.path.join(output_path, f'params_{set_type}_aval_types_dp.npy')) and os.path.exists(os.path.join(output_path,
            f'ret_{set_type}_aval_types_dp.npy'))) and os.path.exists(os.path.join(output_path, f'var_{set_type}_aval_types_dp.npy')) \
                 or not cached_file:

        aval_types_params = np.stack(df_params.progress_apply(lambda row: type_vector(AVAILABLE_TYPES_NUMBER, row.param_aval_enc),
                                                    axis=1), axis=0)
        aval_types_ret = np.stack(df_ret.progress_apply(lambda row: type_vector(AVAILABLE_TYPES_NUMBER, row.ret_aval_enc),
                                            axis=1), axis=0)
        aval_types_var = np.stack(df_var.progress_apply(lambda row: type_vector(AVAILABLE_TYPES_NUMBER, row.var_aval_enc),
                                            axis=1), axis=0)

        np.save(os.path.join(output_path, f'params_{set_type}_aval_types_dp'), aval_types_params)
        np.save(os.path.join(output_path, f'ret_{set_type}_aval_types_dp'), aval_types_ret)
        np.save(os.path.join(output_path, f'var_{set_type}_aval_types_dp'), aval_types_var)

        return aval_types_params, aval_types_ret, aval_types_var
    else:
        logger.warn(f'file params_{set_type}_aval_types_dp.npy exists!')
        logger.warn(f'file ret_{set_type}_aval_types_dp.npy exists!')
        logger.warn(f'file var_{set_type}_aval_types_dp.npy exists!')
        return None, None, None

def gen_labels_vector(params_df: pd.DataFrame, returns_df: pd.DataFrame, var_df: pd.DataFrame,
                      set_type: str, output_path: str):
    """
    It generates a flattened labels vector
    """
    np.save(os.path.join(output_path, f'params_{set_type}_dps_y_all'), params_df['arg_type_enc_all'].values)
    np.save(os.path.join(output_path, f'ret_{set_type}_dps_y_all'), returns_df['return_type_enc_all'].values)
    np.save(os.path.join(output_path, f'var_{set_type}_dps_y_all'), var_df['var_type_enc_all'].values)

    return params_df['arg_type_enc_all'].values, returns_df['return_type_enc_all'].values, var_df['var_type_enc_all'].values

def vectorize_args_ret(output_path: str):
    """
    Creates vector representation of functions' arguments and return values
    """

    train_param_df = pd.read_csv(os.path.join(output_path, "_ml_param_train.csv"), na_filter=False)
    train_return_df = pd.read_csv(os.path.join(output_path, "_ml_ret_train.csv"), na_filter=False)
    train_var_df = pd.read_csv(os.path.join(output_path, "_ml_var_train.csv"), na_filter=False)
    logger.info("Loaded the training data")

    valid_param_df = pd.read_csv(os.path.join(output_path, "_ml_param_valid.csv"), na_filter=False)
    valid_return_df = pd.read_csv(os.path.join(output_path, "_ml_ret_valid.csv"), na_filter=False)
    valid_var_df = pd.read_csv(os.path.join(output_path, "_ml_var_valid.csv"), na_filter=False)
    logger.info("Loaded the validation data")

    test_param_df = pd.read_csv(os.path.join(output_path, "_ml_param_test.csv"), na_filter=False)
    test_return_df = pd.read_csv(os.path.join(output_path, "_ml_ret_test.csv"), na_filter=False)
    test_var_df = pd.read_csv(os.path.join(output_path, "_ml_var_test.csv"), na_filter=False)
    logger.info("Loaded the test data")

    if not os.path.exists(os.path.join(output_path, 'w2v_token_model.bin')):
        embedder = W2VEmbedding(train_param_df, train_return_df, train_var_df,
                                os.path.join(output_path, 'w2v_token_model.bin'))
        embedder.train_token_model()
    else:
        logger.warn("Loading an existing pre-trained W2V model!")

    w2v_token_model = Word2Vec.load(os.path.join(output_path, 'w2v_token_model.bin'))

    # Create dirs for vectors
    mk_dir_not_exist(os.path.join(output_path, "vectors"))
    mk_dir_not_exist(os.path.join(output_path, "vectors", "train"))
    mk_dir_not_exist(os.path.join(output_path, "vectors", "valid"))
    mk_dir_not_exist(os.path.join(output_path, "vectors", "test"))

    #tks_seq_len = (7, 3)
    vts_seq_len = (15, 5)
    # Vectorize functions' arguments
    id_trans_func_param = lambda row: IdentifierSequence(w2v_token_model, row.arg_name, row.other_args,
                                                         row.func_name, None)
    token_trans_func_param = lambda row: TokenSequence(w2v_token_model, TOKEN_SEQ_LEN[0], TOKEN_SEQ_LEN[1],
                                                       row.arg_occur, None, None)

    # Identifiers
    logger.info("[arg][identifiers] Generating vectors")
    process_datapoints(train_param_df,
                       os.path.join(output_path, "vectors", "train"),
                       'identifiers_', 'param_train', id_trans_func_param)
    process_datapoints(valid_param_df,
                       os.path.join(output_path, "vectors", "valid"),
                       'identifiers_', 'param_valid', id_trans_func_param)
    process_datapoints(test_param_df,
                       os.path.join(output_path, "vectors", "test"),
                       'identifiers_', 'param_test', id_trans_func_param)

    # Tokens
    logger.info("[arg][code tokens] Generating vectors")
    process_datapoints(train_param_df,
                       os.path.join(output_path, "vectors", "train"),
                       'tokens_', 'param_train', token_trans_func_param)
    process_datapoints(valid_param_df,
                       os.path.join(output_path, "vectors", "valid"),
                       'tokens_', 'param_valid', token_trans_func_param)
    process_datapoints(test_param_df,
                       os.path.join(output_path, "vectors", "test"),
                       'tokens_', 'param_test', token_trans_func_param)

    # Vectorize functions' return types
    id_trans_func_ret = lambda row: IdentifierSequence(w2v_token_model, None, row.arg_names_str, row.name, None)
    token_trans_func_ret = lambda row: TokenSequence(w2v_token_model, TOKEN_SEQ_LEN[0], TOKEN_SEQ_LEN[1], None,
                                                     row.return_expr_str, None)

    # Identifiers
    logger.info("[ret][identifiers] Generating vectors")
    process_datapoints(train_return_df,
                       os.path.join(output_path, "vectors", "train"),
                       'identifiers_', 'ret_train', id_trans_func_ret)
    process_datapoints(valid_return_df,
                       os.path.join(output_path, "vectors", "valid"),
                       'identifiers_', 'ret_valid', id_trans_func_ret)
    process_datapoints(test_return_df,
                       os.path.join(output_path, "vectors", "test"),
                       'identifiers_', 'ret_test', id_trans_func_ret)

    # Tokens
    logger.info("[ret][code tokens] Generating vectors")
    process_datapoints(train_return_df,
                       os.path.join(output_path, "vectors", "train"),
                       'tokens_', 'ret_train', token_trans_func_ret)
    process_datapoints(valid_return_df,
                       os.path.join(output_path, "vectors", "valid"),
                       'tokens_', 'ret_valid', token_trans_func_ret)
    process_datapoints(test_return_df,
                       os.path.join(output_path, "vectors", "test"),
                       'tokens_', 'ret_test', token_trans_func_ret)

    # Vectorize variables types
    id_trans_func_var = lambda row: IdentifierSequence(w2v_token_model, None, None, None, row.var_name)
    token_trans_func_var = lambda row: TokenSequence(w2v_token_model, TOKEN_SEQ_LEN[0], TOKEN_SEQ_LEN[1], None,
                                                     None, row.var_occur)

    # Identifiers
    logger.info("[var][identifiers] Generating vectors")
    process_datapoints(train_var_df,
                       os.path.join(output_path, "vectors", "train"),
                       'identifiers_', 'var_train', id_trans_func_var)
    process_datapoints(valid_var_df,
                       os.path.join(output_path, "vectors", "valid"),
                       'identifiers_', 'var_valid', id_trans_func_var)
    process_datapoints(test_var_df,
                       os.path.join(output_path, "vectors", "test"),
                       'identifiers_', 'var_test', id_trans_func_var)

    # Tokens
    logger.info("[var][code tokens] Generating vectors")
    process_datapoints(train_var_df,
                       os.path.join(output_path, "vectors", "train"),
                       'tokens_', 'var_train', token_trans_func_var)
    process_datapoints(valid_var_df,
                       os.path.join(output_path, "vectors", "valid"),
                       'tokens_', 'var_valid', token_trans_func_var)
    process_datapoints(test_var_df,
                       os.path.join(output_path, "vectors", "test"),
                       'tokens_', 'var_test', token_trans_func_var)

    # Generate data points for visible type hints
    logger.info("[visible type hints] Generating vectors")
    gen_aval_types_datapoints(train_param_df, train_return_df, train_var_df,
                              'train', os.path.join(output_path, "vectors", "train"))
    gen_aval_types_datapoints(valid_param_df, valid_return_df, valid_var_df,
                              'valid', os.path.join(output_path, "vectors", "valid"))
    gen_aval_types_datapoints(test_param_df, test_return_df, test_var_df,
                              'test', os.path.join(output_path, "vectors", "test"))

    # a flattened vector for labels
    logger.info("[true labels] Generating vectors")
    gen_labels_vector(train_param_df, train_return_df, train_var_df,
                      'train', os.path.join(output_path, "vectors", "train"))
    gen_labels_vector(valid_param_df, valid_return_df, valid_var_df,
                      'valid', os.path.join(output_path, "vectors", "valid"))
    gen_labels_vector(test_param_df, test_return_df, test_var_df,
                      'test', os.path.join(output_path, "vectors", "test"))


text:
"""
translated from matlab
https://github.com/omarshetta/Manuscript_Royal_Society/blob/master/utils/admm_algo_OP_on_graphs.m
by Omar Shetta
"""

import numpy as np
import numpy.linalg as la
import pandas as pd
import matplotlib.pyplot as plt
from scipy.sparse import issparse
from scipy.sparse.linalg import svds

def graph_reg_OP(X, lamb, gamma, phi):
    eta=0.000001
    p, n = X.shape
    L_k = np.random.randn(p,n)
    W_k = np.random.randn(p,n)
    S_k = np.random.randn(p,n)
    S_kmin1 = np.random.randn(p,n)
    W_kmin1 = np.random.randn(p,n)

    Z1_k = X - L_k - S_k
    Z2_k = W_k - L_k
    P1_k = nuclear_norm(L_k)
    P2_k = lamb*sum(np.sqrt(np.sum(S_k**2, axis=0)))
    P3_k =  gamma*np.trace(np.dot(L_k, np.dot(phi, L_k.T)))
    converged=False
    count=0
    r1_k = 1
    r2_k = 1

    maxiter=1000
    obj_func_kp1 = []
    while not converged:
        count=count+1;
        # print('GOP iteration =', count)
        H_1 = X - S_k + (Z1_k/r1_k)
        H_2 = W_k + (Z2_k/r2_k)
        A = (r1_k * H_1 + r2_k * H_2) / (r1_k + r2_k)
        r_k = (r1_k + r2_k)/2

        L_kp1 = prox_nuc_norm(A, 1/r_k)
        S_kp1 = column_thresh((X - L_kp1 + (Z1_k/r1_k)), lamb/r1_k)

        #matrix inversion causes slowdown. Tweak
        W_kp1 = GOP_tweak(gamma, phi, r2_k, n, L_kp1, Z2_k)

        Z1_kp1 = Z1_k + r1_k * (X - L_kp1 - S_kp1)
        Z2_kp1 = Z2_k + r2_k * (W_kp1 - L_kp1)

        P1_kp1 = nuclear_norm(L_kp1)
        P2_kp1 = lamb*sum(np.sqrt(sum(S_kp1**2)))
        P3_kp1 = gamma*np.trace(np.dot(L_kp1, np.dot(phi, L_kp1.T)))

        obj_func_kp1.append(P1_kp1 + P2_kp1 + P3_kp1)

        # % checking convergence
        too_small = 1e-15
        if P1_k >too_small: #preventing division by zero error
            rel_err_1 = (P1_kp1 - P1_k)**2 / P1_k**2
        else:
            rel_err_1 = 0
        if P2_k >too_small:
            rel_err_2 = (P2_kp1 - P2_k)**2 / P2_k**2
        else:
            rel_err_2 = 0
        if P3_k >too_small:
            rel_err_3 = (P3_kp1 - P3_k)**2 / P3_k**2
        else:
            rel_err_3 =0
        if la.norm(Z1_kp1 -Z1_k,'fro') >too_small:
            rel_err_z1 = la.norm(Z1_kp1 - Z1_k,'fro')**2 / (la.norm(Z1_k,'fro')**2)
        else:
            rel_err_z1 = 0
        if la.norm(Z2_kp1 -Z2_k,'fro') >too_small:
            rel_err_z2 = la.norm(Z2_kp1 - Z2_k,'fro')**2 / (la.norm(Z2_k,'fro')**2)
        else:
            rel_err_z2 = 0
        within_tol = rel_err_1<eta and rel_err_2<eta and rel_err_3<eta and rel_err_z1<eta and rel_err_z2<eta
        if within_tol or count==maxiter:
            converged=True
        else:
            # % get ready for new iteration
            S_k = S_kp1
            W_k = W_kp1
            Z1_k = Z1_kp1
            Z2_k = Z2_kp1
            P1_k = P1_kp1
            P2_k = P2_kp1
            P3_k = P3_kp1

    L_hat = L_kp1
    S_hat = S_kp1
    # print('GOP converged in {} iterations'.format(count))
    return L_hat, S_hat, obj_func_kp1

def GOP_tweak(gamma, phi, r2_k, n, L_kp1, Z2_k):
    A = (gamma * phi + (r2_k* np.identity(n)) ).T
    B = (L_kp1 - (Z2_k/r2_k)).T
    XT = la.solve(A, B)
    return XT.T

def prox_nuc_norm(X, eps):

    U,S,V=la.svd(X, full_matrices=False)
    n1 = len(S) #in python S is vector
    # Diagonal soft thresholding
    for i in range(n1):
        if abs(S[i]) <= eps:
            S[i]=0
        else:
            S[i]=S[i]-eps*np.sign(S[i])

    s_mat=np.diag(S)
    X=np.dot(U,np.dot(s_mat,V))
    return X

def column_thresh(C, eps):
    n1 = C.shape[1]
    for i in range(n1):
        if la.norm(C[:,i], 2) < eps: # norm here defaults to 2 norm for vector
            C[:,i]=0
        else:
            C[:,i]=C[:,i]-eps*C[:,i]/la.norm(C[:,i],2)
    return C

def nuclear_norm(x):
    """
    sum of singular values
    Nuclear norm of x
       Usage: norm_nuclear(x)
       Input parameters
           x       : a matrix
       Output parameters
          n       : nuclear norm of x
    """
    if issparse(x):
        u, s, vt = svds(x, np.min(size(x)))
    else:
        u, s, vt = la.svd(x)
    return np.sum(s)

def knn_graph(X,k):
    """
    from matlab version:
    This function computes the K-Nearest Neighbour graph.
    Inputs:
    X, is the data matrix with dimension (n x m) n is the number of samples and m is the number of features.
    k, is a scalar that defines the number of nearest neighbours.

    Outputs:
    Lap, is the graph Laplacian matrix of the k-nearest neighbour graph. It is an (n x n) symmetric positive semi-definite matrix.
    W, is the Weight matrix. It is an (n x n) symmetric matrix. It holds the weight of each edge of the k-nearest neighbour graph.
    """
    K = k+1 #as don't want to include self in neighbours
    n,p = X.shape
    # print('Building KNN graph for', n, 'samples.')
    W = np.zeros((n,n))
    D = np.zeros((n,n))
    dists = np.zeros((n,n))
    for i in range(n):
        for j in range(n):
            d = la.norm(X[i,:] - X[j,:],2)
            dists[i,j] = d

        inds = np.argpartition(dists[i,:], K)
        dists[i,inds[K:]] = 0

    # calculate weights for nonzero values
    nz = np.nonzero(dists)
    const = (np.sum(dists[:])/(K*n))**2
    W[nz] = np.exp(-(dists[nz]**2)/const)
    # symmetrise
    W = (W+W.T)/2
    # calculate laplacian matrix
    d = np.sum(W, axis=1)
    D = np.diag(d)
    laplacian = np.array(D - W)
    return laplacian, W

def GOP(M, lamb, gamma):
    k = min(M.shape[1]-2, 5)
    phi, W = knn_graph(M.T, k)
    L_hat, S_hat, ob_fn = graph_reg_OP(M, lamb, gamma, phi)
    return S_hat

def get_GOP_os(X, lamb=0.5, alpha=0.1):
    """
    to tie in to outlier detection object
    """
    M = X.T
    S_hat = GOP(M, lamb, alpha)
    return np.sum(S_hat, axis=0)

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# -----------------------------------------------------------------------------
# Copyright (C)
#   Software Competence Center Hagenberg GmbH (SCCH)
#   Institute of Computational Perception (CP) @ JKU Linz
# All rights reserved.
# -----------------------------------------------------------------------------
# This document contains proprietary information belonging to SCCH.
# Passing on and copying of this document, use and communication of its
# contents is not permitted without prior written authorization.
# -----------------------------------------------------------------------------
# Created on : 17.09.2018 $
# by : fischer $
# Developers: Hamid Eghbalzadeh, Lukas Fischer

# --- imports -----------------------------------------------------------------
import numpy as np

from network.NetRunner import NetRunner
from utils.metrics import get_sklearn_metrics, write_metrics_to_txt

class RawCopyRunner(NetRunner):
    """
    DeepSNP RawCopyRunner class. Base class for Rawcopy prediction evaluation.
    """
    def __init__(self, args=None, experiment_id=None):
        """
        Initialize RawCopyRunner
        :param args:            configuration parameters
        :param experiment_id:   string identifying the experiment to run
        """
        super().__init__(args, experiment_id)

    def evaluate(self):
        """
        Evaluate Rawcopy predictions
        :return:
        """

        print('... starting rawcopy evaluation ...')

        n_fold = 0
        for train_index, test_index in self.kfold.split(self.kf_X):
            for margin in self.margins:

                y_test, y_pred = self.load_test_rc_data(test_index, margin)

                y_pred = np.argmax(y_pred, 1).astype(np.int32)
                y_test = np.argmax(y_test, 1).astype(np.int32)

                # compute sklearn metrics
                f1_micro_sk, f1_macro_sk, f1_bin_sk, precision_micro_sk, precision_macro_sk, precision_bin_sk, \
                recall_micro_sk, recall_macro_sk, recall_bin_sk, acc_sk = get_sklearn_metrics(y_test, y_pred)

                # save metrics to txt file
                print('WHOLE SEQ TEST FOLD {}:'.format(n_fold))
                print('WINSIZE: {}, MARGIN: {}'.format(margin * 4, margin))
                print('Acc SK: {} '.format(acc_sk))

                print('F1 Micro SK: {} '.format(f1_micro_sk))
                print('F1 Macro SK: {} '.format(f1_macro_sk))
                print('F1 Bin SK: {} '.format(f1_bin_sk))

                print('Precision Micro SK: {} '.format(precision_micro_sk))
                print('Precision Macro SK: {} '.format(precision_macro_sk))
                print('Precision Bin SK: {} '.format(precision_bin_sk))

                print('Recall Micro SK: {} '.format(recall_micro_sk))
                print('Recall Macro SK: {} '.format(recall_macro_sk))
                print('Recall Bin SK: {} '.format(recall_bin_sk))

            n_fold += 1

max_stars_repo_name: eghbalz/deepsnp

max_issues_repo_name: SCCH-KVS/deepsnp

max_forks_repo_name: SCCH-KVS/deepsnp