text:
r"""Matrix.

Usage:
   dasem.matrix save-wikipedia-doc-term-matrix [options] <filename>
   dasem.matrix save-wikipedia-tdidf-doc-term-matrix [options] <filename>
   dasem.matrix save-wikipedia-doc-term-matrix-nmf-factorization \
       [options] <filename>

Options:
  --output-filename    Filename to write to
  -h --help            Help message
  --max-iter=<int>     Maximum number of iterations [default: 200]
  --max-n-pages=<int>  Maximum number of pages
  -v --verbose         Verbose debug messaging

Examples:
   python -m dasem.matrix save-wikipedia-doc-term-matrix \
       --max-n-pages=10 tmp.mtx

"""

from math import ceil, sqrt

from os.path import splitext

import json

from scipy import io

from sklearn.decomposition import NMF
from sklearn.feature_extraction.text import TfidfTransformer

from .wikipedia import XmlDumpFile

class Matrix(object):
    """Matrix."""

    def __init__(self, matrix, rows=None, columns=None):
        """Set up matrix values and row and column annotation.

        Parameters
        ----------
        matrix : matrix from scipy or numpy
            Values
        rows : list of str
            Row annotation.
        columns : list of str
            Column annotation.

        """
        self.matrix = matrix
        self.rows = rows
        self.columns = columns

    def __str__(self):
        """Return string representation."""
        return "<Matrix({}x{})>".format(*self.matrix.shape)

    __repr__ = __str__

    @property
    def shape(self):
        """Return shape of matrix."""
        return self.matrix.shape

    def to_csr(self, copy=False):
        """Convert to CSR sparse matrix.

        Parameters
        ----------
        copy : bool
            Copy the matrix

        """
        if copy:
            # TODO copy lists and dicts
            matrix = Matrix(self.matrix.tocsr(copy=True),
                            rows=self.rows,
                            columns=self.columns)
            return matrix
        else:
            self.matrix = self.matrix.tocsr()
            return self

    def save(self, filename):
        """Save matrix to a Matrix Market file.

        Parameters
        ----------
        filename : str
            Output filename.

        """
        comment = json.dumps({'rows': self.rows,
                              'columns': self.columns})
        io.mmwrite(filename, self.matrix, comment=comment)

    def nmf(self, n_components=None, max_iter=200):
        """Compute the non-negative matrix factorization.

        Parameters
        ----------
        n_components : int or None
            Number of components.

        Returns
        -------
        W : Matrix
            Left factorization.
        H : Matrix
            Right factorization.

        """
        if n_components is None:
            n_components = int(ceil(sqrt(min(self.matrix.shape) / 2)))

        # Prepare the matrix for fast computations
        self.to_csr(copy=False)

        # Compute factorization
        factorizer = NMF(n_components=n_components, max_iter=max_iter)
        W_values = factorizer.fit_transform(self.matrix)

        # Prepare output
        components = ['Component {}'.format(n)
                      for n in range(1, W_values.shape[1]+1)]
        W = Matrix(W_values, rows=self.rows, columns=components)
        H = Matrix(factorizer.components_,
                   rows=components, columns=self.columns)
        return W, H

    def tfidf(self, copy=False):
        """Return tfidf-transformed matrix."""
        transformer = TfidfTransformer()
        if copy:
            matrix = Matrix(transformer.fit_transform(self.matrix),
                            rows=self.rows,
                            columns=self.columns)
            return matrix
        else:
            self.matrix = transformer.fit_transform(self.matrix)
            return self

def load_matrix(filename):
    """Load matrix from Matrix market file.

    Parameters
    ----------
    filename : str
        Input filename.

    """
    matrix_values = io.mmread(filename)
    with open(filename) as f:
        line = f.readline()  # Ignore first line
        line = f.readline()
    data = json.loads(line[1:])
    rows = data['rows']
    columns = data['columns']

    matrix = Matrix(matrix_values, rows, columns)
    return matrix

def main():
    """Handle command-line interface."""
    from docopt import docopt

    arguments = docopt(__doc__)
    filename = arguments['<filename>']
    max_iter = int(arguments['--max-iter'])
    if arguments['--max-n-pages'] is None:
        max_n_pages = None
    else:
        max_n_pages = int(arguments['--max-n-pages'])
    verbose = verbose = arguments['--verbose']

    if arguments['save-wikipedia-doc-term-matrix']:
        dump_file = XmlDumpFile()
        matrix_values, rows, columns = dump_file.doc_term_matrix(
            max_n_pages=max_n_pages,
            verbose=verbose)
        matrix = Matrix(matrix_values, rows, columns)
        matrix.save(filename=filename)

    if arguments['save-wikipedia-tdidf-doc-term-matrix']:
        dump_file = XmlDumpFile()
        matrix_values, rows, columns = dump_file.doc_term_matrix(
            max_n_pages=max_n_pages,
            verbose=verbose)
        matrix = Matrix(matrix_values, rows, columns)
        matrix.tdidf()
        matrix.save(filename=filename)

    elif arguments['save-wikipedia-doc-term-matrix-nmf-factorization']:
        dump_file = XmlDumpFile()
        matrix_values, rows, columns = dump_file.doc_term_matrix(
            max_n_pages=max_n_pages,
            verbose=verbose)
        X = Matrix(matrix_values, rows, columns)
        if verbose:
            print('Matrix factorization on matrix sized {}x{}'.format(
                *X.shape))
        W, H = X.nmf(n_components=10, max_iter=max_iter)
        filename_base, filename_ext = splitext(filename)
        W.save(filename=filename_base + '-W' + filename_ext)
        H.save(filename=filename_base + '-H' + filename_ext)

if __name__ == '__main__':
    main()

reason: from scipy


text:
# To set seed random number in order to get reproducable results in keras
from numpy.random import seed
seed(4)
import tensorflow
tensorflow.random.set_seed(1234)
########################################
import pandas as pd
from pandas import *
import numpy as np
import random
from sklearn import svm
classifier =svm.SVC(gamma='scale',C=1,probability=True)
import plot_learning_curves as plc
from sklearn.preprocessing import MinMaxScaler #For feature normalization
scaler = MinMaxScaler()

df1 = pd.read_csv("Ps_Modification_coors_hek_complete.txt",sep=' ',skiprows=(0),header=(0))#Pseu_Modification_coors_ns_hek.txt
df2 = pd.read_csv("hek-reads-ref.eventalign.txt",sep='\t',skiprows=(0),header=(0))
#df2 = pd.read_csv("xaa.txt",sep='\t',skiprows=(0),header=(0))

print(df2.shape)
print("&&&&&&&&")
print(df1.head())
print("***********************")
print(df2.head())
print("######################")

#get the 2nd column of
#column_2 = df1.iloc[:, 1]

#print(df1['position'].iloc[0:5])
print(df1.iloc[0:5, 1])
print("@@@@@@@@@@@@@@@")
#print(df2['position'].iloc[0:5])
#print(df2.iloc[0:5, 1])
print(df2.iloc[0:5, 9])
print("######################")
model_kmer_list=list(df2.iloc[:, 9]) #10 for model-kmer that
print("333333333333333333", type(model_kmer_list))
print(model_kmer_list[5])
print(model_kmer_list[5][2])
U_kmer_list=[]
for i in model_kmer_list:
    #print(i)
    if i[2]=='T':
        U_kmer_list.append(i)

print("length of U_kmer_list",len(U_kmer_list))
print(U_kmer_list[0:50])

df=df2[df2['model_kmer'].isin(U_kmer_list)]
print(df.shape)
print(df.head())

np.savetxt('filtered_df_U_kmer', df,fmt='%s')

#label the data
x=list(set(df1.iloc[:,1]).intersection(set(df.iloc[:,1])))
print("length of intersection list",len(x))

df_pseu=df[df['position'].isin(x)]
listofones = [1] * len(df_pseu.index)
# Using DataFrame.insert() to add a column
df_pseu.insert(13, "label", listofones, True)
df_U=df[~df['position'].isin(x)]
listofzeros=[0]*len(df_U.index)
df_U.insert(13, "label", listofzeros, True)
print(df_pseu.shape)
print(df_pseu.head())
print(df_U.shape)
print(df_U.head())
#np.savetxt('pseu_samples.txt', df_pseu,fmt='%s')
#np.savetxt('U_samples.txt', df_U,fmt='%s')
##########prepare datast
df_U = df_U.sample(n=len(df_pseu), replace=False) #try replace=false

# Create DataFrame from positive and negative examples
dataset = df_U.append(df_pseu, ignore_index=True)
dataset['label'] = dataset['label'].astype('category')

# #scale training and testing data
columns=['event_level_mean','event_stdv','event_length']
#columns=['event_level_mean']
#columns=['event_stdv']
#columns=['event_length']

#X = dataset[:13]
X = dataset[columns]
#insert onehot encoding of reference-kmer
Onehot=pd.get_dummies(dataset['reference_kmer'], prefix='reference_kmer')
X= pd.concat([X,Onehot],axis=1)
#X= Onehot
print("#############",X.shape)
print(X.head())
#scale training data
X= scaler.fit_transform(X)
Y = dataset['label']
print(",,,,,,,,",X.shape)
from sklearn.model_selection import train_test_split

train, test = train_test_split(df, test_size=0.2)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=0)
#X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.3) for unblanced dataset

#clf = classifier.fit(X_train,y_train)
clf = classifier.fit(X_train,y_train.ravel())

y_pred = classifier.predict(X_test)
y_prob = classifier.predict_proba(X_test)
y_prob = y_prob[:,1]
# Evaluate the model: Model Accuracy, how often is the classifier correct
from sklearn import metrics #Import scikit-learn metrics module for accuracy calculation
from sklearn.metrics import classification_report #for classifier evaluation
from sklearn.metrics import roc_curve, auc
from sklearn.metrics import roc_auc_score # for printing AUC
from sklearn.metrics import confusion_matrix

print("Accuracy:",metrics.accuracy_score(y_test, y_pred)*100)

print(classification_report(y_test, y_pred))
auc=roc_auc_score(y_test.round(),y_pred)
auc = float("{0:.3f}".format(auc))
print("AUC=",auc)
#true negatives c00, false negatives C10, true positives C11, and false positives C01
#tn c00, fpC01, fnC10, tpC11
print('CF=',confusion_matrix(y_test, y_pred))
l=confusion_matrix(y_test, y_pred)#https://towardsdatascience.com/accuracy-precision-recall-or-f1-331fb37c5cb9
print('TN=',l.item((0, 0)))
print('FP=',l.item((0, 1)))
print('FN=',l.item((1, 0)))
print('TP=',l.item((1, 1)))
#print(type(X_train), type(y_train))

#plot learning curve: works with all classifier and all features except x(padded signal) as it leads to error with SVM
#References:https://medium.com/@datalesdatales/why-you-should-be-plotting-learning-curves-in-your-next-machine-learning-project-221bae60c53
import matplotlib.pyplot as plt

plc. plot_learning_curves(classifier, X_train, y_train, X_test, y_test)

# Create plot
#plt.title("Learning Curve")
plt.xlabel("Training Set Size"), plt.ylabel("Accuracy Score"), plt.legend(loc="best")
plt.tight_layout()
#plt.show()
plt.savefig('SVM_onehot_LC.png', dpi=300)
#plt.savefig('SVM_onehot_LC.svg')
plt.close()

#plot ROC curve: https://stackoverflow.com/questions/25009284/how-to-plot-roc-curve-in-python
from sklearn import metrics
import numpy as np
import matplotlib.pyplot as plt

fpr, tpr, thresholds = metrics.roc_curve(y_test, y_prob)

# Print ROC curve
plt.plot(fpr,tpr)
#plt.title("ROC Curve")
# axis labels
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
#plt.show()
plt.savefig('SVM_onehot_ROC.png', dpi=300)
#plt.savefig('SVM_onehot_ROC.svg')

plt.close()

#############################################
#old code to plot learning curve: works only with RandomForest
#Reference: https://www.dataquest.io/blog/learning-curves-machine-learning/
##################

ext: py

max_stars_repo_name: Janga-Lab/Penguin

max_issues_repo_name: Janga-Lab/Penguin

max_forks_repo_name: Janga-Lab/Penguin

reason: import numpy,from numpy


text:
# -*- coding: utf-8 -*-
"""Desafio_CyberLabs.ipynb

Automatically generated by Colaboratory.

Original file is located at https://colab.research.google.com/drive/1o5txnZiNYXfwjhikOJQQsIe5aRbsbwcy

# Classificador para os números 0 e 5 do dataset MNIST

Ao desenvolver um classificador, há duas estratégias a serem seguidas:
Criar um modelo capaz de identificar especificamente as classes desejadas ou criar um modelo capaz de identificar todos os outros cenários, emitindo um alerta para aqueles que fogem dessa regra. A melhor estratégia varia com o cenário.

Nesse caso, é mais inteligente treinar o modelo para aprender a classificar os números 0 e 5, ao invés de ensina-lo a identificar os números 1,2,3,4,6,7,8,9.

Para isso, usaremos algumas bibliotecas de aprendizado profundo e visão computacional, como numpy, matplotlib, sklearn, keras e tensorflow.
"""

import numpy as np
import keras
import matplotlib
from matplotlib import pyplot as plt
import tensorflow as tf
from sklearn.model_selection import train_test_split, KFold
from tensorflow import keras
from tensorflow.keras import layers
import random

# Carregando o conjunto de dados completo do MNIST
mnist = tf.keras.datasets.mnist
(X_all_train, Y_all_train), (X_all_test, Y_all_test) = mnist.load_data()

""" Na forma atual, o conjunto de dados está preparado para um modelo de classificação
multiclasse, ou seja, um modelo que seja capaz de identificar cada dígito separadamente.
Para moldar o conjunto de dados para esse desafio, criaremos um modelo de classificação binária:
Ou o digito é igual a 5 ou 0 --> tendo rótulo igual a 1
Ou o digito é diferente disso --> tendo rótulo igual a 0"""

# Modulando os rótulos do conjunto de treino
for x in range (Y_all_train.shape[0]):
  if (Y_all_train[x]!= 0 and Y_all_train[x]!= 5):
    Y_all_train[x]=0
  else:
    Y_all_train[x]=1

# Modulando os rótulos do conjunto de teste
for x in range (Y_all_test.shape[0]):
  if (Y_all_test[x]!= 0 and Y_all_test[x]!= 5):
    Y_all_test[x]=0
  else:
    Y_all_test[x]=1

# Visualizando nossos novos rótulos -> nao deve conter outro número além de 0 e 1
print(Y_all_train[:10])
print(Y_all_test[:10])

"""Como estamos utilizando um conjunto de dados previamente tratado, não precisamos nos preocupar com outliers ou imagens deformadas. Entretanto, é uma boa tática visualizar algumas amostras do conjunto para facilitar a criação do modelo, particionando corretamente o conjunto de dados de acordo com a quantidade de amostras nele presente.
O conjunto de dados está divido em aproximadamente 85% para treino e 15% para teste.
"""

# Analisando o conjunto de dados para treino
print (X_all_train.shape)
print (X_all_test.shape)

# Mostrando as duas primeiras imagens do conjunto de treino
imagem = plt.figure(figsize=(5, 5))
imagem.add_subplot(2, 2, 1)
plt.imshow(X_all_train[0],cmap='gray')
plt.axis('on')
imagem.add_subplot(2, 2, 2)
plt.imshow(X_all_train[1],cmap='gray')
plt.axis('on')

"""Logo, podemos notar que temos 60000 imagens para treino e outras 10000 para teste, cada uma com 28x28 pixels

Antes de iniciarmos a criação do modelo, podemos dar mais alguns ajustes do conjunto de dados.
Uma boa tática é realizar a normalização dos dados de treino, o que pode agilizar o tempo de aprendizado do modelo, convergindo para um modelo ideal mais rapidamente. Além disso, a normalização garante que, previamente, nenhuma característica tenha um peso maior que a outra, já que é o trabalho do modelo descobrir isso.
"""

# Normalizando o conjunto de treino com a normalização Min-Max
X_all_train = X_all_train.astype('float32')
X_all_train /= 255

"""Buscando um modelo mais robusto e confiável, vamos usar a técnica de validação cruzada k-fold, que consiste em dividir o conjunto de dados em k partes, treinando um modelo em cada um deles, com exceção de um que será usado para avaliar o modelo. Esse procedimento é realizado k vezes, modificando o conjunto da avaliação. Assim, diminuimos as chances de haver sobreajuste do modelo quanto ao conjunto de dados utilizado para treino.

Para isso, iremos reorganizar o conjunto de dados, unindo os conjuntos de treino e teste para separalá-los posteriormente numa proporção de 70% - 30%.
"""

# Unindo os dados
X_all_train = X_all_train.reshape(X_all_train.shape[0],28,28,1)
X_all_test = X_all_test.reshape(X_all_test.shape[0],28,28,1)
data=np.concatenate((X_all_train, X_all_test), axis=0)
labels=np.concatenate((Y_all_train, Y_all_test), axis=0)

# Iniciando a validação cruzada com k = 5

x_train, x_test, y_train, y_test = train_test_split(data, labels, test_size=0.3, random_state=182)
kfold = KFold(n_splits=5, shuffle=True, random_state=182)

# Listas para armazenar as métricas para cada fold
acc = []
prec = []
rec = []
auc = []

contador = 0;

for train, test in kfold.split(data, labels):
  contador+=1
  x_train, x_test = data[train], data[test]
  y_train, y_test = labels[train], labels[test]

  """Aqui começamos a definir o modelo a ser criado. Dentre os algoritmos de visão computacional, a Rede Neural
  Convolucional (CNN - Convolutional Neural Network) apresenta ótimos resultados na análise de imagens. Por ser
  um modelo mais robusto, essa rede pode apresentar um alto custo computacional se analisar imagens grandes e
  ter uma estrutura profunda, isso é, muitas camadas escondidas.
  Como a tarefa atual é simples, não é necessário usar uma rede profunda, como os modelos Inception da Google,
  vamos desenvolver nossa própria rede para melhor se adequar à tarefa."""

  model = keras.Sequential([
  layers.Input(shape=(28, 28,1)),

  layers.Conv2D(16, 5, activation='relu',padding='valid'), # 128

  layers.Conv2D(16,5, activation='tanh',padding='valid'), #1024

  layers.MaxPooling2D(pool_size=(2, 2)), # 12

  layers.Conv2D(32,3, activation='relu',padding='valid'), #

  layers.Conv2D(64,3, activation='relu',padding='valid'), # 4

  layers.Conv2D(64,3, activation='tanh',padding='same'), # 256

  layers.Dense(units=16, activation='relu'),

  layers.Dropout(rate=0.4),

  layers.Flatten(),

  layers.Dense(1, activation='sigmoid')])

  #Definindo as métricas de interesse
  METRICS = [keras.metrics.TruePositives(name='tp'), keras.metrics.FalsePositives(name='fp'), keras.metrics.TrueNegatives(name='tn'), keras.metrics.FalseNegatives(name='fn'), keras.metrics.BinaryAccuracy(name='accuracy'), keras.metrics.Precision(name='precision'), keras.metrics.Recall(name='recall'), keras.metrics.AUC(name='auc')]

  #Aqui usamos o otimizador Adam, um método de descida gradiente estocástico adaptativo. A entropia cruzada binária foi usada como função de perda, já que trasformei a classificação multiclasse em uma classificação binária
  model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss = 'binary_crossentropy', metrics=METRICS)
  model.build()
  model.summary() # aprensentando uma visão geral do modelo criado

  #EarlyStopping - caso a perda não diminua em 3 épocas seguidas, o treinamento é finalizado
  callbacks = [tf.keras.callbacks.EarlyStopping(monitor='loss', mode='min', patience=3, restore_best_weights=True)]

  #Treinando o modelo em 5 épocas, com tamanho do batch igual a 128
  history = model.fit(x_train, y_train, epochs=5, batch_size=128, callbacks=callbacks)

  print("Avaliando o "+ str(contador)+"° modelo: \n")
  results = model.evaluate(x_test, y_test)
  print("[loss, tp, fp, tn, fn, accuracy, precision, recall, auc]: " + str(results))
  acc.append(results[5])
  prec.append(results[6])
  rec.append(results[7])
  auc.append(results[8])

#Apresentando os resultados encontrados
print("Resultados para acurácia: (%f +/- %f)\n" % (np.mean(acc), np.std(acc)))
print("Resultados para precisão: (%f +/- %f)\n" % (np.mean(prec), np.std(prec)))
print("Resultados para sensibilidade: (%f +/- %f)\n" % (np.mean(rec), np.std(rec)))
print("Resultados para auc: (%f +/- %f)\n" % (np.mean(auc), np.std(auc)))

"""Podemos testar o modelo com o método predict para confirmar que nenhuma confusão foi feita durante o processo."""

# Visualizando algumas imagens

a = random.randint(500,510)
b = random.randint(510,520)

imagem = plt.figure(figsize=(7, 7))
x_test = x_test.reshape(x_test.shape[0],28,28)
figure,image = plt.subplots(1,b-a, figsize=(20,20))

contador = 0
for i in range(a,b):
  image[contador].imshow(x_test[i].reshape((28,28)),cmap='gray')
  image[contador].axis('off')
  image[contador].set_title(str(i) + "° - " + str(y_test[i]) + " ")
  contador+=1

# Usando o método predict para confirmar os resultados obtidos
x_test = x_test.reshape(x_test.shape[0],28,28,1)
y_pred = model.predict(x_test)
y_pred = np.round(y_pred).astype(int)

# Se esses números corresponderem ao número das imagens com 0 ou 5, nosso modelo está funcionando corretamente
for i in range(a,b):
  if y_test[i] == 1:
    print(i)

# Checando se todas as imagens nesse intervalo tiveram uma predição correta
for i in range(a,b):
  print(y_pred[i], end="")

"""Assim, podemos concluir que o modelo está funcionando corretamente, obtendo ótimos resultados.
Em cenário diferente, onde os resultados poderiam estar ruins ou inconclusivos, ainda teriamos outras atividades a realizar: Utilizar uma normalização que melhor atendesse o conjunto de dados, modificar o número de épocas, o tamanho do batch e o passo de aprendizagem, além de realizar a otimização de hiperparâmetros na rede, escolhendo a quantidade de filtros, as funções de ativação e número de camadas, por exemplo, que melhor satisfaz a classificação dos números.
"""

reason: import numpy


text:
import numpy as np
from scipy.sparse import csc_matrix
from scipy.sparse.linalg import splu
import cv2
import sys

def getColorExact(localExtrema, ntscIm):
    [n,m,d] = ntscIm.shape
    imgSize = n*m
    nI = np.zeros(ntscIm.shape, dtype=ntscIm.dtype)
    nI[...,0] = ntscIm[...,0]
    indsM = np.arange(imgSize).reshape(n, m)

    wd = 1
    length = 0
    consts_len = 0
    col_inds = np.zeros(imgSize*(2*wd+1)**2, dtype=int)
    row_inds = np.zeros(imgSize*(2*wd+1)**2, dtype=int)
    vals = np.zeros(imgSize*(2*wd+1)**2)
    gvals = np.zeros((2*wd+1)**2)

    for i in range(n):
        for j in range(m):
            if not localExtrema[i,j]:
                tlen = 0
                for ii in range(max(0,i-wd),min(i+wd+1,n)):
                    for jj in range(max(0,j-wd),min(j+wd+1,m)):
                        if(ii != i) or (jj != j):
                            row_inds[length] = consts_len
                            col_inds[length] = indsM[ii,jj]
                            gvals[tlen] = ntscIm[ii,jj,0]
                            length  = length +1
                            tlen = tlen+1
                t_val = ntscIm[i,j,0] # center pixel Y value
                gvals[tlen] = t_val
                c_var = np.mean((gvals[:tlen+1] - np.mean(gvals[:tlen+1]) )**2)
                csig=c_var*0.6
                mgv = min((gvals[:tlen]-t_val)**2)
                if csig < (-mgv/np.log(0.01)):
                    csig = -mgv/np.log(0.01)
                if csig < 0.000002:
                    csig = 0.000002
                gvals[:tlen] = np.exp(-(gvals[:tlen]-t_val)**2/csig)
                gvals[:tlen] = gvals[:tlen]/sum(gvals[:tlen])
                vals[length-tlen:length] = -gvals[:tlen]

            row_inds[length] = consts_len
            col_inds[length] = indsM[i,j]
            vals[length]=1
            length = length+1
            consts_len = consts_len+1

    vals = vals[:length]
    col_inds = col_inds[:length]
    row_inds = row_inds[:length]

    A_csc = csc_matrix((vals, (row_inds, col_inds)), shape=(consts_len, imgSize))
    LU = splu(A_csc)
    b = np.zeros(A_csc.shape[0],dtype=ntscIm.dtype )
    for dim in range(1,d):
        curIm = ntscIm[:,:,dim]
        b[indsM[localExtrema != 0]] = curIm[localExtrema]
        new_vals = LU.solve(b)
        nI[...,dim] = new_vals.reshape((n,m))

    return nI

def EdgePreservingSmooth(I,k=3):
    """
    Implement "Edge-preserving Multiscale Image Decomposition based on Local Extrema"

    Parameters
    -----------
    I: input image( BGR image or grayscale image )
    k: kernel size, default = 3

    Returns
    -----------
    M: smoothed image( BGR image or grayscale image )
    localMax: local maxima extrema( boolean matrix )
    localMin: local minima extrema( boolean matrix )
    MaxEnvelope: extremal envelopes of maxima extrema( Y+ extremal envelopes at each BGR channel )
    MinEnvelope: extermal envelope of minima extrema( Y+ extremal envelopes at each BGR channel )

    """

    # wd: half width of kernel size
    wd = k//2

    if I.ndim == 3:
        channel = I.shape[2]
        YUV = cv2.cvtColor(I,cv2.COLOR_BGR2YUV)
        Y = np.double(YUV[:,:,0])/255
        image = np.double(I)/255
        #cv2.imshow("Y",Y)
        #cv2.waitKey(0)

    else:
        channel = 1
        Y = np.double(I)/255
    print("Extrema location")
    height,width = Y.shape
    localMax = np.zeros( Y.shape, dtype=bool)
    localMin = np.zeros( Y.shape, dtype=bool)
    for i in range(height):
        for j in range(width):
            center = Y[i,j]

            ii_start = max(0,i-wd)
            ii_end = min(i+wd+1,height)
            jj_start = max(0,j-wd)
            jj_end = min(j+wd+1,width)
            cover = Y[ii_start:ii_end,jj_start:jj_end]

            maxcount = np.sum(cover > center)
            mincount = np.sum(center > cover)
            if maxcount <= k-1:
                localMax[i,j] = True
            if mincount <= k-1:
                localMin[i,j] = True

    print("Extermal envelope construction")
    Y_BGR = np.zeros((height,width,4))
    Y_BGR[...,0] = Y;
    for i in range(channel):
        Y_BGR[...,i+1] = image[...,i]

    MaxEnvelope = getColorExact(localMax, Y_BGR)
    MinEnvelope = getColorExact(localMin, Y_BGR)

    print("Computation of the smoothed mean")
    M = (MaxEnvelope[:,:,1:(channel+1)] + MinEnvelope[:,:,1:(channel+1)])/2;
    M = (M*255).astype(np.uint8)
    #cv2.imshow("M",M)
    #cv2.waitKey(0)
    #cv2.destroyAllWindows()
    return M, localMax, localMin, MaxEnvelope, MinEnvelope

if __name__ == '__main__':

    if len(sys.argv) < 4:
        print('Usage:', sys.argv[0], '<ImagePath>', '<KernelSize>', '<Iteration>')
        sys.exit(1)

    imagepath = sys.argv[1]
    kernelsize = int(sys.argv[2])
    iteration = int(sys.argv[3])

    I = cv2.imread(imagepath)
    M= I.copy()
    for i in range(iteration):
        print('Iteration: ', str(i+1))
        M,localmax, localmin, maxenvelope, minenvelope = EdgePreservingSmooth(M,kernelsize)
        kernelsize += 4
        print('')

    I_YUV = cv2.cvtColor(I,cv2.COLOR_BGR2YUV)
    M_YUV = cv2.cvtColor(M,cv2.COLOR_BGR2YUV)
    D = I_YUV[:,:,0]-M_YUV[:,:,0]

    # Make the grey scale image have three channels
    grey_3_channel = cv2.cvtColor(D, cv2.COLOR_GRAY2BGR)
    numpy_horizontal = np.hstack(( I, M, grey_3_channel))
    cv2.imshow('Edge-preserving Smooth Result', numpy_horizontal)
    cv2.waitKey(0)
    cv2.destroyAllWindows()

reason: import numpy,from scipy


text:
"""Slicing label builder."""
import logging
from typing import Any, Callable, Dict, List

import numpy as np
import torch
from torch import Tensor

from emmental.contrib.slicing.slicing_function import slicing_function
from emmental.data import EmmentalDataLoader
from emmental.task import EmmentalTask

logger = logging.getLogger(__name__)

def add_slice_labels(
    task: EmmentalTask,
    dataloaders: List[EmmentalDataLoader],
    slice_func_dict: Dict[str, Callable],
    split: str = "train",
) -> Dict[str, Tensor]:
    """Extend dataloader by adding slice indicator and predictor labels.

    Args:
      task: Task to add slices.
      dataloaders: List of dataloaders to train on the task.
      slice_func_dict: Slicing functions.
      split: Split to use, defaults to "train".

    Returns:
      Slice data class distribution.
    """
    # Calculate class balance
    slice_distribution = {}

    # Add base slice if needed
    if "base" not in slice_func_dict.keys():
        slice_func_dict["base"] = base_slice

    for dataloader in dataloaders:
        labels = dataloader.dataset.Y_dict[  # type: ignore
            dataloader.task_to_label_dict[task.name]
        ]
        for slice_name, slice_func in slice_func_dict.items():
            indicators = slice_func(dataloader.dataset)
            slice_ind_name = f"{task.name}_slice:ind_{slice_name}"
            slice_pred_name = f"{task.name}_slice:pred_{slice_name}"

            pred_labels = indicators * labels
            ind_labels = indicators
            ind_labels[ind_labels == 0] = 2

            if dataloader.split == split and slice_name != "base":
                ind_classes, ind_counts = np.unique(
                    ind_labels.numpy(), return_counts=True
                )
                if ind_classes.shape[0] == 2:
                    slice_distribution[slice_ind_name] = torch.Tensor(
                        np.sum(ind_counts) / ind_counts / ind_classes.shape[0]
                    )
                pred_classes, pred_counts = np.unique(
                    pred_labels.numpy(), return_counts=True
                )
                if (pred_classes[0] == 0 and pred_classes.shape[0] == 3) or (
                    pred_classes[0] == 1 and pred_classes.shape[0] == 2
                ):
                    if pred_classes[0] == 0:
                        slice_distribution[slice_pred_name] = torch.Tensor(
                            1 - pred_counts[1:] / np.sum(pred_counts[1:])
                        )
                    else:
                        slice_distribution[slice_pred_name] = torch.Tensor(
                            1 - pred_counts / np.sum(pred_counts)
                        )

            # Update slice indicator and predictor labels
            dataloader.dataset.Y_dict.update(  # type: ignore
                {slice_ind_name: ind_labels, slice_pred_name: pred_labels}
            )
            # Update dataloader task_to_label_dict
            dataloader.task_to_label_dict.update(
                {slice_ind_name: slice_ind_name, slice_pred_name: slice_pred_name}
            )
        msg = (
            f"Loaded slice labels for task {task.name}, slice {slice_name}, "
            f"split {dataloader.split}."
        )
        logger.info(msg)

    return slice_distribution

@slicing_function()
def base_slice(example: Any) -> bool:
    """Slicing function which always to return True.

    Args:
      example: Sample to check if it's in the slice or not.

    Returns:
      True.
    """
    return True

max_stars_repo_name: KeAWang/emmental

max_issues_repo_name: KeAWang/emmental

max_forks_repo_name: KeAWang/emmental


text:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import sys
import os
import numpy as np
import mxnet as mx
from mxnet.test_utils import *
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
sys.path.insert(0, os.path.join(curr_path, '../unittest'))
from common import setup_module, with_seed
from mxnet.gluon import utils

def _get_model():
    if not os.path.exists('model/Inception-7-symbol.json'):
        download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz', dirname='model')
        os.system("cd model; tar -xf inception-v3.tar.gz --strip-components 1")

def _dump_images(shape):
    import skimage.io
    import skimage.transform
    img_list = []
    for img in sorted(os.listdir('data/test_images/')):
        img = skimage.io.imread('data/test_images/'+img)
        short_egde = min(img.shape[:2])
        yy = int((img.shape[0] - short_egde) / 2)
        xx = int((img.shape[1] - short_egde) / 2)
        img = img[yy : yy + short_egde, xx : xx + short_egde]
        img = skimage.transform.resize(img, shape)
        img_list.append(img)
    imgs = np.asarray(img_list, dtype=np.float32).transpose((0, 3, 1, 2)) - 128
    np.save('data/test_images_%d_%d.npy'%shape, imgs)

def _get_data(shape):
    hash_test_img = "355e15800642286e7fe607d87c38aeeab085b0cc"
    hash_inception_v3 = "91807dfdbd336eb3b265dd62c2408882462752b9"
    utils.download("http://data.mxnet.io/data/test_images_%d_%d.npy" % (shape),
                   path="data/test_images_%d_%d.npy" % (shape),
                   sha1_hash=hash_test_img)
    utils.download("http://data.mxnet.io/data/inception-v3-dump.npz",
                   path='data/inception-v3-dump.npz',
                   sha1_hash=hash_inception_v3)

@with_seed()
def test_consistency(dump=False):
    shape = (299, 299)
    _get_model()
    _get_data(shape)
    if dump:
        _dump_images(shape)
        gt = None
    else:
        gt = {n: mx.nd.array(a) for n, a in np.load('data/inception-v3-dump.npz').items()}
    data = np.load('data/test_images_%d_%d.npy'%shape)
    sym, arg_params, aux_params = mx.model.load_checkpoint('model/Inception-7', 1)
    arg_params['data'] = data
    arg_params['softmax_label'] = np.random.randint(low=1, high=1000, size=(data.shape[0],))
    ctx_list = [{'ctx': mx.gpu(0), 'data': data.shape, 'type_dict': {'data': data.dtype}},
                {'ctx': mx.cpu(0), 'data': data.shape, 'type_dict': {'data': data.dtype}}]
    gt = check_consistency(sym, ctx_list, arg_params=arg_params, aux_params=aux_params,
                           tol=1e-3, grad_req='null', raise_on_err=False, ground_truth=gt)
    if dump:
        np.savez('data/inception-v3-dump.npz', **{n: a.asnumpy() for n, a in gt.items()})

if __name__ == '__main__':
    test_consistency(False)

max_stars_repo_name: xudong-sun/mxnet


text:
import operator
from sympy.core.compatibility import reduce
from sympy.core.function import Function
from sympy.core import sympify, S, Integer

###############################################################################
###################### Kronecker Delta, Levi-Civita etc. ######################
###############################################################################

class Dij(Function):
    """
    Represents the Kronecker Delta Function

    if i == j, Dij(i, j) = 1
    otherwise Dij(i, j) = 0
    where i, j are usually integers
    """
    nargs = (1, 2)

    @classmethod
    def eval(cls, i, j=0):
        i, j = map(sympify, (i, j))
        if i == j:
            return S.One
        elif i.is_number and j.is_number:
            return S.Zero

def Eijk(*args, **kwargs):
    """
    Represent the Levi-Civita symbol.

    This is just compatibility wrapper to LeviCivita().
    """
    return LeviCivita(*args, **kwargs)

def prod(a):
    return reduce(operator.mul, a, 1)

def eval_levicivita(*args):
    """Evaluate Levi-Civita symbol."""
    from sympy import factorial
    n = len(args)
    return prod(
        prod(args[j] - args[i] for j in xrange(i + 1, n))
        / factorial(i) for i in xrange(n))
    # converting factorial(i) to int is slightly faster

class LeviCivita(Function):
    """Represent the Levi-Civita symbol.

    For even permutations of indices it returns 1, for odd permutations -1, and
    for everything else (a repeated index) it returns 0.

    Thus it represents an alternating pseudotensor.

    >>> from sympy import LeviCivita, symbols
    >>> LeviCivita(1,2,3)
    1
    >>> LeviCivita(1,3,2)
    -1
    >>> LeviCivita(1,2,2)
    0
    >>> i,j,k = symbols('i j k')
    >>> LeviCivita(i,j,k)
    LeviCivita(i, j, k)
    >>> LeviCivita(i,j,i)
    0
    """
    @classmethod
    def eval(cls, *args):
        if all(isinstance(a, (int, Integer)) for a in args):
            return eval_levicivita(*args)
        if len(set(args)) < len(args):
            return S.Zero

    def doit(self):
        return eval_levicivita(*self.args)

max_stars_repo_name: minrk/sympy

max_issues_repo_name: curzel-it/KiPyCalc

max_forks_repo_name: curzel-it/KiPyCalc

reason: from sympy


text:
# fine-tune on different dataset
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse
import os
import math
import time
import glob
from collections import defaultdict

import numpy as np
import torch
import torch.nn as nn
import torch.optim
import torch.utils.data
import torchvision
import torchvision.transforms as transforms
import torch.backends.cudnn as cudnn
from sklearn import metrics
from models.resnet import resnet50, resnet18
from PIL import Image
from PIL import ImageFile
ImageFile.LOAD_TRUNCATED_IMAGES = True

class AverageMeter(object):
	"""Computes and stores the average and current value"""
	def __init__(self):
		self.reset()

	def reset(self):
		self.val = 0
		self.avg = 0
		self.sum = 0
		self.count = 0

	def update(self, val, n=1):
		self.val = val
		self.sum += val * n
		self.count += n
		self.avg = self.sum / self.count

def pil_loader(path):
	# open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
	with open(path, 'rb') as f:
		img = Image.open(f)
		return img.convert('RGB')

def accimage_loader(path):
	try:
		return accimage.Image(path)
	except IOError:
		# Potentially a decoding problem, fall back to PIL.Image
		return pil_loader(path)

class Flatten(nn.Module):
	def __init__(self):
		super(Flatten, self).__init__()

	def forward(self, feat):
		return feat.view(feat.size(0), -1)

parser = argparse.ArgumentParser()
parser.add_argument('--vocdir', type=str, required=False,
					default=os.path.expanduser('/data/VOC/VOCdevkit/VOC2007/'), help='pascal voc 2007 dataset')
parser.add_argument('--split', type=str, required=False, default='trainval',
					choices=['train', 'trainval'], help='training split')

parser.add_argument('--pretrain_path', default='', type=str)
parser.add_argument('--model', default='resnet50', type=str)
parser.add_argument('--exp', default='', type=str)

parser.add_argument('--dropout', default=0, type=int)
parser.add_argument('--nit', type=int, default=19*100,
					help='Number of training iterations')
parser.add_argument('--batch_size', type=int, default=16,
					help='Number of Batch Size')
parser.add_argument('--fc6_8', type=int, default=0,
					help='If true, train only the final classifier')
parser.add_argument('--train_batchnorm', type=int, default=0,
					help='If true, train batch-norm layer parameters')
parser.add_argument('--eval_random_crops', type=int, default=1,
					help='If true, eval on 10 random crops, otherwise eval on 10 fixed crops')
parser.add_argument('--stepsize', type=int, default=19*40, help='Decay step')
parser.add_argument('--lr', type=float, required=False,
					default=3.0, help='learning rate')
parser.add_argument('--wd', type=float, required=False,
					default=1e-6, help='weight decay')
parser.add_argument('--seed', type=int, default=31, help='random seed')

os.system('ulimit -n 10000')

def main():
	args = parser.parse_args()
	print(args)

	# fix random seeds
	torch.manual_seed(args.seed)
	torch.cuda.manual_seed_all(args.seed)
	np.random.seed(args.seed)

	# create model and move it to gpu
	if args.model == 'resnet50':
		model = resnet50(non_linear_head=True)
		model = nn.DataParallel(model)
		classifier = nn.Linear(7*7*2048, 20).cuda()
		classifier = nn.Sequential(
			nn.AdaptiveAvgPool2d((2,2)),
			Flatten(),
			nn.Linear(2*2*2048, 20),
		).cuda()
	elif args.model == 'resnet18':
		model = resnet18()
		model = nn.DataParallel(model)
		classifier = nn.Sequential(
			nn.Linear(512, 20),
		).cuda()

	ckpt = torch.load(args.pretrain_path)
	model.load_state_dict(ckpt['state_dict'])

	model.eval()

	# model.cuda()
	cudnn.benchmark = True

	# what partition of the data to use
	if args.split == 'train':
		args.test = 'val'
	elif args.split == 'trainval':
		args.test = 'test'
	# data loader
	normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
									 std=[0.229, 0.224, 0.225])
	dataset = VOC2007_dataset(args.vocdir, split=args.split, transform=transforms.Compose([
		transforms.RandomHorizontalFlip(),
		transforms.RandomResizedCrop(224, scale=(0.2,1.0)),
		transforms.ToTensor(),
		normalize,
	]))

	loader = torch.utils.data.DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=24, pin_memory=False)

	print('PASCAL VOC 2007 ' + args.split + ' dataset loaded')

	if args.fc6_8:
	   # freeze some layers
		for param in model.parameters():
			param.requires_grad = False
		# unfreeze batchnorm scaling
		if args.train_batchnorm:
			for layer in model.modules():
				if isinstance(layer, torch.nn.BatchNorm2d):
					for param in layer.parameters():
						param.requires_grad = True

	device = torch.device('cuda:0')
	model.to(device)
	# set optimizer
	# optimizer = torch.optim.SGD(
	# 	filter(lambda x: x.requires_grad, model.parameters()),
	# 	lr=args.lr,
	# 	momentum=0.9,
	# 	weight_decay=args.wd,
	# )
	cls_optimizer = torch.optim.SGD(
		filter(lambda x: x.requires_grad, classifier.parameters()),
		lr=args.lr,
		momentum=0.9,
		weight_decay=args.wd,
	)
	criterion = nn.BCEWithLogitsLoss(reduction='none')

	transform_eval = [
		transforms.Resize(256),
		transforms.RandomHorizontalFlip(),
		transforms.CenterCrop(224),
		transforms.ToTensor(),
		normalize,
	]

	print('Test set')
	test_dataset = VOC2007_dataset(
		args.vocdir, split=args.test, transform=transforms.Compose(transform_eval))
	test_loader = torch.utils.data.DataLoader(
		test_dataset,
		batch_size=256,
		shuffle=False,
		num_workers=4,
		pin_memory=False,
	)

	print('Start training')
	it = 0
	losses = AverageMeter()
	count = 0
	while it < args.nit:
		print('Epoch {}'.format(count))
		it = train(
			loader,
			model,
			classifier,
			None,
			cls_optimizer,
			criterion,
			args.fc6_8,
			losses,
			it=it,
			total_iterations=args.nit,
			stepsize=args.stepsize,
			device=device
		)
		count += 1
		if count % 1 == 0:
			evaluate2(test_loader, model, classifier, args.eval_random_crops, device)

	print('Evaluation')
	map = evaluate2(test_loader, model, classifier, args.eval_random_crops, device)

def evaluate(loader, model, classifier, eval_random_crops, device):
	model.eval()
	classifier.eval()
	gts = []
	scr = []
	for crop in range(9 * eval_random_crops + 1):
		for i, (input, target) in enumerate(loader):
			# move input to gpu and optionally reshape it
			if len(input.size()) == 5:
				bs, ncrops, c, h, w = input.size()
				input = input.view(-1, c, h, w)
			#input = input.cuda(non_blocking=True)
			input = input.to(device)

			# forward pass without grad computation
			with torch.no_grad():
				output = model(input, 5)
				output.to(device)
				output = classifier(output)
			if crop < 1:
				scr.append(torch.sum(output, 0, keepdim=True).cpu().numpy())
				gts.append(target)
			else:
				scr[i] += output.cpu().numpy()
	gts = np.concatenate(gts, axis=0).T
	scr = np.concatenate(scr, axis=0).T
	aps = []
	for i in range(20):
		# Subtract eps from score to make AP work for tied scores
		ap = metrics.average_precision_score(
			gts[i][gts[i] <= 1], scr[i][gts[i] <= 1]-1e-5*gts[i][gts[i] <= 1])
		aps.append(ap)
	print(np.mean(aps), '  ', ' '.join(['%0.2f' % a for a in aps]))
	return np.mean(aps)

def evaluate2(loader, model, classifier, eval_random_crops, device):
	model.eval()
	classifier.eval()
	gts = []
	scr = []
	# for crop in range(9 * eval_random_crops + 1):
	for i, (input, target) in enumerate(loader):
		# move input to gpu and optionally reshape it
		#input = input.cuda(non_blocking=True)
		input = input.to(device)

		# forward pass without grad computation
		with torch.no_grad():
			# output = model(input, 5)
			output = model(input, 5, all_blocks=True)[-2]
			output.to(device)
			output = classifier(output)
		scr.append(output.cpu().numpy())
		gts.append(target)
	gts = np.concatenate(gts, axis=0).T
	scr = np.concatenate(scr, axis=0).T
	aps = []
	for i in range(20):
		# Subtract eps from score to make AP work for tied scores
		ap = metrics.average_precision_score(
			gts[i][gts[i] <= 1], scr[i][gts[i] <= 1]-1e-5*gts[i][gts[i] <= 1])
		aps.append(ap)
	print(np.mean(aps), '  ', ' '.join(['%0.2f' % a for a in aps]))
	return np.mean(aps)

def train(loader, model, classifier, optimizer, cls_optimizer, criterion, fc6_8, losses, it=0, total_iterations=None, stepsize=None, verbose=True, device=None):
	# to log
	batch_time = AverageMeter()
	data_time = AverageMeter()
	top1 = AverageMeter()
	end = time.time()

	current_iteration = it

	# use dropout for the MLP
	model.eval()
	# in the batch norms always use global statistics

	for (input, target) in loader:
		# measure data loading time
		data_time.update(time.time() - end)

		# adjust learning rate
		if current_iteration != 0 and current_iteration % stepsize == 0:
			for param_group in cls_optimizer.param_groups:
				param_group['lr'] = param_group['lr'] * 0.5
				print('iter {0} learning rate is {1}'.format(
					current_iteration, param_group['lr']))

		# move input to gpu
		input = input.to(device)
		target = target.float().to(device)
		#input = input.cuda(non_blocking=True)

		# forward pass with or without grad computation
		output = model(input, 5, all_blocks=True)[-2]
		output.to(device)
		output = classifier(output)

		#target = target.float().cuda()
		mask = (target == 255)
		loss = torch.sum(criterion(output, target).masked_fill_(
			mask, 0)) / target.size(0)

		# backward
		# optimizer.zero_grad()
		cls_optimizer.zero_grad()
		loss.backward()
		# optimizer.step()
		cls_optimizer.step()
		# clip gradients
		# torch.nn.utils.clip_grad_norm_(model.parameters(), 10)
		# and weights update

		# measure accuracy and record loss
		losses.update(loss.item(), input.size(0))

		# measure elapsed time
		batch_time.update(time.time() - end)
		end = time.time()
		if verbose is True and current_iteration % 1 == 0:
			print('Iteration[{0}]\t'
				  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
				  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
				  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
					  current_iteration, batch_time=batch_time,
					  data_time=data_time, loss=losses))
		current_iteration = current_iteration + 1
		if total_iterations is not None and current_iteration == total_iterations:
			break
	return current_iteration

class VOC2007_dataset(torch.utils.data.Dataset):
	def __init__(self, voc_dir, split='train', transform=None):
		# Find the image sets
		image_set_dir = os.path.join(voc_dir, 'ImageSets', 'Main')
		image_sets = glob.glob(os.path.join(
			image_set_dir, '*_' + split + '.txt'))
		assert len(image_sets) == 20
		# Read the labels
		self.n_labels = len(image_sets)
		images = defaultdict(lambda: -np.ones(self.n_labels, dtype=np.uint8))
		for k, s in enumerate(sorted(image_sets)):
			for l in open(s, 'r'):
				name, lbl = l.strip().split()
				lbl = int(lbl)
				# Switch the ignore label and 0 label (in VOC -1: not present, 0: ignore)
				if lbl < 0:
					lbl = 0
				elif lbl == 0:
					lbl = 255
				images[os.path.join(voc_dir, 'JPEGImages',
									name + '.jpg')][k] = lbl
		self.images = [(k, images[k]) for k in images.keys()]
		np.random.shuffle(self.images)
		self.transform = transform

	def __len__(self):
		return len(self.images)

	def __getitem__(self, i):
		#img = Image.open(self.images[i][0])
		#img = img.convert('RGB')
		#img = accimage.Image(self.images[i][0])
		img = pil_loader(self.images[i][0])
		if self.transform is not None:
			img = self.transform(img)
		return img, self.images[i][1]

if __name__ == '__main__':
	main()


text:
# ------------------------------------------------------------------------------
#  File: model.py
#  Author: Jan Kukacka
#  Date: 3/2021
# ------------------------------------------------------------------------------
#  Application data model component
# ------------------------------------------------------------------------------

import numpy as np
import traceback
import happy as hp
from matplotlib.colors import PowerNorm, to_hex
from multiprocessing import Process, Queue
from queue import Empty

from PIL import ImageTk, Image

try:
    from .ObservableCollections.observablelist import ObservableList
    from .ObservableCollections.observabledict import ObservableDict
    from .ObservableCollections.observable import Observable
    from .ObservableCollections.event import Event
    from .renderer import render
except ImportError:
    from ObservableCollections.observablelist import ObservableList
    from ObservableCollections.observabledict import ObservableDict
    from ObservableCollections.observable import Observable
    from ObservableCollections.event import Event
    from renderer import render

class Model(Observable):
    '''
    Data model object
    '''

    def __init__(self, use_gpu=True, debug=False, drop_tasks=True):
        Observable.__init__(self)

        ## Setup image rendering process
        self.rendering_queue = Queue()
        self.rendered_queue = Queue()
        self.rendering_process = Process(target=render, args=(self.rendering_queue, self.rendered_queue, use_gpu, debug, drop_tasks))

        ## Setup IO process
        self.io_task_queue = Queue()
        self.io_response_queue = Queue()
        self.io_process = Process(target=reader, args=(self.io_task_queue, self.io_response_queue))
        self.n_io_pending = 0

        self._filename = None
        self._image = None
        self._color_space = 'RGB'
        self._render = None
        self.suspend_render = False

        # self.histograms = None
        # self.responses = None
        self.response_images = None

        self.channel_props = ObservableList()
        self.channel_props.attach(lambda e, self=self: self.raiseEvent('propertyChanged', propertyName='channel_props', child=e))
        self.channel_prop_clipboard = None

        imoptions = {
            'use_unsharp_mask': True,
            'unsharp_mask_radius': 0.5,
            'unsharp_mask_amount': 1.0,
            'wavelength': 10,
            'wavelength_median': 5
        }
        self.imoptions = ObservableDict(imoptions)
        self.imoptions.attach(lambda x, self=self: self.raiseEvent('propertyChanged', propertyName='imoptions'))
        self.imoptions.attach(self.update_render)

        self.special_options = {
            ## Allows to selectively switch off unsharp mask for channel 2
            'no_sharpening_channel_2': False
        }

    def __enter__(self):
        self.rendering_process.start()
        self.io_process.start()
        return self

    def __exit__(self, type, value, traceback):
        ## Send termination signals
        self.rendering_queue.put(None)
        self.io_task_queue.put(None)

        ## Empty the result queues
        render = 1
        while render is not None:
            render = self.rendered_queue.get()
            if render is not None:
                self.render = render[0]

        io_response = 1
        while io_response is not None:
            io_response = self.io_response_queue.get()

        ## Join processes
        self.rendering_process.join()
        self.io_process.join()

    @property
    def filename(self):
        return self._filename

    @filename.setter
    def filename(self, val):
        if val is not self._filename:
            self._filename = val
            self.raiseEvent('propertyChanged', propertyName='filename')
            self.load_image()
            # image =
            # self.update_image(image)

    @property
    def image(self):
        return self._image

    @image.setter
    def image(self, val):
        if val is not self._image:
            self._image = val
            self.raiseEvent('propertyChanged', propertyName='image')
            e = Event('propertyChanged', self)
            e.propertyName = 'image'
            self.update_render(e)

    @property
    def color_space(self):
        return self._color_space

    @color_space.setter
    def color_space(self, val):
        if val is not self._color_space:
            self._color_space = val
            self.raiseEvent('propertyChanged', propertyName='color_space')
            self.update_cmap()

    @property
    def render(self):
        return self._render

    @render.setter
    def render(self, val):
        self._render = val
        self.raiseEvent('propertyChanged', propertyName='render')

    def load_image(self, event=None):
        task = {'type': 'load_image', 'filename':self.filename}
        self.io_task_queue.put(task)
        self.n_io_pending += 1
        self.raiseEvent('ioTask')

    def update_image(self, image):
        '''
        Used to update the image and reload channels
        '''
        self.suspend_render = True
        self.image = image
        self.update_channels()
        self.suspend_render = False
        e = Event('propertyChanged', self)
        e.propertyName = 'image'
        self.update_render(e)

    def update_channels(self):
        ## Clear old channel props
        while len(self.channel_props) > 0:
            del self.channel_props[0]

        n_channels = self.image.shape[2]

        for channel in range(n_channels):
            channel_property = {}
            channel_property['name'] = f'Channel {channel}'
            channel_property['use_local_contrast'] = True
            channel_property['local_contrast_neighborhood'] = 31
            channel_property['local_contrast_cut_off'] = 80
            channel_property['use_gamma'] = True
            channel_property['gamma'] = 1
            channel_property['use_sigmoid'] = True
            channel_property['sigmoid_low'] = 0
            channel_property['sigmoid_high'] = 100
            channel_property['sigmoid_new_low'] = 49
            channel_property['sigmoid_new_high'] = 51
            channel_property['color'] = '#ffffff'
            channel_property['visible'] = True

            channel_property = ObservableDict(channel_property)
            channel_property.attach(lambda x, self=self: self.raiseEvent('propertyChanged', propertyName='channel_props'))
            channel_property.attach(self.update_render)
            self.channel_props.append(channel_property)

    def check_for_render(self):
        try:
            # render, self.histograms, self.responses = self.rendered_queue.get_nowait()
            render, self.response_images = self.rendered_queue.get_nowait()
            self.render = render
        except Empty as e:
            pass

    def check_for_io(self):
        try:
            response = self.io_response_queue.get_nowait()
            self.n_io_pending -= 1
            if response['type'] == 'load_image':
                self.update_image(response['image'])
        except Empty as e:
            pass

    def update_render(self, event=None):
        ## Check we have all images
        if self.image is None:
            return
        ## Check render is not suspended
        if self.suspend_render:
            return

        render_task = {}
        render_task['channel_properties'] = [dict(channel_property) for channel_property in self.channel_props]
        render_task['imoptions'] = dict(self.imoptions)
        render_task['special_options'] = self.special_options

        ## If image has changed, pass it to the rendering thread too
        if event is not None and event.action == 'propertyChanged' and event.propertyName == 'image':
            render_task['image'] = self.image

        self.rendering_queue.put(render_task)

    def save(self):
        model_dict = {}
        model_dict['channel_props'] = [dict(cp) for cp in self.channel_props]
        return model_dict

    def load(self, model_dict):
        ## Suspend rendering while loading
        self.suspend_render = True

        for i, channel_property in enumerate(model_dict['channel_props']):
            if i >= len(self.channel_props):
                break
            for key, value in channel_property.items():
                if key in self.channel_props[i]:
                    self.channel_props[i][key] = value
                else:
                    print(f'Config key {key} could not be loaded.')

        self.suspend_render = False
        self.update_render()

    def transpose_image(self):
        self.image = self.image.transpose(1,0,2)

    def autocolor(self):
        for i, channel_prop in enumerate(self.channel_props):
            channel_prop['color'] = str(to_hex(f'C{i%10}'))

    def copy_params(self, channel_index):
        self.channel_prop_clipboard = dict(self.channel_props[channel_index])

    def paste_params(self, channel_index):
        if self.channel_prop_clipboard is None:
            return
        for key, value in self.channel_prop_clipboard.items():
            if key not in ['name', 'color', 'visible']:
                self.channel_props[channel_index][key] = value

def reader(input_queue, output_queue):
    while True:
        task = input_queue.get()

        ## Termination signal
        if task is None:
            # print('Exiting rendering thread')
            break

        try:
            response = {'type': task['type']}
            if task['type'] == 'load_image':
                filename = task['filename']
                response['image'] = load_image_internal(filename)
                # except Exception:
                #     print('Error loading image')
                #     response['image'] = None

            output_queue.put(response)

        except Exception as e:
            track = traceback.format_exc()
            print('Error in IO Thread:')
            print(track)
            response['exception'] = True
            output_queue.put(response)

    ## Signal finish of the rendered queue before quitting - it needs to be emptied
    output_queue.put(None)

def load_image_internal(filename):
    image = hp.io.load(filename)

    ## Handle MAT files:
    try:
        keys = [k for k in image.keys() if 'rec' in k]
        if len(keys) == 1:
            image = image[key]
        else:
            print('Could not load image (ambiguous keys)')
    except Exception:
        pass

    if image.ndim == 2:
        image = image[...,None]

    return image


text:
from time import process_time as time
import numpy as np
from math import sqrt

from classes.result import Result
from utils.settings import INF
from utils.debugger import Logger

import pycuda.autoinit
import pycuda.driver as drv
from pycuda.compiler import SourceModule

cuFilepath = './method/apsp/cu/dijkstra.cu'
logger = Logger(__name__)

def dijkstra(para):
    """
    function:
        use dijkstra algorithm in GPU to solve the APSP.

    parameters:
        class, Parameter object. (see the 'sparry/classes/parameter.py/Parameter').

    return:
        class, Result object. (see the 'sparry/classes/result.py/Result').
    """
    logger.debug("turning to func dijkstra-gpu-apsp")

    from utils.judgeDivide import judge_apsp

    tag = judge_apsp(para)

    # 2 is can run apsp directly. put all to the video memory.
    if tag == 2:
        dist, timeCost = nodivide(para.graph.graph, para.graph.n, para.pathRecordBool, para.BLOCK, para.GRID)

    # otherwise it's necessary to devide it into many sssp.
    else:
        dist, timeCost = divide(para.graph.graph, para.graph.n, para.graph.m, para.part, para.pathRecordBool, para.BLOCK, para.GRID, tag)

    result = Result(dist = dist, timeCost = timeCost, graph = para.graph)

    if para.pathRecordBool:
        result.calcPath()

    return result

# 整个图拷贝
def nodivide(CSR, n, pathRecordBool, BLOCK, GRID):
    """
    function:
        use dijkstra algorithm in GPU to solve the APSP.

    parameters:
        CSR: CSR graph data. (more info please see the developer documentation).
        n: int, the number of the vertices in the graph.
        pathRecordBool: bool, record the path or not.
        block: tuple, a 3-tuple of integers as (x, y, z), the block size, to shape the kernal threads.
        grid: tuple, a 2-tuple of integers as (x, y), the grid size, to shape the kernal blocks.

    return:
        class, Result object. (see the 'sparry/classes/result.py/Result').
    """

    logger.debug("turning to func dijkstra-gpu-apsp no-divide")

    with open(cuFilepath, 'r', encoding = 'utf-8') as f:
        cuf = f.read()
    mod = SourceModule(cuf)

    t1 = time()

    V, E, W = CSR[0], CSR[1], CSR[2]

    if BLOCK == None:
        BLOCK = (1024, 1, 1)

    if GRID == None:
        GRID = (512, 1)

    # malloc the space
    dist = np.full((n * n, ), INF).astype(np.int32)
    vis = np.full((n * n, ), 1).astype(np.int32)
    predist = np.full((n * n, ), INF).astype(np.int32)

    # init the all sources
    for i in range(n):
        # i is the source
        dist[i * n + i] = np.int32(0)
        vis[i * n + i] = np.int32(0)

    dij_apsp_cuda_fuc = mod.get_function('dijkstra')

    # run!
    dij_apsp_cuda_fuc(drv.In(V),
                        drv.In(E),
                        drv.In(W),
                        drv.In(n),
                        drv.In(vis),
                        drv.InOut(dist),
                        drv.In(predist),
                        block = BLOCK,
                        grid = GRID)

    timeCost = time() - t1

    # result
    return dist, timeCost

# I will not divide the apsp, just call sssp many times to solve it.

def divide(CSR, n, m, part, pathRecordBool, BLOCK, GRID, tag):
    """
    function:
        use dijkstra algorithm in GPU to solve the APSP, but this func can devide the graph if it's too large to put it in GPU memory.

    parameters:
        CSR: CSR graph data. (more info please see the developer documentation) .
        n: int, the number of the vertices in the graph.
        m: int, the number of the edge in the graph.
        part: int, the number of the edges that will put to GPU at a time.
        pathRecordBool: bool, record the path or not.
        block: tuple, a 3-tuple of integers as (x, y, z), the block size, to shape the kernal threads.
        grid: tuple, a 2-tuple of integers as (x, y), the grid size, to shape the kernal blocks
        tag: bool, convert APSP to SSSP, then the SSSP need to devide or not.

    return:
        class, Result object. (see the 'sparry/classes/result.py/Result').
    """

    logger.debug("turning to func dijkstra-gpu-apsp divide")

    # start time
    t1 = time()

    # devide sssp is not necessary to divide.
    if tag == 0:
        from method.sssp.dijkstra_gpu import direct as dij
    # divided sssp need to divide graph.
    else:
        from method.sssp.dijkstra_gpu import noStream as dij

    # the distence array.
    dist = []

    # goto sssp, so the other block is no use, so I set it as one block.
    if GRID != None:
        temp = 1
        for i in GRID:
            temp *= i
        if temp > 1:
            GRID = (1, 1)

    for s in range(n):
        disti = dij(CSR, n, m, np.int32(s), part, pathRecordBool, BLOCK, GRID)
        dist.append(disti)

    timeCost = time() - t1

    # result
    return dist, timeCost

reason: import numpy,import pycuda,from pycuda


text:
import os
import torch
import numpy as np

from src import configs
from src.model import BaselineResnet

def predict_test(test_loader):
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    model = BaselineResnet().to(device)
    preds = []

    for fold_folder in os.listdir(configs.submission_folder):
        fold_dir = configs.submission_folder / fold_folder

        for i, model_checkpoint in enumerate(os.listdir(fold_dir)):
            model_checkpoint_path = fold_dir / model_checkpoint
            model.load_state_dict(torch.load(model_checkpoint_path), strict=False)
            model_preds = []

            with torch.no_grad():
                for i_test, test_batch_data in enumerate(test_loader):
                    img, tab = test_batch_data
                    img, tab = img.to(device), tab.to(device)

                    model.eval()
                    test_logits = model(img, tab)

                    model_preds.append(test_logits.data.cpu().numpy())

                model_preds = np.concatenate(model_preds, axis=0)
                model_preds = model_preds.reshape((-1, 53, 5))
                model_preds = np.mean(model_preds, axis=1)

            preds.append(model_preds)

    preds = np.concatenate(preds, axis=1)
    preds = np.reshape(preds, (len(preds), -1, 5))
    preds = np.mean(preds, axis=1)

    return preds


text:
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from sentence_transformers import SentenceTransformer, models
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import roc_auc_score, adjusted_rand_score
from sklearn.metrics.pairwise import cosine_distances
from sklearn.cluster import AgglomerativeClustering
import random
import time
import math
import os
import argparse
from tqdm import tqdm
from model.models import CATSSimilarityModel
random.seed(42)
torch.manual_seed(42)
from numpy.random import seed
seed(42)

class CATS_arxiv(nn.Module): # CATS
    def __init__(self, emb_size):
        super(CATS_arxiv, self).__init__()
        self.emb_size = emb_size
        self.LL1 = nn.Linear(emb_size, 5 * emb_size)
        self.LL2 = nn.Linear(5 * emb_size, emb_size)
        self.LL3 = nn.Linear(5 * emb_size, 1)

    def forward(self, X):
        '''

        :param X: The input tensor is of shape (mC2 X 3*vec size) where m = num of paras for each query
        :return s: Pairwise CATS scores of shape (mC2 X 1)
        '''
        self.Xq = X[:, :self.emb_size]
        self.Xp1 = X[:, self.emb_size:2 * self.emb_size]
        self.Xp2 = X[:, 2 * self.emb_size:]
        self.z1 = torch.abs(self.Xp1 - self.Xq)
        self.z2 = torch.abs(self.Xp2 - self.Xq)
        self.zdiff = torch.abs(self.Xp1 - self.Xp2)
        self.zp1 = torch.relu(self.LL2(torch.relu(self.LL1(self.Xp1))))
        self.zp2 = torch.relu(self.LL2(torch.relu(self.LL1(self.Xp2))))
        self.zql = torch.relu(self.LL2(torch.relu(self.LL1(self.Xq))))
        self.zd = torch.abs(self.zp1 - self.zp2)
        self.zdqp1 = torch.abs(self.zp1 - self.zql)
        self.zdqp2 = torch.abs(self.zp2 - self.zql)
        self.z = torch.cat((self.zp1, self.zp2, self.zd, self.zdqp1, self.zdqp2), dim=1)
        o = torch.relu(self.LL3(self.z))
        o = o.reshape(-1)
        return o

    def num_flat_features(self, X):
        size = X.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features

    def predict(self, X_test):
        y_pred = self.forward(X_test)
        return y_pred

class CATSSimilarityModel_arxiv(nn.Module):
    def __init__(self, emb_size):
        super(CATSSimilarityModel_arxiv, self).__init__()
        self.cats = CATS_arxiv(emb_size)

    def forward(self, X):
        self.pair_scores = self.cats(X)
        return self.pair_scores

def arxiv_experiment(arxiv_qlabel, query_map, sbert_model_name, select_queries, lrate, epochs, batch,
                     eval_num, save, arxiv_vecs=None, arxiv_docs=None):
    if '/' in sbert_model_name:
        model = SentenceTransformer(sbert_model_name)
    else:
        bert_embed_model = models.Transformer(sbert_model_name, max_seq_length=512)
        pooling_model = models.Pooling(bert_embed_model.get_word_embedding_dimension(), pooling_mode_cls_token=False,
                                   pooling_mode_max_tokens=False, pooling_mode_mean_tokens=True)
        model = SentenceTransformer(modules=[bert_embed_model, pooling_model])
    queries, texts = [], []
    for q in query_map.keys():
        queries.append(q)
        texts.append(query_map[q])
    vecs = model.encode(texts)
    query_vecs = {}
    for i in range(len(queries)):
        query_vecs[queries[i]] = vecs[i]
    if arxiv_vecs is not None:
        abs_vecs = np.load(arxiv_vecs, allow_pickle=True)[()]['data']
    else:
        abstracts = np.load(arxiv_docs, allow_pickle=True)[()]['data']
        abstract_ids, abstract_texts = [], []
        for a in abstracts.keys():
            abstract_ids.append(a)
            abstract_texts.append(abstracts[a])
        abs_vecs = {}
        print('Going to embed paras')
        embeds = model.encode(abstract_texts, show_progress_bar=True)
        for i in range(len(abstract_ids)):
            abs_vecs[abstract_ids[i]] = embeds[i]
    abs_qlabels = np.load(arxiv_qlabel, allow_pickle=True)[()]['data']
    qdocs, labels = [], []
    for q in abs_qlabels.keys():
        for k in abs_qlabels[q].keys():
            for d in abs_qlabels[q][k]:
                qdocs.append({'a': d, 'q': q, 'cat': k})
                labels.append(q+':'+k)
    skf = StratifiedKFold(n_splits=2)
    for train_index, test_index in skf.split(qdocs, labels):
        train_qdocs = [qdocs[i] for i in train_index]
        test_qdocs = [qdocs[i] for i in test_index]
        abs_qlabels_train, abs_qlabels_test = {}, {}
        for d in train_qdocs:
            q, k, d = d['q'], d['cat'], d['a']
            if q in abs_qlabels_train.keys():
                if k in abs_qlabels_train[q].keys():
                    abs_qlabels_train[q][k].append(d)
                else:
                    abs_qlabels_train[q][k] = [d]
            else:
                abs_qlabels_train[q] = {k: [d]}
        for d in test_qdocs:
            q, k, d = d['q'], d['cat'], d['a']
            if q in abs_qlabels_test.keys():
                if k in abs_qlabels_test[q].keys():
                    abs_qlabels_test[q][k].append(d)
                else:
                    abs_qlabels_test[q][k] = [d]
            else:
                abs_qlabels_test[q] = {k: [d]}

        bal_pairs, labels = [], []
        for q in select_queries:
            dat = abs_qlabels_train[q]
            for k in dat.keys():
                curr_docs = dat[k]
                for i in range(len(curr_docs)-1):
                    for j in range(1, len(curr_docs)):
                        bal_pairs.append((q, curr_docs[i], curr_docs[j]))
                        labels.append(1)
                        neg_k = random.sample(list(dat.keys()), 1)[0]
                        while neg_k == k:
                            neg_k = random.sample(list(dat.keys()), 1)[0]
                        bal_pairs.append((q, curr_docs[i], random.sample(dat[neg_k], 1)[0]))
                        labels.append(0)
        emb_dim = model.get_sentence_embedding_dimension()
        X_train = torch.zeros((len(bal_pairs), 3*emb_dim))
        y_train = torch.tensor(labels, dtype=torch.float32)
        for i in range(len(bal_pairs)):
            X_train[i] = torch.tensor(np.hstack((query_vecs[bal_pairs[i][0]], abs_vecs[bal_pairs[i][1]], abs_vecs[bal_pairs[i][2]])))

        bal_pairs_test, labels_test = [], []
        for q in select_queries:
            dat = abs_qlabels_test[q]
            for k in dat.keys():
                curr_docs = dat[k]
                for i in range(len(curr_docs) - 1):
                    for j in range(1, len(curr_docs)):
                        bal_pairs_test.append((q, curr_docs[i], curr_docs[j]))
                        labels_test.append(1)
                        neg_k = random.sample(list(dat.keys()), 1)[0]
                        while neg_k == k:
                            neg_k = random.sample(list(dat.keys()), 1)[0]
                        bal_pairs_test.append((q, curr_docs[i], random.sample(dat[neg_k], 1)[0]))
                        labels_test.append(0)
        X_test = torch.zeros((len(bal_pairs_test), 3 * emb_dim))
        y_test = torch.tensor(labels_test, dtype=torch.float32)
        for i in range(len(bal_pairs_test)):
            X_test[i] = torch.tensor(np.hstack((query_vecs[bal_pairs_test[i][0]], abs_vecs[bal_pairs_test[i][1]], abs_vecs[bal_pairs_test[i][2]])))

        if torch.cuda.is_available():
            device = torch.device('cuda:0')
            print('CUDA available, using CUDA')
        else:
            device = torch.device('cpu')
            print('CUDA not available, using cpu')
        fold = 1

        cos = nn.CosineSimilarity(dim=1, eps=1e-6)
        y_cos = cos(X_test[:, emb_dim:emb_dim * 2], X_test[:, emb_dim * 2:])
        cos_auc = roc_auc_score(y_test.detach().cpu().numpy(), y_cos.detach().cpu().numpy())
        print('Test data Baseline cosine auc: %.5f', cos_auc)
        y_euclid = torch.sqrt(torch.sum((X_test[:, emb_dim:emb_dim * 2] - X_test[:, emb_dim * 2:]) ** 2, 1)).detach().cpu().numpy()
        y_euclid = 1 - (y_euclid - np.min(y_euclid)) / (np.max(y_euclid) - np.min(y_euclid))
        euclid_auc = roc_auc_score(y_test.detach().cpu().numpy(), y_euclid)
        print('Test data Baseline euclidean auc: %.5f', euclid_auc)
        print('Test data clustering eval')
        rands = []
        for q in select_queries:
            dat = abs_qlabels_test[q]
            docs, l = [], []
            for k in dat.keys():
                docs += dat[k]
                l += [k] * len(dat[k])
            vecs = np.zeros((len(docs), emb_dim))
            for i in range(len(docs)):
                vecs[i] = abs_vecs[docs[i]]
            k = len(set(l))
            score_matrix = cosine_distances(vecs, vecs)
            cl = AgglomerativeClustering(n_clusters=k, affinity='precomputed', linkage='average')
            cl_labels = cl.fit_predict(score_matrix)
            base_rand = adjusted_rand_score(l, cl_labels)
            rands.append(base_rand)
            print(q+' ARI: %.5f' % base_rand)
        rands = np.array(rands)
        print('Test data mean ARI: %.5f, stderr: %.5f' % (np.mean(rands), np.std(rands)))
        print('\n\n')

        train_samples = X_train.shape[0]
        test_samples = X_test.shape[0]
        m = CATSSimilarityModel(emb_dim, 'cats').to(device)
        m.cats.to(device)
        opt = optim.Adam(m.parameters(), lr=lrate)
        mseloss = nn.MSELoss()
        X_test = X_test.to(device)
        y_test = y_test.to(device)
        print('Starting training...')
        for i in range(epochs):
            print('\nEpoch ' + str(i + 1))
            for b in tqdm(range(math.ceil(train_samples // batch))):
                m.train()
                opt.zero_grad()
                curr_x = X_train[b * batch:b * batch + batch].to(device)
                ypred = m(curr_x)
                y_train_curr = y_train[b * batch:b * batch + batch].to(device)
                loss = mseloss(ypred, y_train_curr)
                auc = roc_auc_score(y_train_curr.detach().cpu().numpy(), ypred.detach().cpu().numpy())
                loss.backward()
                opt.step()
                if b % eval_num == 0:
                    m.eval()
                    test_loss = 0
                    test_auc = 0
                    n = 0
                    for bt in range(math.ceil(test_samples // batch)):
                        curr_x_test = X_test[bt * batch:bt * batch + batch].to(device)
                        y_test_curr = y_test[bt * batch:bt * batch + batch].to(device)
                        ypred_test = m(curr_x_test)
                        test_loss += mseloss(ypred_test, y_test_curr).item()
                        test_auc += roc_auc_score(y_test_curr.detach().cpu().numpy(), ypred_test.detach().cpu().numpy())
                        n += 1
                    print(
                        '\rTrain loss: %.5f, Train auc: %.5f, Test loss: %.5f, Test auc: %.5f' %
                        (loss.item(), auc, test_loss/n, test_auc/n), end='')
        m.eval()
        test_loss = 0
        test_auc = 0
        n = 0
        for bt in range(math.ceil(test_samples // batch)):
            curr_x_test = X_test[bt * batch:bt * batch + batch].to(device)
            y_test_curr = y_test[bt * batch:bt * batch + batch].to(device)
            ypred_test = m(curr_x_test)
            test_loss += mseloss(ypred_test, y_test_curr).item()
            test_auc += roc_auc_score(y_test_curr.detach().cpu().numpy(), ypred_test.detach().cpu().numpy())
            n += 1
        print(
            '\rTrain loss: %.5f, Train auc: %.5f, Test loss: %.5f, Test auc: %.5f' %
            (loss.item(), auc, test_loss / n, test_auc / n), end='')
        print('\n\nTest loss: %.5f, Test auc: %.5f' % (test_loss/n, test_auc/n))
        print('Test cluster eval')
        rands = []
        for q in select_queries:
            dat = abs_qlabels_test[q]
            docs, l = [], []
            for k in dat.keys():
                docs += dat[k]
                l += [k] * len(dat[k])
            vecs = np.zeros((len(docs), emb_dim))
            for i in range(len(docs)):
                vecs[i] = abs_vecs[docs[i]]
            k = len(set(l))
            score_matrix = np.zeros((len(docs), len(docs)))
            for i in range(len(docs)):
                rdata = torch.zeros((len(docs), 3*emb_dim))
                for j in range(len(docs)):
                    rdata[j] = torch.tensor(np.hstack((query_vecs[q], abs_vecs[docs[i]], abs_vecs[docs[j]])))
                rdata = rdata.to(device)
                rdata_scores = m(rdata).detach().cpu().numpy()
                score_matrix[i] = rdata_scores
            score_matrix = (score_matrix - np.min(score_matrix))/(np.max(score_matrix) - np.min(score_matrix))
            score_matrix = 1 - score_matrix
            cl = AgglomerativeClustering(n_clusters=k, affinity='precomputed', linkage='average')
            cl_labels = cl.fit_predict(score_matrix)
            base_rand = adjusted_rand_score(l, cl_labels)
            rands.append(base_rand)
            print(q+' ARI: %.5f' % base_rand)
        rands = np.array(rands)
        print('Test data mean ARI: %.5f, stderr: %.5f' % (np.mean(rands), np.std(rands)))
        print('\n\n')

        if save:
            if not os.path.isdir('saved_models'):
                os.makedirs('saved_models')
            torch.save(m.state_dict(), 'saved_models/fold-' + str(fold) + time.strftime('%b-%d-%Y_%H%M', time.localtime()) + '.model')
        fold += 1

def main():
    parser = argparse.ArgumentParser(description='Run CATS model')
    parser.add_argument('-ad', '--arxiv_docs', default=None)
    parser.add_argument('-av', '--arxiv_vecs', default=None)
    parser.add_argument('-ql', '--arxiv_qlabels', default='/home/sk1105/sumanta/arxiv_data_for_cats/arxiv_qlabels_for_cats.npy')
    parser.add_argument('-mn', '--model_name', default='bert-base-uncased')
    parser.add_argument('-lr', '--lrate', type=float, default=0.00001)
    parser.add_argument('-ep', '--epochs', type=int, default=3)
    parser.add_argument('-bt', '--batch', type=int, default=32)
    parser.add_argument('-en', '--eval_num', type=int, default=100)
    parser.add_argument('--save', action='store_true')
    args = parser.parse_args()

    selected_queries = ['eess', 'astro-ph', 'cond-mat', 'nlin', 'q-bio', 'q-fin', 'stat']
    query_map = {'cs': 'Computer Science',
             'econ': 'Economics',
             'eess': 'Electrical Engineering and Systems',
             'math': 'Mathematics',
             'astro-ph': 'Astrophysics',
             'cond-mat': 'Condensed Matter',
             'nlin': 'Nonlinear',
             'physics': 'Physics',
             'q-bio': 'Biology',
             'q-fin': 'Finance',
             'stat': 'Statistics'}
    arxiv_experiment(args.arxiv_qlabels, query_map, args.model_name, selected_queries, args.lrate,
                     args.epochs, args.batch, args.eval_num, args.save, args.arxiv_vecs, args.arxiv_docs)

if __name__ == '__main__':
    main()


text:
import datetime
import json

import numpy as np

from qstrader import settings
import qstrader.statistics.performance as perf

class JSONStatistics(object):
    """
    Standalone class to output basic backtesting statistics
    into a JSON file format.

    Parameters
    ----------
    equity_curve : `pd.DataFrame`
        The equity curve DataFrame indexed by date-time.
    strategy_id : `str`, optional
        The optional ID string for the strategy to pass to
        the statistics dict.
    strategy_name : `str`, optional
        The optional name string for the strategy to pass to
        the statistics dict.
    benchmark_curve : `pd.DataFrame`, optional
        The (optional) equity curve DataFrame for the benchmark
        indexed by time.
    benchmark_id : `str`, optional
        The optional ID string for the benchmark to pass to
        the statistics dict.
    benchmark_name : `str`, optional
        The optional name string for the benchmark to pass to
        the statistics dict.
    periods : `int`, optional
        The number of periods to use for Sharpe ratio calculation.
    output_filename : `str`
        The filename to output the JSON statistics dictionary to.
    """

    def __init__(
        self,
        equity_curve,
        target_allocations,
        strategy_id=None,
        strategy_name=None,
        benchmark_curve=None,
        benchmark_id=None,
        benchmark_name=None,
        periods=252,
        output_filename='statistics.json'
    ):
        self.equity_curve = equity_curve
        self.target_allocations = target_allocations
        self.strategy_id = strategy_id
        self.strategy_name = strategy_name
        self.benchmark_curve = benchmark_curve
        self.benchmark_id = benchmark_id
        self.benchmark_name = benchmark_name
        self.periods = periods
        self.output_filename = output_filename
        self.statistics = self._create_full_statistics()

    @staticmethod
    def _series_to_tuple_list(series):
        """
        Converts Pandas Series indexed by date-time into
        list of tuples indexed by milliseconds since epoch.

        Parameters
        ----------
        series : `pd.Series`
            The Pandas Series to be converted.

        Returns
        -------
        `list[tuple]`
            The list of epoch-indexed tuple values.
        """
        return [
            (
                int(
                    datetime.datetime.combine(
                        k, datetime.datetime.min.time()
                    ).timestamp() * 1000.0
                ), v if not np.isnan(v) else 0.0
            )
            for k, v in series.to_dict().items()
        ]

    @staticmethod
    def _dataframe_to_column_list(df):
        """
        Converts Pandas DataFrame indexed by date-time into
        list of tuples indexed by milliseconds since epoch.

        Parameters
        ----------
        df : `pd.DataFrame`
            The Pandas DataFrame to be converted.

        Returns
        -------
        `list[tuple]`
            The list of epoch-indexed tuple values.
        """
        col_list = []
        for k, v in df.to_dict().items():
            name = k.replace('EQ:', '')
            date_val_tups = [
                (
                    int(
                        datetime.datetime.combine(
                            date_key, datetime.datetime.min.time()
                        ).timestamp() * 1000.0
                    ), date_val if not np.isnan(date_val) else 0.0
                )
                for date_key, date_val in v.items()
            ]
            col_list.append({'name': name, 'data': date_val_tups})
        return col_list

    @staticmethod
    def _calculate_returns(curve):
        """
        Appends returns and cumulative returns to the supplied equity
        curve DataFrame.

        Parameters
        ----------
        curve : `pd.DataFrame`
            The equity curve DataFrame.
        """
        curve['Returns'] = curve['Equity'].pct_change().fillna(0.0)
        curve['CumReturns'] = np.exp(np.log(1 + curve['Returns']).cumsum())

    def _calculate_monthly_aggregated_returns(self, returns):
        """
        Calculate the monthly aggregated returns as a list of tuples,
        with the first entry a further tuple of (year, month) and the
        second entry the returns. 0% -> 0.0, 100% -> 1.0

        Parameters
        ----------
        returns : `pd.Series`
            The Series of daily returns values.

        Returns
        -------
        `list[tuple]`
            The list of tuple-based returns: [((year, month), return)]
        """
        month_returns = perf.aggregate_returns(returns, 'monthly')
        return list(zip(month_returns.index, month_returns))

    def _calculate_monthly_aggregated_returns_hc(self, returns):
        """
        Calculate the monthly aggregated returns in the format
        utilised by Highcharts. 0% -> 0.0, 100% -> 100.0

        Parameters
        ----------
        returns : `pd.Series`
            The Series of daily returns values.

        Returns
        -------
        `list[list]`
            The list of list-based returns: [[month, year, return]]
        """
        month_returns = perf.aggregate_returns(returns, 'monthly')

        data = []

        years = month_returns.index.levels[0].tolist()
        years_range = range(0, len(years))
        months_range = range(0, 12)

        for month in months_range:
            for year in years_range:
                try:
                    data.append([month, year, 100.0 * month_returns.loc[(years[year], month + 1)]])
                except KeyError:  # Truncated year, so no data available
                    pass

        return data

    def _calculate_yearly_aggregated_returns(self, returns):
        """
        Calculate the yearly aggregated returns as a list of tuples,
        with the first entry being the year integer and the
        second entry the returns. 0% -> 0.0, 100% -> 1.0

        Parameters
        ----------
        returns : `pd.Series`
            The Series of daily returns values.

        Returns
        -------
        `list[tuple]`
            The list of tuple-based returns: [(year, return)]
        """
        year_returns = perf.aggregate_returns(returns, 'yearly')
        return list(zip(year_returns.index, year_returns))

    def _calculate_yearly_aggregated_returns_hc(self, returns):
        """
        Calculate the yearly aggregated returns in the format
        utilised by Highcharts. 0% -> 0.0, 100% -> 100.0

        Parameters
        ----------
        returns : `list[tuple]`
            The list of tuples, with the first index being the year
            integer and the second index being the return.

        Returns
        -------
        `list[float]`
            The list of returns.
        """
        year_returns = self._calculate_yearly_aggregated_returns(returns)
        return [year[1] * 100.0 for year in year_returns]

    def _calculate_returns_quantiles_dict(self, returns):
        """
        Creates a dictionary with quantiles for the
        provided returns series.

        Parameters
        ----------
        returns : `pd.Series` or `list[float]`
            The Series/list of returns values.

        Returns
        -------
        `dict{str: float}`
            The quantiles of the provided returns series.
        """
        return {
            'min': np.min(returns),
            'lq': np.percentile(returns, 25),
            'med': np.median(returns),
            'uq': np.percentile(returns, 75),
            'max': np.max(returns)
        }

    def _calculate_returns_quantiles(self, daily_returns):
        """
        Creates a dict-of-dicts with quantiles for the
        daily, monthly and yearly returns series.

        Parameters
        ----------
        daily_returns : `pd.Series`
            The Series of daily returns values.

        Returns
        -------
        `dict{str: dict{str: float}}`
            The quantiles of the daily, monthly and yearly returns.
        """
        monthly_returns = [m[1] for m in self._calculate_monthly_aggregated_returns(daily_returns)]
        yearly_returns = [y[1] for y in self._calculate_yearly_aggregated_returns(daily_returns)]
        return {
            'daily': self._calculate_returns_quantiles_dict(daily_returns),
            'monthly': self._calculate_returns_quantiles_dict(monthly_returns),
            'yearly': self._calculate_returns_quantiles_dict(yearly_returns)
        }

    def _calculate_returns_quantiles_hc(self, returns_quantiles):
        """
        Convert the returns quantiles dict-of-dicts into
        a format suitable for Highcharts boxplots.

        Parameters
        ----------
        `dict{str: dict{str: float}}`
            The quantiles of the daily, monthly and yearly returns.

        Returns
        -------
        `list[list[float]]`
            The list-of-lists of return quantiles (in 0-100 percent terms).
        """
        percentiles = ['min', 'lq', 'med', 'uq', 'max']
        return [
            [returns_quantiles['daily'][stat] * 100.0 for stat in percentiles],
            [returns_quantiles['monthly'][stat] * 100.0 for stat in percentiles],
            [returns_quantiles['yearly'][stat] * 100.0 for stat in percentiles]
        ]

    def _calculate_statistics(self, curve):
        """
        Creates a dictionary of various statistics associated with
        the backtest of a trading strategy via a supplied equity curve.

        All Pandas Series indexed by date-time are converted into
        milliseconds since epoch representation.

        Parameters
        ----------
        curve : `pd.DataFrame`
            The equity curve DataFrame.

        Returns
        -------
        `dict`
            The statistics dictionary.
        """
        stats = {}

        # Drawdown, max drawdown, max drawdown duration
        dd_s, max_dd, dd_dur = perf.create_drawdowns(curve['CumReturns'])

        # Equity curve and returns
        stats['equity_curve'] = JSONStatistics._series_to_tuple_list(curve['Equity'])
        stats['returns'] = JSONStatistics._series_to_tuple_list(curve['Returns'])
        stats['cum_returns'] = JSONStatistics._series_to_tuple_list(curve['CumReturns'])

        # Month/year aggregated returns
        stats['monthly_agg_returns'] = self._calculate_monthly_aggregated_returns(curve['Returns'])
        stats['monthly_agg_returns_hc'] = self._calculate_monthly_aggregated_returns_hc(curve['Returns'])
        stats['yearly_agg_returns'] = self._calculate_yearly_aggregated_returns(curve['Returns'])
        stats['yearly_agg_returns_hc'] = self._calculate_yearly_aggregated_returns_hc(curve['Returns'])

        # Returns quantiles
        stats['returns_quantiles'] = self._calculate_returns_quantiles(curve['Returns'])
        stats['returns_quantiles_hc'] = self._calculate_returns_quantiles_hc(stats['returns_quantiles'])

        # Drawdown statistics
        stats['drawdowns'] = JSONStatistics._series_to_tuple_list(dd_s)
        stats['max_drawdown'] = max_dd
        stats['max_drawdown_duration'] = dd_dur

        # Performance
        stats['mean_returns'] = np.mean(curve['Returns'])
        stats['stdev_returns'] = np.std(curve['Returns'])
        stats['cagr'] = perf.create_cagr(curve['CumReturns'], self.periods)
        stats['annualised_vol'] = np.std(curve['Returns']) * np.sqrt(self.periods)
        stats['sharpe'] = perf.create_sharpe_ratio(curve['Returns'], self.periods)
        stats['sortino'] = perf.create_sortino_ratio(curve['Returns'], self.periods)

        return stats

    def _calculate_allocations(self, allocations):
        """
        """
        return JSONStatistics._dataframe_to_column_list(allocations)

    def _create_full_statistics(self):
        """
        Create the 'full' statistics dictionary, which has an entry for the
        strategy and an optional entry for any supplied benchmark.

        Returns
        -------
        `dict`
            The strategy and (optional) benchmark statistics dictionary.
        """
        full_stats = {}

        JSONStatistics._calculate_returns(self.equity_curve)
        full_stats['strategy'] = self._calculate_statistics(self.equity_curve)
        full_stats['strategy']['target_allocations'] = self._calculate_allocations(
            self.target_allocations
        )

        if self.benchmark_curve is not None:
            JSONStatistics._calculate_returns(self.benchmark_curve)
            full_stats['benchmark'] = self._calculate_statistics(self.benchmark_curve)

        if self.strategy_id is not None:
            full_stats['strategy_id'] = self.strategy_id
        if self.strategy_name is not None:
            full_stats['strategy_name'] = self.strategy_name
        if self.benchmark_id is not None:
            full_stats['benchmark_id'] = self.benchmark_id
        if self.benchmark_name is not None:
            full_stats['benchmark_name'] = self.benchmark_name

        return full_stats

    def to_file(self):
        """
        Outputs the statistics dictionary to a JSON file.
        """
        if settings.PRINT_EVENTS:
            print('Outputting JSON results to "%s"...' % self.output_filename)
        with open(self.output_filename, 'w') as outfile:
            json.dump(self.statistics, outfile)

max_stars_repo_name: calumrussell/qstrader

max_issues_repo_name: calumrussell/qstrader

max_forks_repo_name: calumrussell/qstrader


text:
import gym
import numpy as np

class MultiDiscrete(gym.Space):
    def __init__(self, nvec):
        """
        nvec: vector of counts of each categorical variable
        """
        self.nvec = np.asarray(nvec, dtype=np.int32)
        gym.Space.__init__(self, (self.nvec.shape,), np.int8)
    def sample(self):
        return (gym.spaces.np_random.random_sample(self.nvec.shape) * self.nvec).astype(self.dtype)
    def contains(self, x):
        return (0 <= x).all() and (x < self.nvec).all() and x.dtype.kind in 'ui'

    def to_jsonable(self, sample_n):
        return [sample.tolist() for sample in sample_n]
    def from_jsonable(self, sample_n):
        return np.array(sample_n)

max_stars_repo_name: sharma-arjun/gym

max_issues_repo_name: LucasSilve/openAIgym

max_forks_repo_name: LucasSilve/openAIgym


text:
# -*- coding: utf-8 -*-

import numpy as np
import os
import sys
from tensorflow.keras import models
from tensorflow.keras.preprocessing.image import img_to_array
from tensorflow.keras.preprocessing.image import load_img

img_rows = None
img_cols = None
digits_in_img = 6
model = None
np.set_printoptions(suppress=True, linewidth=150, precision=9, formatter={'float': '{: 0.9f}'.format})

def split_digits_in_img(img_array):
    x_list = list()
    for i in range(digits_in_img):
        step = img_cols // digits_in_img #Floor division
        x_list.append(img_array[:, i * step:(i + 1) * step] / 255)
    return x_list

if os.path.isfile('cnn_model.h5'):
    model = models.load_model('cnn_model.h5')
else:
    print('No trained model found.')
    exit(-1)

img_filenames = os.listdir('testing')

for img_filename in img_filenames:
    if '.png' not in img_filename:
        continue
    img = load_img('testing/{0}'.format(img_filename), color_mode='grayscale')
    img_array = img_to_array(img)
    img_rows, img_cols, _ = img_array.shape
    x_list = split_digits_in_img(img_array)

    varification_code = list()
    for i in range(digits_in_img):
        confidences = model.predict(np.array([x_list[i]]), verbose=0)
        result_class = model.predict_classes(np.array([x_list[i]]), verbose=0)
        varification_code.append(result_class[0])
        print('Digit {0}: Confidence=> {1}    Predict=> {2}'.format(i + 1, np.squeeze(confidences), np.squeeze(result_class)))
    print('Predicted varification code:', varification_code)

'''
img_filename = os.listdir('testing')  # renew
img = load_img(img_filename, color_mode='grayscale')
img_array = img_to_array(img)
img_rows, img_cols, _ = img_array.shape
x_list = split_digits_in_img(img_array)

varification_code = list()
for i in range(digits_in_img):
    confidences = model.predict(np.array([x_list[i]]), verbose=0)
    result_class = model.predict_classes(np.array([x_list[i]]), verbose=0)
    varification_code.append(result_class[0])
    print('Digit {0}: Confidence=> {1}    Predict=> {2}'.format(i + 1, np.squeeze(confidences), np.squeeze(result_class)))
print('Predicted varification code:', varification_code)

'''