text:
from ot_sparse_projection.matrix import MatrixHandler
import numpy as np
from scipy.ndimage import convolve1d

CONVOLUTION = 'convolution'
DEFAULT = CONVOLUTION
MATRIX = 'matrix'

def get_distance_matrix(shape):
    x = np.indices(shape).reshape([len(shape), -1]).swapaxes(0, 1)
    dist = x.reshape([-1, 1, len(shape)]) - x.reshape([1, -1, len(shape)])
    dist = (dist ** 2).sum(axis=2)
    return np.sqrt(dist)

def get_handler(type, shape, gamma):
    if type == CONVOLUTION:
        return ConvolutionGroundMetricHandler(shape, gamma)
    if type == MATRIX:
        return MatrixGroundMetricHandler(get_distance_matrix(shape) ** 2.0, gamma)

class GroundMetricHandler(MatrixHandler):
    """An abstract adapter class for transportation cost matrix.

    Some cost matrix can be represented more efficiently than by storing the actual matrix
    """

    def __init__(self, gamma):
        """
        Arguments:
             gamma: the regularization strength for entropy-regulatized optimal transport
        """
        super(GroundMetricHandler, self).__init__()
        self.gamma = gamma

    def conjugate(self, x, h, entropy=0, grad=False):
        """ Computes the convex conjugate of ot_gamma(x,.) evaluated at h
        Arguments:
             x, h: the argument where we want to evaluate the function
             entropy: optional pre-computed entropy of x
             grad: if true, also returns the gradient
        Returns:
            obj, grad: obj, the convex conjugate of ot_gamma(x,.) evaluated at h; and optionally the gradient grad
        """
        max_e = h.max(axis=0).reshape(1, -1) - self.gamma * 20
        return self._conjugate_unsafe(x, h - max_e, entropy=entropy, grad=grad, add_log=max_e)

    def _conjugate_unsafe(self, x, h, entropy=0, grad=False, add_log=0):
        """ Computes the convex conjugate of ot_gamma(x,.) evaluated at h unsafely, that is without any rescaling
        Arguments:
             x, h: the argument where we want to evaluate the function
             entropy: optional pre-computed entropy of x
             grad: if true, also returns the gradient
             add_log: an additive rescaling used to avoid overflows due to exponentials
        Returns:
            obj, grad: obj, the convex conjugate of ot_gamma(x,.) evaluated at h; and optionally the gradient grad
        """
        alpha = np.exp(h / self.gamma)
        tmp = self.dot(alpha)
        obj = (entropy + np.multiply(x, np.log(tmp) + add_log / self.gamma).sum(axis=0).reshape(1, -1)) * self.gamma
        if not grad:
            return obj
        tmp = self.transpose_dot(np.divide(x, tmp))
        gradient = np.multiply(alpha, tmp)
        return obj, gradient

def get_1d_filter(sz, gamma, threshold):
    filter = np.exp(-np.abs(np.array(range(-sz, sz, 1))).astype(np.float64) ** 2.0 / gamma) - threshold
    return filter[filter > 0]

class ConvolutionGroundMetricHandler(GroundMetricHandler):
    """A representation of a cost matrix that amounts to euclidean distances on a grid, as is the case for grayscale images

    With an entropy regularized optimal transport, all we need is to be able to perform products with the exponentiated
     matrix, which in this case amount to gaussian blur (see Justin Solomon, Fernando de Goes, Gabriel Peyré,
     Marco Cuturi, Adrian Butscher, Andy Nguyen, Tao Du, and Leonidas Guibas. Convolutional wasserstein distances:
     Efficient optimal transportation on geometric domains. ACM Transactions on Graphics, 34(4), July 2015a.
     ISSN 0730-0301.)
    """

    def __init__(self, shape, gamma, threshold=1e-200):
        """
        Arguments:
            shape: a tupple representing the shape of input arguments along each dimension
            gamma: the regularization strength for entropy-regulatized optimal transport
            threshold: a threshold for the exponentiated cost matrix, so that underflows to 0 are reassigned to a small
                        value
        """
        GroundMetricHandler.__init__(self, gamma)
        self._shape = shape
        self._numFilters = len(shape)
        self._filters = [get_1d_filter(sz, gamma, threshold=threshold) for sz in shape]
        self._reshape = shape + (-1,)
        self.threshold = threshold

    def cols(self):
        return np.prod(self._shape)

    def rows(self):
        return np.prod(self._shape)

    def dot(self, x):
        y = x.reshape(self._reshape)
        for i in range(self._numFilters):
            y = convolve1d(y, self._filters[i], axis=i, mode='constant')
        return y.reshape(x.shape) + x * self.threshold

    def transpose_dot(self, x):
        return self.dot(x)

class MatrixGroundMetricHandler(GroundMetricHandler):
    """A cost matrix, actually represented as a matrix"""

    def __init__(self, C, gamma, threshold=1e-200):
        """
        Arguments:
            C: the transportation cost matrix
            gamma: the regularization strength for entropy-regulatized optimal transport
            threshold: a threshold for the exponentiated cost matrix, so that underflows to 0 are reassigned to a small
                        value
        """
        GroundMetricHandler.__init__(self, gamma)
        self._shape = C.shape
        self.M = np.exp(-C / gamma)
        self.M[self.M < threshold] = threshold

    def cols(self):
        return self._shape[1]

    def rows(self):
        return self._shape[0]

    def dot(self, x):
        return self.M.dot(x)

    def transpose_dot(self, x):
        return self.M.T.dot(x)

max_stars_repo_name: arolet/ot-sparse-projection

max_issues_repo_name: arolet/ot-sparse-projection

max_forks_repo_name: arolet/ot-sparse-projection


text:
#-*- coding:UTF-8 -*-
#!/usr/bin/python2.7
import cv2
import argparse
import os
import sys
import numpy as np
path_video='/home/ecl/make/video2img/video/'         # path of  the video to be converted
path_image='/home/ecl/make/video2img/image/'         # path of  the images to be saved
VideoPath = os.listdir(path_video)

j=2
def video_to_img():
  for video_name in VideoPath:

         image_file = path_video + video_name

         print(image_file)
         # 使用opencv按一定间隔截取视频帧，并保存为图片
         vc = cv2.VideoCapture(image_file) #读取视频文件
         if vc.isOpened():#判断是否正常打开
            flag,frame = vc.read()
         else:
             flag = False
         c = 1
         d=0
         timeF = 30 #视频帧计数间隔频率
         while flag: #循环读取视频帧
              flag,frame = vc.read()
              if frame is not None :
                 if (c%timeF == 0):  #每隔timeF帧进行存储操作
                    cv2.imwrite('./image/'+str(c)+'.jpg',frame) # 存储为图像
                    print("*************  Number ---> %s *************** " % (d))
                    d+=1

              c = c + 1
              cv2.waitKey(1)
         vc.release()
         global j
         print("*************************")
         rename()
         j+=1

def rename():
    filelist = os.listdir(path_image)
    total_num = len(filelist)
    i = 0 #图片名称起始值

    for item in filelist:
        if item.endswith('.jpg'):
            src = os.path.join(os.path.abspath(path_image), item)
            str1=str(i)
            dst = os.path.join(os.path.abspath(path_image), str1.zfill(j) +'_tunnel'+'.jpg') #图片名称为5位数，即第一张图片为000000，第二张图片为000001，以此类推.......
            try:
                os.rename(src, dst)
                print ('converting %s to %s ...' % (src, dst))
                i+= 1
            except:
                  continue

       # print ('total %d to rename & converted %d jpgs' % (total_num, i))

if __name__ == '__main__':
     video_to_img()

ext: py

reason: import numpy


text:
import pytest
import librosa
import torch
import sys

sys.path.insert(0, "./")

import os

dir_path = os.path.dirname(os.path.realpath(__file__))

from nnAudio.features import CQT2010v2, VQT
import numpy as np
from parameters import *
import warnings

gpu_idx = 0  # Choose which GPU to use

# If GPU is avaliable, also test on GPU
if torch.cuda.is_available():
    device_args = ["cpu", f"cuda:{gpu_idx}"]
else:
    warnings.warn("GPU is not avaliable, testing only on CPU")
    device_args = ["cpu"]

# librosa example audio for testing
y, sr = librosa.load(librosa.ex('choice'), duration=5)

@pytest.mark.parametrize("device", [*device_args])
def test_vqt_gamma_zero(device):
    # nnAudio cqt
    spec = CQT2010v2(sr=sr, verbose=False)
    C2 = spec(torch.tensor(y).unsqueeze(0), output_format="Magnitude", normalization_type='librosa')
    C2 = C2.numpy().squeeze()

    # nnAudio vqt
    spec = VQT(sr=sr, gamma=0, verbose=False)
    V2 = spec(torch.tensor(y).unsqueeze(0), output_format="Magnitude", normalization_type='librosa')
    V2 = V2.numpy().squeeze()

    assert (C2 == V2).all() == True

@pytest.mark.parametrize("device", [*device_args])
def test_vqt(device):
    for gamma in [0, 1, 2, 5, 10]:

        # librosa vqt
        V1 = np.abs(librosa.vqt(y, sr=sr, gamma=gamma))

        # nnAudio vqt
        spec = VQT(sr=sr, gamma=gamma, verbose=False)
        V2 = spec(torch.tensor(y).unsqueeze(0), output_format="Magnitude", normalization_type='librosa')
        V2 = V2.numpy().squeeze()

        # NOTE: there will still be some diff between librosa and nnAudio vqt values (same as cqt)
        # mainly due to the lengths of both - librosa uses float but nnAudio uses int
        # this test aims to keep the diff range within a baseline threshold
        vqt_diff = np.abs(V1 - V2)

        if gamma == 0:
            assert np.amin(vqt_diff) < 1e-8
            assert np.amax(vqt_diff) < 0.6785
        elif gamma == 1:
            assert np.amin(vqt_diff) < 1e-8
            assert np.amax(vqt_diff) < 0.6510
        elif gamma == 2:
            assert np.amin(vqt_diff) < 1e-8
            assert np.amax(vqt_diff) < 0.5962
        elif gamma == 5:
            assert np.amin(vqt_diff) < 1e-8
            assert np.amax(vqt_diff) < 0.3695
        else:
            assert np.amin(vqt_diff) < 1e-8
            assert np.amax(vqt_diff) < 0.1


text:
import numpy as np
import os
import re
import itertools
import scipy.sparse as sp
import pickle
from collections import Counter
from nltk.corpus import stopwords
from tqdm import tqdm
cachedStopWords = stopwords.words("spanish")

def clean_str(string):
    # remove stopwords
    # string = ' '.join([word for word in string.split() if word not in cachedStopWords])
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    return string.strip().lower()

def pad_sentences(sentences, padding_word="<PAD/>", max_length=500):
    sequence_length = min(max(len(x) for x in sentences), max_length)
    padded_sentences = []
    for i in range(len(sentences)):
        sentence = sentences[i]
        if len(sentence) < max_length:
            num_padding = sequence_length - len(sentence)
            new_sentence = sentence + [padding_word] * num_padding
        else:
            new_sentence = sentence[:max_length]
        padded_sentences.append(new_sentence)
    return padded_sentences

def load_data_and_labels(data):

    import pandas as pd
    df_ejemplo = pd.read_csv('data_ejemplo.csv', sep=';')
    L = df_ejemplo['catgy'].values.tolist()
    L = [x.split(',') for x in L]
    new_L = []
    for lst in L:
        new_lst = []
        for x in lst:
            new_lst.append(int(x))
        new_L.append(new_lst)

    labels = new_L.copy()
    x_text = df_ejemplo['text'].values.tolist()
    x_text = [clean_str(doc) for doc in x_text]
    x_text = [s.split(" ") for s in x_text]

#    x_text = [clean_str(doc['text']) for doc in data]
#    x_text = [s.split(" ") for s in x_text]
#    labels = [doc['catgy'] for doc in data]

    row_idx, col_idx, val_idx = [], [], []
    for i in tqdm(range(len(labels))):
        l_list = list(set(labels[i])) # remove duplicate cateories to avoid double count
        for y in l_list:
            row_idx.append(i)
            col_idx.append(y)
            val_idx.append(1)
    m = max(row_idx) + 1
    n = max(col_idx) + 1
    Y = sp.csr_matrix((val_idx, (row_idx, col_idx)), shape=(m, n))
    return [x_text, Y]

def build_vocab(sentences, vocab_size=50000):
    word_counts = Counter(itertools.chain(*sentences))
    vocabulary_inv = [x[0] for x in word_counts.most_common(vocab_size)]
    vocabulary = {x: i for i, x in enumerate(vocabulary_inv)}
    # append <UNK/> symbol to the vocabulary
    vocabulary['<UNK/>'] = len(vocabulary)
    vocabulary_inv.append('<UNK/>')
    return [vocabulary, vocabulary_inv]

def build_input_data(sentences, vocabulary):
    x = np.array([[vocabulary[word] if word in vocabulary else vocabulary['<UNK/>'] for word in sentence] for sentence in sentences])
    #x = np.array([[vocabulary[word] if word in vocabulary else len(vocabulary) for word in sentence] for sentence in sentences])
    return x

def load_data(data_path, max_length=500, vocab_size=50000):
    # Load and preprocess data
    with open(os.path.join(data_path), 'rb') as fin:
#         [train, test, vocab, catgy] = pickle.load(fin)
        [train, test, vocab, catgy] = pickle.load(fin,encoding='latin1')
    print('Train:')
    print(train[2])

    # dirty trick to prevent errors happen when test is empty
    if len(test) == 0:
        test[:5] = train[:5]

    trn_sents, Y_trn = load_data_and_labels(train)
    tst_sents, Y_tst = load_data_and_labels(test)

    print('trn_sents:')
    print(type(trn_sents))
    print(len(trn_sents))
 #   print(trn_sents[2])

    print('Y_trn:')
    print(type(Y_trn))
    print(Y_trn[2])

    trn_sents_padded = pad_sentences(trn_sents, max_length=max_length)
    tst_sents_padded = pad_sentences(tst_sents, max_length=max_length)
    print("len:",len(trn_sents_padded),len(tst_sents_padded))
    vocabulary, vocabulary_inv = build_vocab(trn_sents_padded + tst_sents_padded, vocab_size=vocab_size)
    X_trn = build_input_data(trn_sents_padded, vocabulary)
    X_tst = build_input_data(tst_sents_padded, vocabulary)
    return X_trn, Y_trn, X_tst, Y_tst, vocabulary, vocabulary_inv
    # return X_trn, Y_trn, vocabulary, vocabulary_inv

def batch_iter(data, batch_size, num_epochs):
    """
    Generates a batch iterator for a dataset.
    """
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data)/batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        shuffle_indices = np.random.permutation(np.arange(data_size))
        shuffled_data = data[shuffle_indices]
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]

reason: import numpy,import scipy


text:
import argparse
import sys
sys.path.append('../code')
import subprocess
import matplotlib.pyplot as plt
import numpy as np
import os
import fnmatch
import time
import pickle
from tools import general_tools as gt

pathToResults = "../../results"

def main():
	""" This program launches several static reflex recordings with different random seeds """

	parser = argparse.ArgumentParser(description="launch several static reflex recordings with different random seeds")
	parser.add_argument("eesStartingAmplitude", help="ees starting amplitude (times motor thresold 0-3)")
	parser.add_argument("eesMaxAmplitude", help="ees max amplitude (times motor thresold 0-3)")
	parser.add_argument("eesSteps", help="number of stimulatin steps")
	parser.add_argument("nSim", help="number of simulations to run", type=int)
	parser.add_argument("inputFile", help="neural network structure file")
	parser.add_argument("outFileName", help="name to add at the output files")
	args = parser.parse_args()

	nProc = 4
	count=0.
	percLastPrint=0.
	printPeriod = 0.05

	for n in xrange(args.nSim):
		name = "batchSRR_seedn_"+str(n)+"_"+args.outFileName

		resultName = "_static_reflex_recording_min"+args.eesStartingAmplitude+"_max"+str(float(args.eesMaxAmplitude))+name
		resultFile = gt.find("*"+resultName+".p",pathToResults)
		if not resultFile:
			program = ['python','./scripts/runStaticReflexRecordings.py',\
					args.eesStartingAmplitude,args.eesMaxAmplitude,args.eesSteps,args.inputFile,name,"--noPlot","--seed",str(n),"--membranePotential"]#,"--seed",str(1) 'mpiexec','-np',str(nProc),
			gt.run_subprocess(program)

		count+=1
		if count/args.nSim-percLastPrint>=printPeriod:
			percLastPrint=count/args.nSim
			print str(round(count/args.nSim*100))+"% of simulations performed..."

	figName = time.strftime("/%Y_%m_%d_Batch_SRR_nSim_"+str(args.nSim)+"_min"+args.eesStartingAmplitude+"_max"+args.eesMaxAmplitude+"_"+args.outFileName+".pdf")
	plot_afferents(args.nSim,args.eesStartingAmplitude,args.eesMaxAmplitude,figName,args.outFileName)

def plot_afferents(nSimulations,eesStartingAmplitude,eesMaxAmplitude,figName,outFileName,showPlot=False):
	""" Plots the averaged afferents firing rates. """

	taStat = []
	gmStat = []
	for n in xrange(nSimulations):
		name = "batchSRR_seedn_"+str(n)+"_"+outFileName
		#name = "_static_reflex_recording_min"+eesStartingAmplitude+"_max"+str(float(eesMaxAmplitude))+"batchSRR_seedn_"+str(n)+"_"+outFileName
		resultFile = gt.find("*"+name+".p",pathToResults)
		print resultFile
		if len(resultFile)>1: print "Warning: multiple result files found!!!"
		with open(resultFile[0], 'r') as pickle_file:
			taSpikes = pickle.load(pickle_file)
			taEmg = pickle.load(pickle_file)
			taStatVal = pickle.load(pickle_file)
			gmSpikes = pickle.load(pickle_file)
			gmEmg = pickle.load(pickle_file)
			gmStatVal = pickle.load(pickle_file)
		taStat.append(taStatVal)
		gmStat.append(gmStatVal)

	fig, ax = plt.subplots(2, 3,figsize=(16,9),sharex='col',sharey='col')
	for flexOneSeed,extOneSeed in zip(taEmg,gmEmg):
		ax[0,1].plot(flexOneSeed)
		ax[1,1].plot(extOneSeed)
	for flexOneSeed,extOneSeed in zip(taSpikes,gmSpikes):
		ax[0,0].plot(flexOneSeed)
		ax[1,0].plot(extOneSeed)
	ax[0,2].bar(0.4, np.array(taStat).mean(), 0.2, yerr=np.array(taStat).std())
	ax[1,2].bar(0.4, np.array(gmStat).mean(), 0.2, yerr=np.array(gmStat).std())
	ax[0,0].set_ylabel('Flexor')
	ax[0,0].set_title('Mn action potentials')
	ax[0,1].set_title('EMG response')
	ax[0,2].set_title('Statistic')
	ax[0,2].set_xlim([0,1])
	ax[1,0].set_ylabel('Extensor')
	plt.savefig(pathToResults+figName, format="pdf",transparent=True)
	if showPlot: plt.show()

if __name__ == '__main__':
	main()

max_stars_repo_name: neurorestore/neuraldynamicplatform

max_issues_repo_name: neurorestore/neuraldynamicplatform

max_forks_repo_name: neurorestore/neuraldynamicplatform


text:
import collections
import itertools
import os

from fileparser import tough3
from fileparser.fileread import FileReadSingle
import numpy as np

loca1 = r"C:\Users\tajayi3\OneDrive - Louisiana State University\Second numerical paper\flow through brine only"
loca2 = r"C:\Users\tajayi3\OneDrive - Louisiana State University\Second numerical paper\Python Run\Shale-Sand-Shale-Limestone"
loca3 = r"C:\Users\tajayi3\OneDrive - Louisiana State University\Second numerical paper\Python Run\Limestone-Dolomite-Limestone-Limestone"
loca4 = r"C:\Users\tajayi3\OneDrive - Louisiana State University\Second numerical paper\Python Run\An-Gy-Hal-Hal"

filetype_toughreact = 'kdd_conc.tec'

testcodetoughreact = FileReadSingle("toughreact", loca4, filetype_toughreact)

testcodetoughreact.plot2D('x', 'z', 'pH', 2.592e+2, 'grid')

loca3 = r"C:\Users\tajayi3\Desktop\Research\Coding Practice\write_to_TOUGH\RunScripts"
#loca3 = r"C:\Users\tajayi3\OneDrive - Louisiana State University\tmvoc\mymodels\Auto Simulations\AllComponents"
filetype_tmvoc = "OUTPUT_ELEME.csv"

# fileReader = tough3.Tough3('tmvoc', loca3, filetype_tmvoc)
#
# time_year = fileReader.convert_times('day')
# result_array = fileReader.get_timeseries_data('X_Toluen_L', 0)


text:
#!/usr/bin/env python3
"""
benchmark ecef2geodetic
"""
import time
from pymap3d.ecef import ecef2geodetic
import numpy as np
import argparse

ll0 = (42.0, 82.0)

def bench(N: int) -> float:

    x = np.random.random(N)
    y = np.random.random(N)
    z = np.random.random(N)

    tic = time.monotonic()
    lat, lon, alt = ecef2geodetic(x, y, z)

    return time.monotonic() - tic

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("N", type=int)
    p = p.parse_args()
    N = p.N

    print(f"ecef2geodetic: {bench(N):.3f} seconds")


text:
import awkward as ak
import numpy as np

def conn_to_awkward(conn_map):
    r = {}
    for station, sats in conn_map.items():
        local = {}
        for prn, connections in sats.items():
            conn_stack = []
            for conn in connections.connections:
                conn_stack.append(ak.from_numpy(conn.observations)[np.newaxis])

            if len(conn_stack) == 0:
                continue
            tmp = ak.concatenate(conn_stack)
            local[prn] = ak.from_regular(
                ak.zip({k: tmp[k] for k in ak.fields(tmp)}, depth_limit=1)[np.newaxis],
                axis=1,
            )

        r[station] = ak.zip(local, depth_limit=1)

    R = ak.zip(r, depth_limit=1)[0]
    return R

max_stars_repo_name: BenMueller/missile-tid

max_issues_repo_name: BenMueller/missile-tid

max_forks_repo_name: BenMueller/missile-tid


text:
import matplotlib.pyplot as plt
import numpy as np
import glob
from scipy.interpolate import interp1d
from scipy.signal import savgol_filter
from math import factorial
path ='/home/cui-group-wz/Documents/Code/MRO-Asyn-RL/plot/RL/HO'
path2 = '/home/cui-group-wz/Documents/Code/MRO-Asyn-RL/plot/BAN/HO'
path3 ='/home/cui-group-wz/Documents/Code/MRO-Asyn-RL/plot/RLonline/HO'
allFiles = glob.glob(path + "/*.csv")
allFiles2 = glob.glob(path2 + "/*.csv")
allFiles3 = glob.glob(path3 + "/*.csv")

data_list=[]
for file_ in allFiles:
    data = np.genfromtxt(file_, delimiter=',', skip_header=0,
                         skip_footer=0, names=['x', 'y'])  # ,max_rows=500
    data_list.append(data)
data2_list = []

n=0
for file2_ in allFiles2:

    data2 = np.genfromtxt(file2_, delimiter=',', skip_header=0,
                     skip_footer=0, names=['x2', 'y2'])
    data2_list.append(data2)
    n+=1
sum2 = 0
for j in range(n):
    sum2 = sum2 + sum(data2_list[j]['y2'][0:19])

data3_list = []
for file3_ in allFiles3:
     data3 = np.genfromtxt(file3_, delimiter=',', skip_header=0,
                      skip_footer=0, names=['x3', 'y3'])
     data3_list.append(data3)

ave = sum(data['y'])/1000
ave2 = sum2/n/20
print(ave2)
ave3 = sum(data3['y3'])/100
print(ave,ave3)
mean_values = [ave2,ave,ave3]
# print(data['y'],np.min(data['y']),np.max(data['y']))
# var = (np.max(data['y'])-ave)

# variance = [var]
bar_labels = ['UCB','a3c-offline','a3c-online']

x_pos = list(range(len(bar_labels)))

plt.bar(x_pos, mean_values, align='center', alpha=0.5)

plt.ylabel('Handover Rate')
plt.xticks(x_pos, bar_labels)

plt.legend()
plt.show()

reason: import numpy,from scipy


text:
import numpy as np
import matplotlib.pyplot as plt

from scipy.optimize import curve_fit
from scipy.signal import find_peaks
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import train_test_split

from qiskit import *
from qiskit.pulse import ShiftPhase,Play,Drag,GaussianSquare,DriveChannel,ControlChannel
import qiskit.pulse as pulse
import qiskit.pulse.library as pulse_lib
from qiskit.compiler import assemble
from qiskit.pulse.library import Waveform
from qiskit.tools.monitor import job_monitor
from qiskit_experiments.library import EFSpectroscopy, EFRabi

class sweep():

    def __init__(self, qubit, shots, backend, scale_factor):
        self.qubit = qubit
        self.shots = shots
        self.backend = backend
        self.scale_factor = scale_factor

        # Backend
        backend_config = self.backend.configuration()
        assert backend_config.open_pulse, "Backend doesn't support Pulse"
        self.dt = backend_config.dt
        backend_defaults = self.backend.defaults()
        self.default_qubit_freq = backend_defaults.qubit_freq_est[self.qubit]

        # Default measurement pulse
        qc_meas = QuantumCircuit(1,1)
        qc_meas.measure(0,0)
        transpiled_qc_meas = transpile(qc_meas, self.backend, initial_layout = [self.qubit])
        self.sched_meas = schedule(transpiled_qc_meas, backend)

        # Default pi pulse
        qc_x = QuantumCircuit(1)
        qc_x.x(0)
        transpiled_qc_x = transpile(qc_x, self.backend, initial_layout = [self.qubit])
        self.sched_x = schedule(transpiled_qc_x, self.backend)

        pulse_info = self.sched_x.instructions[0][1].pulse
        self.drive_samples = pulse_info.duration
        self.pi_amp_01 = pulse_info.amp
        self.drive_sigma = pulse_info.sigma

        self.times = 50

    def freq(self, amp, freq_med):
        if freq_med == None:
            ground_sweep_freqs = self.default_qubit_freq + np.linspace(-40e6, 40e6, self.times)
        else:
            ground_sweep_freqs = freq_med + np.linspace(-20e6, 20e6, self.times)

        ground_freq_sweep_program = self.create_ground_freq_sweep_program(ground_sweep_freqs, amp=amp)

        ground_freq_sweep_job = self.backend.run(ground_freq_sweep_program)
        job_monitor(ground_freq_sweep_job)
        ground_freq_sweep_data = self.get_job_data(ground_freq_sweep_job, average=True)

        return ground_freq_sweep_data, ground_sweep_freqs

    def fitting_freq(self, ground_freq_sweep_data, ground_sweep_freqs, parameters_real, parameters_imag):
        GHz = 1e9

        # Real fit
        (ground_sweep_fit_params_real,
         ground_sweep_i_fit_real) = self.fit_function_single(ground_sweep_freqs, np.real(ground_freq_sweep_data),
                                           lambda x, A, q_freq, B, C: (A / np.pi) * (B / ((x - q_freq)**2 + B**2)) + C,
                                           parameters_real, # initial parameters for curve_fit
                                           ([-np.inf, 0, 0, -np.inf], [np.inf, np.inf, np.inf, np.inf])) # ranges of parameters

        # Imag fit
        (ground_sweep_fit_params_imag,
         ground_sweep_i_fit_imag) = self.fit_function_single(ground_sweep_freqs, np.imag(ground_freq_sweep_data),
                                           lambda x, A, q_freq, B, C: (A / np.pi) * (B / ((x - q_freq)**2 + B**2)) + C,
                                           parameters_imag, # initial parameters for curve_fit
                                           ([-np.inf, 0, 0, -np.inf], [np.inf, np.inf, np.inf, np.inf])) # ranges of parameters

        print(ground_sweep_fit_params_real, ground_sweep_fit_params_imag)

        # 注：シグナルの実部だけをプロットします
        plt.scatter(ground_sweep_freqs/GHz, np.real(ground_freq_sweep_data), color='black')
        plt.plot(ground_sweep_freqs/GHz, ground_sweep_i_fit_real, color='red')
        plt.xlim([min(ground_sweep_freqs/GHz)+0.01, max(ground_sweep_freqs/GHz)]) # ignore min point (is off)
        plt.xlabel("Frequency [GHz]", fontsize=15)
        plt.ylabel("Measured Signal (Real) [a.u.]", fontsize=15)
        plt.title("0->1 Frequency Sweep (first pass)", fontsize=15)
        plt.show()

        # 注：シグナルの虚部だけをプロットします
        plt.scatter(ground_sweep_freqs/GHz, np.imag(ground_freq_sweep_data), color='black')
        plt.plot(ground_sweep_freqs/GHz, ground_sweep_i_fit_imag, color='red')
        plt.xlim([min(ground_sweep_freqs/GHz)+0.01, max(ground_sweep_freqs/GHz)]) # ignore min point (is off)
        plt.xlabel("Frequency [GHz]", fontsize=15)
        plt.ylabel("Measured Signal (Imag) [a.u.]", fontsize=15)
        plt.title("0->1 Frequency Sweep (first pass)", fontsize=15)
        plt.show()

        return ground_sweep_fit_params_real[1], ground_sweep_fit_params_imag[1]

    def create_ground_freq_sweep_program(self, freqs, amp):
        if len(freqs) > 75:
            raise ValueError("You can only run 75 schedules at a time.")

        # Define the drive pulse
        ground_sweep_drive_pulse = pulse_lib.gaussian(duration=self.drive_samples, sigma=self.drive_sigma,
                                                      amp=amp, name='ground_sweep_drive_pulse')
        # Create the base schedule
        schedule = pulse.Schedule(name='Frequency sweep starting from ground state.')
        schedule |= pulse.Play(ground_sweep_drive_pulse, DriveChannel(self.qubit))
        schedule |= self.sched_meas << schedule.duration

        # define frequencies for the sweep
        schedule_freqs = [{DriveChannel(self.qubit): freq} for freq in freqs]

        # assemble the program
        ground_freq_sweep_program = assemble(schedule, backend=self.backend, meas_level=1, meas_return='avg',
                                             shots=self.shots, schedule_los=schedule_freqs)

        return ground_freq_sweep_program

    def amp(self, rabi_freq, amp_med):
        if amp_med == None:
            drive_amp_min = 0
            drive_amp_max = 0.5
            drive_amps = np.linspace(drive_amp_min, drive_amp_max, self.times)
        else:
            drive_amps = np.linspace(0, amp_med+0.1, self.times)

        # スケジュールの作成
        rabi_01_schedules = []

        # loop over all drive amplitudes
        for ii, drive_amp in enumerate(drive_amps):
            # drive pulse
            rabi_01_pulse = pulse_lib.gaussian(duration=self.drive_samples, amp=drive_amp,
                                               sigma=self.drive_sigma, name='rabi_01_pulse_%d' % ii)

            # add commands to schedule
            schedule = pulse.Schedule(name='Rabi Experiment at drive amp = %s' % drive_amp)
            schedule |= pulse.Play(rabi_01_pulse, DriveChannel(self.qubit))
            schedule |= self.sched_meas << schedule.duration # shift measurement to after drive pulse
            rabi_01_schedules.append(schedule)

        rabi_01_expt_program = assemble(rabi_01_schedules, backend=self.backend, meas_level=1,
                                        meas_return='avg', shots=self.shots,
                                        schedule_los=[{DriveChannel(self.qubit): rabi_freq}] * self.times)

        rabi_01_job = self.backend.run(rabi_01_expt_program)
        job_monitor(rabi_01_job)
        rabi_01_data = self.get_job_data(rabi_01_job, average=True)

        return rabi_01_data, drive_amps

    def fitting_amp(self, rabi_12_data, drive_amps, parameters_real, parameters_imag):

        (rabi_12_fit_params_real, rabi_12_y_fit_real) = self.fit_function(drive_amps, np.real(rabi_12_data),
                                    lambda x, A, B, drive_12_period, phi: (A*np.cos(2*np.pi*x/drive_12_period - phi) + B),
                                    parameters_real)

        (rabi_12_fit_params_imag, rabi_12_y_fit_imag) = self.fit_function(drive_amps, np.imag(rabi_12_data),
                                    lambda x, A, B, drive_12_period, phi: (A*np.cos(2*np.pi*x/drive_12_period - phi) + B),
                                    parameters_imag)

        pi_amp_12_real = rabi_12_fit_params_real[2]/2
        pi_amp_12_imag = rabi_12_fit_params_imag[2]/2

        print(rabi_12_fit_params_real, rabi_12_fit_params_imag)

        plt.scatter(drive_amps, np.real(rabi_12_data), color='black')
        plt.plot(drive_amps, rabi_12_y_fit_real, color='red')
        plt.xlabel("Drive amp [a.u.]", fontsize=15)
        plt.ylabel("Measured signal (Real) [a.u.]", fontsize=15)
        plt.title('Rabi Experiment (0->1)', fontsize=20)
        plt.show()

        plt.scatter(drive_amps, np.imag(rabi_12_data), color='black')
        plt.plot(drive_amps, rabi_12_y_fit_imag, color='red')
        plt.xlabel("Drive amp [a.u.]", fontsize=15)
        plt.ylabel("Measured signal (Imag) [a.u.]", fontsize=15)
        plt.title('Rabi Experiment (0->1)', fontsize=20)
        plt.show()

        return pi_amp_12_real, pi_amp_12_imag

    def check(self, pi_pulse_01, rabi_freq):
        # 基底状態のスケジュール
        zero_schedule = pulse.Schedule(name="zero schedule")
        zero_schedule |= self.sched_meas

        # 励起状態のスケジュール
        one_schedule = pulse.Schedule(name="one schedule")
        one_schedule |= pulse.Play(pi_pulse_01, DriveChannel(self.qubit))
        one_schedule |= self.sched_meas << one_schedule.duration

        # プログラムにスケジュールを組み込みます
        IQ_01_program = assemble([zero_schedule, one_schedule], backend=self.backend, meas_level=1,
                                   meas_return='single', shots=self.shots, schedule_los=[{DriveChannel(self.qubit): rabi_freq}] * 2)

        IQ_01_job = self.backend.run(IQ_01_program)
        job_monitor(IQ_01_job)

        IQ_01_data = self.get_job_data(IQ_01_job, average=False)

        return IQ_01_data

    def IQ_01_plot(self, IQ_01_data, x_min, x_max, y_min, y_max):
        zero_data = IQ_01_data[0]
        one_data = IQ_01_data[1]

        """0、1のIQ平面をプロットするための補助関数。引数としてプロットの制限を与えます。
        """
        # 0のデータは青でプロット
        plt.scatter(np.real(zero_data), np.imag(zero_data),
                        s=5, cmap='viridis', c='blue', alpha=0.5, label=r'$|0\rangle$')
        # 1のデータは赤でプロット
        plt.scatter(np.real(one_data), np.imag(one_data),
                        s=5, cmap='viridis', c='red', alpha=0.5, label=r'$|1\rangle$')

        # 0、1の状態の結果の平均を大きなドットでプロット
        mean_zero = np.mean(zero_data) # 実部と虚部それぞれの平均をとる
        mean_one = np.mean(one_data)
        plt.scatter(np.real(mean_zero), np.imag(mean_zero),
                    s=200, cmap='viridis', c='black',alpha=1.0)
        plt.scatter(np.real(mean_one), np.imag(mean_one),
                    s=200, cmap='viridis', c='black',alpha=1.0)
        plt.xlim(x_min, x_max)
        plt.ylim(y_min,y_max)
        plt.legend()
        plt.ylabel('Imag [a.u.]', fontsize=15)
        plt.xlabel('Real [a.u.]', fontsize=15)
        plt.title("0-1 discrimination", fontsize=15)

    def discriminator(self, IQ_01_data):
        # IQベクトルを作成します（実部と虚部で構成されています）
        zero_data_reshaped = self.reshape_complex_vec(IQ_01_data[0])
        one_data_reshaped = self.reshape_complex_vec(IQ_01_data[1])

        IQ_01_data_copy = np.concatenate((zero_data_reshaped, one_data_reshaped))

        # （テスト用に）0と1の値が含まれたベクトルを構築します
        state_01 = np.zeros(self.shots) # 実験のショット数
        state_01 = np.concatenate((state_01, np.ones(self.shots)))

        # データをシャッフルして学習用セットとテスト用セットに分割します
        IQ_01_train, IQ_01_test, state_01_train, state_01_test = train_test_split(IQ_01_data_copy, state_01, test_size=0.5)

        # LDAを設定します
        LDA_01 = LinearDiscriminantAnalysis()
        LDA_01.fit(IQ_01_train, state_01_train)

        # 精度を計算します
        score_01 = LDA_01.score(IQ_01_test, state_01_test)
        print(score_01)

        return LDA_01

    # セパラトリックスを表示データの上にプロットします
    def separatrixPlot(self, lda, x_min, x_max, y_min, y_max):
        nx, ny = self.shots, self.shots

        xx, yy = np.meshgrid(np.linspace(x_min, x_max, nx), np.linspace(y_min, y_max, ny))
        Z = lda.predict_proba(np.c_[xx.ravel(), yy.ravel()])
        Z = Z[:, 1].reshape(xx.shape)

        plt.contour(xx, yy, Z, [0.5], linewidths=2., colors='black')

    def calibrated_x01(self, pi_amp_01):
        pi_pulse_01 = pulse_lib.gaussian(duration=self.drive_samples, amp=pi_amp_01, sigma=self.drive_sigma, name='pi_pulse_01')

        return pi_pulse_01

    def get_job_data(self, job, average):
        job_results = job.result() # タイムアウトパラメーターは120秒にセット
        result_data = []
        for i in range(len(job_results.results)):
            if average: # 平均データを得る
                result_data.append(job_results.get_memory(i)[0]*self.scale_factor)
            else: # シングルデータを得る
                result_data.append(job_results.get_memory(i)[:, 0]*self.scale_factor)
        return result_data

    @staticmethod
    def fit_function(x_values, y_values, function, init_params):
        """Fit a function using scipy curve_fit."""
        fitparams, conv = curve_fit(function, x_values, y_values, init_params)
        y_fit = function(x_values, *fitparams)

        return fitparams, y_fit

    @staticmethod
    def fit_function_single(x_values, y_values, function, init_params, ranges):
        """Fit a function using scipy curve_fit."""
        fitparams, conv = curve_fit(function, x_values, y_values, p0 = init_params, bounds = ranges)
        y_fit = function(x_values, *fitparams)

        return fitparams, y_fit

    @staticmethod
    def reshape_complex_vec(vec):
        length = len(vec)
        vec_reshaped = np.zeros((length, 2))
        for i in range(len(vec)):
            vec_reshaped[i]=[np.real(vec[i]), np.imag(vec[i])]

        return vec_reshaped

max_stars_repo_name: pearcandy/aqcel

max_issues_repo_name: pearcandy/aqcel

max_forks_repo_name: pearcandy/aqcel


text:
#!/usr/bin/env python
# encoding: utf-8

import sys
#指定路径
sys.path.append("../src/")

import graph
import embeddings
import sample
import classifier
from classifier import loop_dataset
import subgraphs
import argparse
import torch
import torch.optim as optim
import networkx as nx
import until
import random
from tqdm import tqdm

#导入配置参数
from parameters import parser, cmd_embed, cmd_opt

#参数转换
args = parser.parse_args()
args.cuda = not args.noCuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
    torch.cuda.manual_seed(args.seed)
if args.hop != 'auto':
    args.hop = int(args.hop)
if args.maxNodesPerHop is not None:
    args.maxNodesPerHop = int(args.maxNodesPerHop)

#读取数据
g = graph.Graph()
g.read_edgelist(filename=args.dataName, weighted=args.weighted, directed=args.directed)
g.read_node_status(filename=args.labelName)

#获取全图节点的Embedding
embed_args = cmd_embed.parse_args()
embeddings = embeddings.learn_embeddings(g, embed_args)
node_information = embeddings
#print node_information
#正负节点对采样
train, train_status, test, test_status = sample.sample_single(g, args.testRatio, max_train_num=args.maxTrainNum)

#抽取节点对的封闭子图
net = until.nxG_to_mat(g)
#print net
train_graphs, test_graphs, max_n_label = subgraphs.singleSubgraphs(net, train, train_status, test, test_status, args.hop, args.maxNodesPerHop, node_information)
print('# train: %d, # test: %d' % (len(train_graphs), len(test_graphs)))

#加载网络模型，并在classifier中配置相关参数
cmd_args = cmd_opt.parse_args()
cmd_args.feat_dim = max_n_label + 1
cmd_args.attr_dim = node_information.shape[1]
cmd_args.latent_dim = [int(x) for x in cmd_args.latent_dim.split('-')]
if len(cmd_args.latent_dim) == 1:
    cmd_args.latent_dim = cmd_args.latent_dim[0]
model = classifier.Classifier(cmd_args)
optimizer = optim.Adam(model.parameters(), lr=args.learningRate)

#训练和测试
train_idxes = list(range(len(train_graphs)))
best_loss = None
for epoch in range(args.num_epochs):
    random.shuffle(train_idxes)

    model.train()
    avg_loss = loop_dataset(train_graphs, model, train_idxes, cmd_args.batch_size, optimizer=optimizer)
    print('\033[92maverage training of epoch %d: loss %.5f acc %.5f auc %.5f\033[0m' % (epoch, avg_loss[0], avg_loss[1], avg_loss[2]))

    model.eval()
    test_loss = loop_dataset(test_graphs, model, list(range(len(test_graphs))), cmd_args.batch_size)
    print('\033[93maverage test of epoch %d: loss %.5f acc %.5f auc %.5f\033[0m' % (epoch, test_loss[0], test_loss[1], test_loss[2]))

reason: import networkx


text:
# -*- coding: utf-8 -*-
"""
    Confusion Matrix Plotting
    @author Sentinel
"""

import torch
import matplotlib.pyplot as plt
import numpy as np

class ConfusionMatrix:
    def __init__(self, class_num):
        self.mat = np.zeros((class_num, class_num))
        self.class_num = class_num

    def addRawElement(self, truth, pred):
        norms = torch.norm(pred.detach(), dim = -1)
        _, idx = norms.max(dim = 1)
        self.mat[truth.cpu().numpy(), idx.cpu().numpy()] += 1

    def addElement(self, truth, pred):
        self.mat[truth, pred] += 1

    def saveConfusionMatrix(self, path):
        plt.cla()
        plt.clf()
        plt.imshow(self.mat, cmap = 'inferno')
        plt.colorbar()
        plt.xlabel("Prediction result")
        plt.ylabel("Ground truth")
        plt.savefig(path)

max_stars_repo_name: Enigmatisms/CapsNet

max_issues_repo_name: Enigmatisms/CapsNet

max_forks_repo_name: Enigmatisms/CapsNet


text:
"""
data_ops

This file contains access to data and methods for assembly of data.

- Rakshit Agrawal, 2018
"""
import argparse
import os
import random
from collections import Counter, OrderedDict, defaultdict

import networkx as nx
import numpy as np
import pandas as pd
import scipy.io as sio
import tensorflow as tf

from log_control import *
from utils import Utils

kl = tf.keras.layers

DATA_DIR_DICT = {
    'seal': Utils.data_file('public_seal'),
    'snap': Utils.data_file('public_snap'),
    'snap_csv': Utils.data_file('public_snap'),
    'wsn': Utils.data_file('public_wsn')
}

DATA_PARSE = {
    'seal': 'mat',
    'snap': 'txt',
    'snap_csv': 'csv',
    'wsn': 'csv_wsn'
}

class DataHandling(object):

    def __init__(self,
                 dataset_name,
                 dataset_type='seal',
                 ordered_args=None):
        self.dataset_name = dataset_name
        self.dataset_type = dataset_type
        self.ordered_args = ordered_args

        self.g = None
        self.g_features = None

        if DATA_PARSE[self.dataset_type] == 'mat':
            self.adj_mat, self.g_features = self._load_adj_from_mat_file(dataset_name)
            self.g = nx.from_scipy_sparse_matrix(self.adj_mat, )
        elif DATA_PARSE[self.dataset_type] == 'txt':
            self.g = self._load_nxg_from_txt_file(dataset_name)
        elif DATA_PARSE[self.dataset_type] == 'csv':
            self.g = self._load_nxg_from_csv_edgelist(dataset_name)
        elif DATA_PARSE[self.dataset_type] == 'csv_wsn':
            self.g = self._load_nxg_from_csv_wsn(dataset_name)

        self.g = nx.convert_node_labels_to_integers(self.g, first_label=1)
        self.node_ref = {i: i for i in self.g.nodes}
        self.node_len = len(self.g.nodes)

        self.learning_dataset = None
        self.is_data_generated = False

        self.additional_embeddings = []

        if self.ordered_args.get('use_node_features', True):
            if self.g_features is not None:
                node_feature_layer = self._get_node_features_from_scipy_sparse_matrix(self.g_features)
                self.additional_embeddings.append(node_feature_layer)

        if len(self.additional_embeddings):
            logi("Additional embeddings will be used")

        self.baselines_only = False
        if self.ordered_args['for_baselines']:
            self.baselines_only = True

        self.metadata_file = None
        if self.ordered_args['visualize']:
            # Generate metadata file to visualize the data
            FIELDS = {'degree': nx.degree,
                      'centrality': nx.degree_centrality,
                      'triangles': nx.triangles}

            logi("Generating metadata for visualization using fields {}".format(FIELDS))

            field_dicts = {k: dict(v(self.g)) for k, v in FIELDS.items()}
            metadata_dict = defaultdict(dict)

            for field, fd in field_dicts.items():
                for k, v in fd.items():
                    metadata_dict[field][k] = v

            df = pd.DataFrame(metadata_dict)
            logi("Data frame shape: {}".format(df.shape))

            meta_filename = os.path.join(DATA_DIR_DICT[self.dataset_type], "meta_{}.tsv".format(self.dataset_name))

            df.to_csv(meta_filename, sep='\t', index_label='Node')
            logi("Metadata saved to {}".format(meta_filename))
            self.metadata_file = meta_filename

    def _load_adj_from_mat_file(self, dataset_name):
        data_file = os.path.join(DATA_DIR_DICT[self.dataset_type], '%s.mat' % dataset_name)
        file_data = sio.loadmat(data_file)

        if 'group' in file_data.keys():
            return file_data['net'], file_data['group']
        return file_data['net'], None

    def _load_nxg_from_csv_edgelist(self, dataset_name):
        data_file = os.path.join(DATA_DIR_DICT[self.dataset_type], '%s.csv' % dataset_name)
        graph = nx.Graph()
        data = pd.read_csv(data_file, header=None, index_col=None)
        for idx, row in data.iterrows():
            from_node, to_node = row[0], row[1]
            graph.add_edge(int(from_node), int(to_node))

        return graph

    def _load_nxg_from_csv_wsn(self, dataset_name):
        data_file = os.path.join(DATA_DIR_DICT[self.dataset_type], '%s.csv' % dataset_name)
        graph = nx.DiGraph()
        data = pd.read_csv(data_file, header=None, index_col=None, names=['from', 'to', 'rating'])
        for idx, row in data.iterrows():
            from_node, to_node, rating = row['from'], row['to'], row['rating']
            graph.add_edge(int(from_node), int(to_node), rating=rating)

        return graph

    def _load_nxg_from_txt_file(self, dataset_name):
        data_file = os.path.join(DATA_DIR_DICT[self.dataset_type], '%s.txt' % dataset_name)

        graph = nx.Graph()
        with open(data_file, 'r') as inp:
            for line in inp.readlines():
                if line.startswith("#"):
                    continue

                # Otherwise split by tab and get the from and to_nodes
                from_node, to_node = line.strip().split('\t')
                graph.add_edge(int(from_node), int(to_node))

        return graph

    def pad_data(self, data, vec_size, max_len):
        assert isinstance(data, np.ndarray)

        ret_vec = np.zeros((max_len, vec_size))
        if len(data):
            ret_vec[:min(len(data), max_len)] = data[:min(len(data), max_len)]
        return ret_vec

    def _nodes_to_ref(self, l):
        return map(lambda x: self.node_ref[x], l)

    def create_labeled_link_dataset(self, path_lengths, max_paths, shuffle=True, sample_size=None):
        """ Create the dataset. For each path length, create a separate list"""

        pos_data = set(self.g.edges)
        neg_data = set()
        for start in self.g.nodes:
            targets = random.sample(self.g.nodes, 10)
            targets = [i for i in targets if not i in nx.neighbors(self.g, start)]
            for i in targets:
                neg_data.add((start, i))

        logi("Raw\t Pos data: %d\t Neg data:%d" % (len(pos_data), len(neg_data)))
        if len(pos_data) < len(neg_data):
            neg_data = random.sample(neg_data, len(pos_data))
        logi("Balanced\t Pos data: %d\t Neg data:%d" % (len(pos_data), len(neg_data)))

        all_data = list(pos_data.union(neg_data))
        logi("Size of data: %d" % len(all_data))
        if shuffle:
            random.shuffle(all_data)

        node_pairs = []
        path_lists = {i: [] for i in path_lengths}
        labels = []

        if sample_size is not None:
            data = random.sample(all_data, sample_size) if sample_size < len(all_data) else all_data
        else:
            data = all_data

        max_path_len = max(path_lengths)
        for itr, (u, v) in enumerate(data):
            caches = {i: [] for i in path_lengths}
            label = 0
            try:
                for path in nx.shortest_simple_paths(self.g, source=u, target=v):
                    path_len = len(path)
                    if path_len > max_path_len:
                        break

                    path = self._nodes_to_ref(path)

                    if path_len == 1:
                        continue

                    if path_len == 2:
                        label = 1

                    else:
                        if path_len in caches.keys():
                            if len(caches[path_len]) < max_paths[path_len]:
                                caches[path_len].append(path)
                            else:
                                if path_len == max_path_len:
                                    break
                        else:
                            continue

            except nx.NetworkXNoPath as e:
                pass

            node_pairs.append(self._nodes_to_ref([u, v]))
            labels.append(label)
            assert len(node_pairs) == len(labels)
            for path_len in path_lengths:
                caches[path_len] = self.pad_data(np.asarray(caches[path_len]),
                                                 path_len, max_paths[path_len])
                path_lists[path_len].append(np.asarray(caches[path_len]))
                assert len(path_lists[path_len]) == len(labels)

            if np.random.random() > 0.999:
                logi("Label Distribution at step %d: %s" % (itr, Counter(labels)))

        logi("Final Label Distribution: %s" % (Counter(labels)))

        self.is_data_generated = True
        return node_pairs, path_lists, labels

    def create_labeled_wsn_dataset(self, path_lengths, max_paths, shuffle=True, sample_size=None):
        """ Create the dataset. For each path length, create a separate list

            For WSN, we do not need to collect any data from non-edge pairs.
            And since it is regression, there is no need for balance as well.
            So the rating of each edge is the label, and each edge is an entry.
            And from all the edges, we simply perform the split as needed.

        """
        all_data = list(self.g.edges)
        logi("Size of data: %d" % len(all_data))
        if shuffle:
            random.shuffle(all_data)

        node_pairs = []
        path_lists = {i: [] for i in path_lengths}
        # path_lists = defaultdict(list)
        labels = []

        if sample_size is not None:
            data = random.sample(all_data, sample_size) if sample_size < len(all_data) else all_data
        else:
            data = all_data

        max_path_len = max(path_lengths)
        for itr, (u, v) in enumerate(data):
            caches = {i: [] for i in path_lengths}
            # caches = defaultdict(list)
            label = 0  # Means no edge between the nodes
            try:
                for path in nx.shortest_simple_paths(self.g, source=u, target=v):
                    path_len = len(path)
                    if path_len > max_path_len:
                        break

                    path = self._nodes_to_ref(path)

                    if path_len == 1:
                        continue

                    if path_len == 2:
                        label = self.g[u][v]['rating']

                    else:
                        if path_len in caches.keys():
                            if len(caches[path_len]) < max_paths[path_len]:
                                caches[path_len].append(np.asarray(path, dtype=np.int32))
                            else:
                                if path_len == max_path_len:
                                    break
                        else:
                            continue

            except nx.NetworkXNoPath as e:
                pass

            node_pairs.append(self._nodes_to_ref([u, v]))
            labels.append(label)
            assert len(node_pairs) == len(labels)
            for path_len in path_lengths:
                caches[path_len] = self.pad_data(np.asarray(caches[path_len], dtype=np.int32),
                                                 path_len, max_paths[path_len])
                path_lists[path_len].append(np.asarray(caches[path_len], dtype=np.int32))
                assert len(path_lists[path_len]) == len(labels)

            if np.random.random() > 0.999:
                logi("Label Distribution at step %d: %s" % (itr, Counter(labels)))

        # logi("Final Label Distribution: \n", (np.histogram(labels, bins=20)))

        self.is_data_generated = True
        return node_pairs, path_lists, labels

    def create_labeled_link_dataset_for_baselines(self, path_lengths, max_paths, shuffle=True, sample_size=None):
        """ Create the dataset. For each path length, create a separate list"""

        pos_data = set(self.g.edges)
        neg_data = set()
        for start in self.g.nodes:
            targets = random.sample(self.g.nodes, 10)
            targets = [i for i in targets if not i in nx.neighbors(self.g, start)]
            for i in targets:
                neg_data.add((start, i))

        logi("Raw\t Pos data: %d\t Neg data:%d" % (len(pos_data), len(neg_data)))
        if len(pos_data) < len(neg_data):
            neg_data = random.sample(neg_data, len(pos_data))
        logi("Balanced\t Pos data: %d\t Neg data:%d" % (len(pos_data), len(neg_data)))

        all_data = list(pos_data.union(neg_data))
        logi("Size of data: %d" % len(all_data))
        if shuffle:
            random.shuffle(all_data)

        node_pairs = []
        path_lists = {}
        labels = []

        if sample_size is not None:
            data = random.sample(all_data, sample_size) if sample_size < len(all_data) else all_data
        else:
            data = all_data

        for itr, (u, v) in enumerate(data):
            caches = {i: [] for i in path_lengths}
            label = 1 if (u, v) in self.g.edges else 0

            node_pairs.append(self._nodes_to_ref([u, v]))
            labels.append(label)
            assert len(node_pairs) == len(labels)

            if np.random.random() > 0.999:
                logi("Label Distribution at step %d: %s" % (itr, Counter(labels)))

        logi("Final Label Distribution: %s" % (Counter(labels)))

        self.is_data_generated = True
        return node_pairs, path_lists, labels

    def create_labeled_wsn_dataset_for_baselines(self,
                                                 path_lengths, max_paths,
                                                 shuffle=True, sample_size=None):
        """ Create the dataset. For each path length, create a separate list

            For WSN, we do not need to collect any data from non-edge pairs.
            And since it is regression, there is no need for balance as well.
            So the rating of each edge is the label, and each edge is an entry.
            And from all the edges, we simply perform the split as needed.

        """
        all_data = list(self.g.edges)
        logi("Size of data: %d" % len(all_data))
        if shuffle:
            random.shuffle(all_data)

        node_pairs = []
        path_lists = {}
        labels = []

        if sample_size is not None:
            data = random.sample(all_data, sample_size) if sample_size < len(all_data) else all_data
        else:
            data = all_data

        for itr, (u, v) in enumerate(data):
            caches = {i: [] for i in path_lengths}
            # caches = defaultdict(list)
            label = self.g[u][v]['rating']

            node_pairs.append(self._nodes_to_ref([u, v]))
            labels.append(label)
            assert len(node_pairs) == len(labels)

            if np.random.random() > 0.999:
                logi("Label Distribution at step %d: %s" % (itr, Counter(labels)))

        # logi("Final Label Distribution: \n", (np.histogram(labels, bins=20)))

        self.is_data_generated = True
        return node_pairs, path_lists, labels

    def get_train_test_data(self, path_lengths,
                            max_paths,
                            shuffle=True,
                            problem='link',
                            sample_size=None,
                            split_point=0.9,
                            store=True):
        """ Create dataset for entire data at once,
            and retain in class object.
            This can then be re-partitioned for different split-points.
        """

        if self.learning_dataset is None:
            logi("Generating fresh dataset")
            if problem == 'link':

                if self.baselines_only:
                    self.learning_dataset = self.create_labeled_link_dataset_for_baselines(path_lengths,
                                                                                max_paths, shuffle=shuffle,
                                                                                sample_size=sample_size)
                else:
                    self.learning_dataset = self.create_labeled_link_dataset(path_lengths,
                                                                             max_paths, shuffle=shuffle,
                                                                             sample_size=sample_size)

            elif problem == 'wsn':
                if self.baselines_only:
                    self.learning_dataset = self.create_labeled_wsn_dataset_for_baselines(path_lengths,
                                                                                max_paths,
                                                                                shuffle=shuffle,
                                                                                sample_size=sample_size)
                else:
                    self.learning_dataset = self.create_labeled_wsn_dataset(path_lengths,
                                                                            max_paths, shuffle=shuffle,
                                                                            sample_size=sample_size)

        else:
            logi("Using pre-generated dataset")

        logi("Split point: {}".format(split_point))

        node_pairs, path_lists, labels = self.learning_dataset
        X = [node_pairs] + path_lists.values()
        Y = labels

        split_index = int(len(labels) * split_point)

        train_x = [np.asarray(i[:split_index]) for i in X]
        test_x = [np.asarray(i[split_index:]) for i in X]

        train_y = np.asarray(Y[:split_index])
        test_y = np.asarray(Y[split_index:])

        if store:
            train_x_file = "data/npy/{}_train_x.npy".format(self.dataset_name)
            train_y_file = "data/npy/{}_train_y.npy".format(self.dataset_name)
            test_x_file = "data/npy/{}_test_x.npy".format(self.dataset_name)
            test_y_file = "data/npy/{}_test_y.npy".format(self.dataset_name)

            np.save(train_x_file, train_x)
            np.save(train_y_file, train_y)
            np.save(test_x_file, test_x)
            np.save(test_y_file, test_y)

        logi("Sizes: Train = {}, Test = {}".format(len(train_x), len(test_x)))

        return train_x, train_y, test_x, test_y

    def _get_node_features_from_scipy_sparse_matrix(self, features, add_zero=True,
                                                    layer_name='node_features'):
        """ Get keras embedding layer of features from sparse scipy matrix"""
        dense_features = features.toarray()
        feature_dim = dense_features.shape[1]
        if add_zero:
            dense_features = np.vstack([np.zeros((1, feature_dim)), dense_features])

        return dense_features

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='LEAP data ops')

    parser.add_argument('--dataset', type=str, help="Dataset name", default="USAir")
    parser.add_argument('--dataset_type', type=str, help="Dataset type", default="seal")
    parser.add_argument('--path_lengths', type=list, help="Path lengths", default=[3, 4])
    parser.add_argument('--max_paths', type=dict, help="Max path size per length", default={3: 50, 4: 50})
    parser.add_argument('--store_data', type=bool, help="Store data", default=False)
    parser.add_argument('--visualize', type=bool, help="Visualize model", default=False)
    parser.add_argument('--problem', type=str, help="Type of problem", default='link')
    parser.add_argument('--sample_size', type=int, help="Sample size (2X links)", default=None)
    parser.add_argument('--for_baselines', type=bool, help="Perform for baselines", default=False)

    args = parser.parse_args()
    ordered_args = OrderedDict(vars(args))

    dh = DataHandling(dataset_name=ordered_args['dataset'], dataset_type=ordered_args['dataset_type'],
                      ordered_args=ordered_args)

    dx, dy, tdx, tdy = dh.get_train_test_data(path_lengths=ordered_args['path_lengths'],
                                              max_paths=ordered_args['max_paths'],
                                              store=ordered_args['store_data'],
                                              problem=ordered_args['problem'],
                                              sample_size=ordered_args['sample_size'])

max_stars_repo_name: rakshit-agrawal/LEAP

max_issues_repo_name: rakshit-agrawal/LEAP

max_forks_repo_name: rakshit-agrawal/LEAP

reason: import numpy,import scipy,import networkx


text:
import numpy as np
import scipy.signal as sig

class RefInput:
    """
    Base class for reference inputs
    """

    def __init__(self, param, t_init=0.0):
        self.param = param
        self.t_init = t_init
        self._count = -1
        self._last_count = -2

    @property
    def event(self):
        return self._count != self._last_count

    @property
    def count(self):
        return self._count

    @count.setter
    def count(self,value):
        self._last_count = self._count
        self._count = value

    @property
    def t_settle(self):
        try:
            t_settle = self.param['t_settle']
        except KeyError:
            t_settle = 0.0
        return t_settle

    @property
    def max_count(self):
        try:
            max_count = self.param['max_count']
        except KeyError:
            max_count = None
        return max_count

    @property
    def done(self):
        if self.max_count is not None:
            rval = self.count > self.max_count
        else:
            rval = False
        return rval

    def t_rel(self,t):
        return t - self.t_init - self.t_settle

    def is_trial(self,t):
        return self.t_rel(t) >= 0.0

    def velocity(self,t):
        t_rel = self.t_rel(t)
        if t_rel < 0.0:
            self.count = -1
        else:
            self.count = 0
        vx = 0.0
        vy = 0.0
        return np.array([vx, vy])

class NoMotion(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)

class DoneMotion(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)

    @property
    def done(self):
        return True

class BaseSeries(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)
        self.motion = DoneMotion({})
        self.motion_model = DoneMotion
        self._count = 0
        self._last_count = -1
        self.is_first = True

    @property
    def event(self):
        #return (self._count != self._last_count) and self.motion.event
        return self._count != self._last_count

    @property
    def done(self):
        return self.count == self.repetitions

    @property
    def repetitions(self):
        return self.param['repetitions']

    @property
    def cycles(self):
        return self.param['cycles']

    @property
    def motion_param(self):
        return {}

    def is_trial(self,t):
        return self.motion.t_rel(t) >= 0.0

    def next_motion(self,t):
        if not self.is_first:
            self.count += 1
        else:
            self.is_first = False
        if self.count < self.repetitions:
            motion = self.motion_model(self.motion_param, t_init=t)
        else:
            motion = DoneMotion({})
        return motion

    def velocity(self,t):
        vel = self.motion.velocity(t)
        if self.motion.done:
            self.motion = self.next_motion(t)
            vel = self.motion.velocity(t)
        else:
            self.count = self.count
        return vel

class Sin(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)

    @property
    def amplitude(self):
        return self.param['amplitude']

    @property
    def period(self):
        return self.param['period']

    def velocity(self,t):
        t_rel = self.t_rel(t)
        vx = 0.0
        vy = 0.0
        if t_rel >= 0.0:
            self.count = int(np.floor(t_rel/self.period))
            vx = self.amplitude*np.sin(2.0*np.pi*t_rel/self.period)
        else:
            self.count = -1
        return np.array([vx, vy])

class SinSeries(BaseSeries):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)
        self.motion_model = Sin

    @property
    def motion_param(self):
        motion_param = {
                'amplitude' : self.amplitude(self.count),
                'period'    : self.period(self.count),
                't_settle'  : self.t_settle,
                'max_count' : self.cycles-1,
                }
        return motion_param

    def amplitude(self, num):
        return self.param['amplitude']

    def period(self, num):
        return self.param['period']

class SinPeriodSeries(SinSeries):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)
        self.motion = DoneMotion({})
        self.is_first = True

    @property
    def period_list(self):
        return self.param['period_list']

    @property
    def repetitions(self):
        return len(self.period_list)

    def period(self, num):
        return self.period_list[num]

class Step(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)

    @property
    def amplitude(self):
        return self.param['amplitude']

    @property
    def period(self):
        return self.param['period']

    def velocity(self,t):
        t_rel = self.t_rel(t)
        vx = 0.0
        vy = 0.0
        if t_rel >= 0.0:
            s = float(t_rel%self.period)/self.period
            self.count = int(np.floor(t_rel/self.period))
            if s <= 0.5:
                sign = 1.0
            else:
                sign = -1.0
            vx = sign*self.amplitude
        else:
            self.count = -1
        return np.array([vx,vy])

class StepSeries(BaseSeries):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)
        self.motion_model = Step

    @property
    def motion_param(self):
        motion_param = {
                'amplitude' : self.amplitude(self.count),
                'period'    : self.period(self.count),
                't_settle'  : self.t_settle,
                'max_count' : self.cycles-1,
                }
        return motion_param

    def amplitude(self, num):
        return self.param['amplitude']

    def period(self, num):
        return self.param['period']

class StepZeroStep(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)

    @property
    def amplitude(self):
        return self.param['amplitude']

    @property
    def t_step(self):
        return self.param['t_step']

    @property
    def t_zero(self):
        return self.param['t_zero']

    @property
    def t_total(self):
        return 2*(self.t_step + self.t_zero)

    def velocity(self,t):
        t_rel = self.t_rel(t)
        vx = 0.0
        vy = 0.0
        if t_rel >= 0.0:
            t_mod = t_rel % self.t_total
            self.count = int(np.floor(t_rel/(0.5*self.t_total)))
            if t_mod < self.t_step:
                vx = self.amplitude
            elif t_mod < (self.t_step + self.t_zero):
                vx = 0.0
            elif t_mod < (2*self.t_step + self.t_zero):
                vx = -self.amplitude
            else:
                vx = 0.0
        else:
            self.count = -1
        return np.array([vx, vy])

class RandomStep(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)
        self.t_last = None
        self.duration = None
        self.cur_velocity = None

    @property
    def max_velocity(self):
        return self.param['max_velocity']

    @property
    def min_velocity(self):
        return self.param['min_velocity']

    @property
    def max_duration(self):
        return self.param['max_duration']

    @property
    def min_duration(self):
        return self.param['min_duration']

    def velocity(self,t):
        t_rel = self.t_rel(t)
        vx = 0.0
        vy = 0.0
        if t_rel >= 0.0:
            if (self.t_last is None) or (t_rel > (self.t_last + self.duration)):
                self.count += 1
                self.t_last = t_rel
                rand_val = np.random.rand()
                delta_duration = self.max_duration - self.min_duration
                self.duration = self.min_duration + rand_val*delta_duration
                rand_val = np.random.rand()
                delta_velocity = self.max_velocity - self.min_velocity
                self.curr_velocity = self.min_velocity + rand_val*delta_velocity
            vx = self.curr_velocity
        else:
            self.count = -1
        return np.array([vx,vy])

class CyclicChirp(RefInput):

    def __init__(self, param, t_init=0.0):
        super().__init__(param, t_init=t_init)

    @property
    def amplitude(self):
        return self.param['amplitude']

    @property
    def min_freq(self):
        return self.param['min_freq']

    @property
    def max_freq(self):
        return self.param['max_freq']

    @property
    def n_cycles(self):
        return int(self.param['n_cycles'])

    @property
    def method(self):
        return self.param['method']

    @property
    def chirp_t1(self):
        if self.method == 'linear':
            val =  2.0*self.n_cycles/(self.max_freq + self.min_freq)
        elif self.method == 'logarithmic':
            val = self.n_cycles*np.log(self.max_freq/self.min_freq)
            val = val/(self.max_freq - self.min_freq)
        return val

    @property
    def period(self):
        return 2*self.chirp_t1

    def velocity(self,t):
        t_rel = self.t_rel(t)
        vx = 0.0
        vy = 0.0
        if t_rel >= 0.0:
            self.count = int(np.floor(t_rel/self.period))
            if t_rel%self.period <= 0.5*self.period:
                s = t_rel%(0.5*self.period)
                f0 = self.min_freq
                t1 = self.chirp_t1
                f1 = self.max_freq
                vx = sig.chirp(s,f0,t1,f1,method=self.method,phi=-90)
            else:
                s = t_rel%self.period - 0.5*self.period
                f0 = self.max_freq
                t1 = self.chirp_t1
                f1 = self.min_freq
                vx = sig.chirp(s,f0,t1,f1,method=self.method,phi=-90)
            vx = self.amplitude*vx
        else:
            self.count = -1
        return np.array([vx,vy])

# -----------------------------------------------------------------------------
if __name__ == '__main__':

    import matplotlib.pyplot as plt

    param = {
            'amplitude'  : 400.0,
            'min_freq'   : 1/60.0,
            'max_freq'   : 1/1.0,
            'n_cycles'   : 10,
            't_settle'   : 10.0,
            'method'     : 'logarithmic',
            }

    motion_model = CyclicChirp(param)
    print('fmin', motion_model.min_freq)
    print('fmax', motion_model.max_freq)
    print('tmax', 1/motion_model.min_freq)
    print('tmin', 1/motion_model.max_freq)
    print('T/2 ', motion_model.period/2)
    print('T   ', motion_model.period)

    #assert 1==0

    t = np.linspace(0.0,2*motion_model.period, 2000)
    vel = [motion_model.velocity(item) for item in t]
    vx = [v[0] for v in vel]

    fig, ax = plt.subplots(1,1,sharex=True)
    ax.plot(t, vx)
    ax.grid(True)
    ax.set_xlabel('t (sec)')
    ax.set_ylabel('value')
    ax.set_title('cyclic chirp')
    plt.show()

max_stars_repo_name: willdickson/imafly

max_issues_repo_name: willdickson/imafly

max_forks_repo_name: willdickson/imafly


text:
"""
Created on Mon Oct 25 12:03:05 2021

@author: cbadenes

Training Data Set: https://cogcomp.seas.upenn.edu/Data/QA/QC/
"""

import os
import sys
import logging
import pandas
import csv
import spacy

from sklearn.naive_bayes import GaussianNB
from sklearn.svm import LinearSVC
import joblib
from scipy.sparse import csr_matrix

logger = logging.getLogger(__name__)

def get_data_info(question_df):
    logger.debug("\n{0}".format(question_df.head()))
    logger.debug("\n{0}".format(question_df.info()))
    logger.debug("\n{0}".format(question_df.describe()))
    logger.debug("\n{0}".format(question_df.columns))

def pre_process(question_df):
    return pandas.get_dummies(question_df)

def transform_data_matrix(df_question_train):

    # Generate Compressed Sparse Row matrix:
    # https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html
    logger.debug("Training data: {0}".format(df_question_train.shape))
    df_question_train = csr_matrix(df_question_train)

    return df_question_train

def naive_bayes_classifier(df_question_train, df_question_class):
    gnb = GaussianNB()
    gnb.fit(df_question_train, df_question_class)
    logger.info("Gaussian Naive Bayes: {0}".format(gnb))

    return gnb

def support_vector_machine(df_question_train, df_question_class):
    lin_clf = LinearSVC()
    lin_clf.fit(df_question_train, df_question_class)
    logger.info("Linear SVC: {0}".format(lin_clf))

    return lin_clf

def save_classifier_model(df_question_train, df_question_class, model_type="linearSVC"):

    """
    FIXME: Although the classifier is being saved in Pickle file. It is not being used to predict.
    Since, Support Vector Classifier, fails when it encounters new features it failed to see while training.
    """

    classifier_model = None
    training_model_path = "./question_classifier.pkl"

    if model_type == "linearSVC":
        classifier_model = support_vector_machine(df_question_train, df_question_class)
    else:
        logger.error("Undefined Classifier")

    if classifier_model is not None:
        joblib.dump(classifier_model, training_model_path)
        logger.info("Model saved at {0}".format(training_model_path))
    else:
        logger.error("Model empty")

def remove_irrelevant_features(df_question):
    df_question_class = df_question.pop('Class')

    df_question.pop('Question')
    df_question.pop('WH-Bigram')

    return df_question_class

def train_question_classifier(training_data_path):
    """
    Question Classifier based on its feature.
    CSV: Training Data `qclassifier_trainer.csv`
        #Question|WH|WH-Bigram|WH-POS|WH-NBOR-POS|Root-POS|Class
    Using: Linear Support Vector Machine
    Model: Saved as Pickle `question_classifier.pkl`
    """

    df_question = pandas.read_csv(training_data_path, sep='|', header=0, encoding = "ISO-8859-1")

    get_data_info(df_question)

    df_question_class = remove_irrelevant_features(df_question)

    df_question_train = pre_process(df_question)

    df_question_train = transform_data_matrix(df_question_train)

    save_classifier_model(df_question_train, df_question_class)

def read_input_file(raw_data_file, training_data_path, en_nlp):

    with open(training_data_path, 'a', newline='', encoding = "ISO-8859-1") as csv_fp:
        csv_fp_writer = csv.writer(csv_fp, delimiter='|')
        for row in raw_data_file:
            list_row = row.split(" ")
            question_class_list = list_row[0].split(":")
            question = " ".join(list_row[1:len(list_row)])
            question = question.strip("\n")
            question_class = question_class_list[0]

            process_question(question, question_class, en_nlp, training_data_path, csv_fp_writer)

        csv_fp.close()

def process_question(question, question_class, en_nlp, training_data_path, csv_fp_writer):
    en_doc = en_nlp(u'' + question)
    sentence_list = list(en_doc.sents)

    # Currently question classifier classifies only the 1st sentence of the question
    sentence = sentence_list[0]

    wh_bi_gram = []
    root_token, wh_pos, wh_nbor_pos, wh_word = [""] * 4
    missing_wh = True

    for token in sentence:

        # if token is of WH question type
        if token.tag_ == "WDT" or token.tag_ == "WP" or token.tag_ == "WP$" or token.tag_ == "WRB":
            wh_pos = token.tag_
            wh_word = token.text
            wh_bi_gram.append(token.text)
            wh_bi_gram.append(str(en_doc[token.i + 1]))
            wh_nbor_pos = en_doc[token.i + 1].tag_
        elif missing_wh and token.dep_ == "ROOT":
            missing_wh = False
            wh_pos = "WDT"
            wh_word = token.text
            wh_bi_gram.append(token.text)
            wh_bi_gram.append(str(en_doc[token.i + 1]))
            wh_nbor_pos = en_doc[token.i + 1].tag_

        # if token is the root of sentence
        if token.dep_ == "ROOT":
            root_token = token.tag_

    if wh_word != "" and " ".join(wh_bi_gram) != "" and wh_pos != "" and wh_nbor_pos != "":
        csv_fp_writer.writerow([question, wh_word, " ".join(wh_bi_gram), wh_pos, wh_nbor_pos, root_token, question_class])
    else:
        print("wh_word:",wh_word, ", wh_bi_gram:", " ".join(wh_bi_gram), "wh_pos:", wh_pos, "wh_nbor_pos:",wh_nbor_pos )
        for token in sentence:
            print(token, token.dep_)
        logger.error("Extraction failed: {0}:{1}".format(question, question_class))

def clean_old_data(training_data_path):

    question_features = ['Question', 'WH', 'WH-Bigram', 'WH-POS', 'WH-NBOR-POS', 'Root-POS', 'Class']

    with open(training_data_path, 'w', newline='', encoding = "ISO-8859-1") as csv_fp:
        csv_fp_writer = csv.writer(csv_fp, delimiter='|')
        csv_fp_writer.writerow(question_features)
        csv_fp.close()

def extract_training_features(raw_data_path, training_data_path, en_nlp):
    with open(raw_data_path, 'r', encoding = "ISO-8859-1") as fp:
        read_input_file(fp, training_data_path, en_nlp)
        fp.close()
        logger.info("Extracted features from raw data.")
        logger.info("Excluded data where features failed to extract.")

if __name__ == "__main__":

    from time import time

    logging.basicConfig(level=logging.DEBUG)
    if len(sys.argv) > 1:
        start_time = time()

        should_extract = sys.argv[1]

        training_path = "./train_5500.csv"
        raw_path = "./train_5500.label"

        if should_extract:
            logger.info("Cleaning enabled.")
            clean_old_data(training_path)
            en_nlp_l = spacy.load("en_core_web_lg")

            extract_training_features(raw_path, training_path, en_nlp_l)

        train_question_classifier(training_path)

        end_time = time()
        logger.info("Total training time : {0}".format(end_time - start_time))
    else:
        raise ValueError('Missing option to enable to disable feature extraction')

max_stars_repo_name: librairy/explainable-qa

max_issues_repo_name: librairy/explainable-qa

max_forks_repo_name: librairy/explainable-qa

reason: from scipy


text:
# -*- coding: utf-8 -*-
# vispy: testskip
# -----------------------------------------------------------------------------
# Copyright (c) 2015, Vispy Development Team.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------
"""
Compare an optimal pan/zoom implementation to the same functionality
provided by scenegraph.

Use --vispy-cprofile to see an overview of time spent in all functions.
Use util.profiler and --vispy-profile=ClassName.method_name for more directed
profiling measurements.
"""
import numpy as np
import math

from vispy import gloo, app, scene
from vispy.visuals import Visual
from vispy.visuals.shaders import Function, Variable
from vispy.visuals.transforms import TransformSystem, BaseTransform
from vispy.util.profiler import Profiler

class PanZoomTransform(BaseTransform):
    glsl_map = """
        vec4 pz_transform_map(vec4 pos) {
            return vec4($zoom * (pos.xy + $pan), 0., 1.);
        }
    """

    glsl_imap = """
        vec4 pz_transform_imap(vec2 pos) {
            return vec4(pos / $zoom - $pan, 0, 1);
        }
    """

    Linear = True
    Orthogonal = True
    NonScaling = False
    Isometric = False

    def __init__(self):
        super(PanZoomTransform, self).__init__()
        self._pan = None
        self._zoom = None

    @property
    def pan(self):
        if isinstance(self._pan, Variable):
            return np.array(self._pan.value, dtype=np.float32)
        else:
            raise NotImplementedError()

    @pan.setter
    def pan(self, value):
        if isinstance(value, Variable):
            self._pan = value
            self._shader_map['pan'] = self._pan
        elif isinstance(self._pan, Variable):
            self._pan.value = value
        else:
            raise NotImplementedError()

    @property
    def zoom(self):
        if isinstance(self._zoom, Variable):
            return np.array(self._zoom.value, dtype=np.float32)
        else:
            raise NotImplementedError()

    @zoom.setter
    def zoom(self, value):
        if isinstance(value, Variable):
            self._zoom = value
            self._shader_map['zoom'] = self._zoom
        elif isinstance(self._zoom, Variable):
            self._zoom.value = value
        else:
            raise NotImplementedError()

    def map(self, coords):
        if not isinstance(coords, np.ndarray):
            coords = np.array(coords)
        return self.zoom[None, :] * (coords + self.pan[None, :])

    def imap(self, coords):
        if not isinstance(coords, np.ndarray):
            coords = np.array(coords)
        return (coords / self.zoom[None, :]) - self.pan[None, :]

class PanZoomCanvas(app.Canvas):
    def __init__(self, **kwargs):
        super(PanZoomCanvas, self).__init__(keys='interactive', **kwargs)
        self._visuals = []

        self._pz = PanZoomTransform()
        self._pz.pan = Variable('uniform vec2 u_pan', (0, 0))
        self._pz.zoom = Variable('uniform vec2 u_zoom', (1, 1))

        self.width, self.height = self.size
        self.context.set_viewport(0, 0, self.physical_size[0],
                                  self.physical_size[1])
        self.context.set_state(clear_color='black', blend=True,
                               blend_func=('src_alpha', 'one_minus_src_alpha'))

        self._tr = TransformSystem(self)
        self.show()

    def on_resize(self, event):
        self.width, self.height = event.size
        self.context.set_viewport(0, 0, event.physical_size[0],
                                  event.physical_size[1])

    def _normalize(self, x_y):
        x, y = x_y
        w, h = float(self.width), float(self.height)
        return x/(w/2.)-1., y/(h/2.)-1.

    def bounds(self):
        pan_x, pan_y = self._pz.pan
        zoom_x, zoom_y = self._pz.zoom
        xmin = -1 / zoom_x - pan_x
        xmax = +1 / zoom_x - pan_x
        ymin = -1 / zoom_y - pan_y
        ymax = +1 / zoom_y - pan_y
        return (xmin, ymin, xmax, ymax)

    def on_mouse_move(self, event):
        if event.is_dragging and not event.modifiers:
            x0, y0 = self._normalize(event.press_event.pos)
            x1, y1 = self._normalize(event.last_event.pos)
            x, y = self._normalize(event.pos)
            dx, dy = x - x1, -(y - y1)
            button = event.press_event.button

            pan_x, pan_y = self._pz.pan
            zoom_x, zoom_y = self._pz.zoom

            if button == 1:
                self._pz.pan = (pan_x + dx/zoom_x,
                                pan_y + dy/zoom_y)
            elif button == 2:
                zoom_x_new, zoom_y_new = (zoom_x * math.exp(2.5 * dx),
                                          zoom_y * math.exp(2.5 * dy))
                self._pz.zoom = (zoom_x_new, zoom_y_new)
                self._pz.pan = (pan_x - x0 * (1./zoom_x - 1./zoom_x_new),
                                pan_y + y0 * (1./zoom_y - 1./zoom_y_new))
            self.update()

    def on_mouse_wheel(self, event):
        prof = Profiler()  # noqa
        if not event.modifiers:
            dx = np.sign(event.delta[1])*.05
            x0, y0 = self._normalize(event.pos)
            pan_x, pan_y = self._pz.pan
            zoom_x, zoom_y = self._pz.zoom
            zoom_x_new, zoom_y_new = (zoom_x * math.exp(2.5 * dx),
                                      zoom_y * math.exp(2.5 * dx))
            self._pz.zoom = (zoom_x_new, zoom_y_new)
            self._pz.pan = (pan_x - x0 * (1./zoom_x - 1./zoom_x_new),
                            pan_y + y0 * (1./zoom_y - 1./zoom_y_new))
            self.update()

    def on_key_press(self, event):
        if event.key == 'R':
            self._pz.zoom = (1., 1.)
            self._pz.pan = (0., 0.)
            self.update()

    def add_visual(self, name, value):
        value.shared_program.vert['transform'] = self._pz
        value.events.update.connect(self.update)
        self._visuals.append(value)

    def __setattr__(self, name, value):
        if isinstance(value, Visual):
            self.add_visual(name, value)
        super(PanZoomCanvas, self).__setattr__(name, value)

    @property
    def visuals(self):
        return self._visuals

    def on_draw(self, event):
        prof = Profiler()
        self.context.clear()
        for visual in self.visuals:
            visual.draw()
            prof('draw visual')

X_TRANSFORM = """
float get_x(float x_index) {
    // 'x_index' is between 0 and nsamples.
    return -1. + 2. * x_index / (float($nsamples) - 1.);
}
"""

Y_TRANSFORM = """
float get_y(float y_index, float sample) {
    // 'y_index' is between 0 and nsignals.
    float a = float($scale) / float($nsignals);
    float b = -1. + 2. * (y_index + .5) / float($nsignals);

    return a * sample + b;
}
"""

DISCRETE_CMAP = """
vec3 get_color(float index) {
    float x = (index + .5) / float($ncolors);
    return texture2D($colormap, vec2(x, .5)).rgb;
}
"""

class SignalsVisual(Visual):
    VERTEX_SHADER = """
    attribute float a_position;

    attribute vec2 a_index;
    varying vec2 v_index;

    uniform float u_nsignals;
    uniform float u_nsamples;

    void main() {
        vec4 position = vec4($get_x(a_index.y),
                             $get_y(a_index.x, a_position), 0., 1.);
        gl_Position = $transform(position);

        v_index = a_index;
    }
    """

    FRAGMENT_SHADER = """
    varying vec2 v_index;

    void main() {
        gl_FragColor = vec4($get_color(v_index.x), 1.);

        // Discard vertices between two signals.
        if ((fract(v_index.x) > 0.))
            discard;
    }
    """

    def __init__(self, data):
        Visual.__init__(self, self.VERTEX_SHADER, self.FRAGMENT_SHADER)

        nsignals, nsamples = data.shape
        # nsamples, nsignals = data.shape

        self._data = data

        a_index = np.c_[np.repeat(np.arange(nsignals), nsamples),
                        np.tile(np.arange(nsamples), nsignals)
                        ].astype(np.float32)

        # Doesn't seem to work nor to be very efficient.
        # indices = nsignals * np.arange(nsamples)
        # indices = indices[None, :] + np.arange(nsignals)[:, None]
        # indices = indices.flatten().astype(np.uint32)
        # self._ibuffer = gloo.IndexBuffer(indices)

        self._buffer = gloo.VertexBuffer(data.reshape(-1, 1))
        self.shared_program['a_position'] = self._buffer
        self.shared_program['a_index'] = a_index

        x_transform = Function(X_TRANSFORM)
        x_transform['nsamples'] = nsamples
        self.shared_program.vert['get_x'] = x_transform

        y_transform = Function(Y_TRANSFORM)
        y_transform['scale'] = Variable('uniform float u_signal_scale', 1.)
        y_transform['nsignals'] = nsignals
        self.shared_program.vert['get_y'] = y_transform
        self._y_transform = y_transform

        colormap = Function(DISCRETE_CMAP)
        rng = np.random.RandomState(0)
        cmap = rng.uniform(size=(1, nsignals, 3),
                           low=.5, high=.9).astype(np.float32)
        tex = gloo.Texture2D((cmap * 255).astype(np.uint8))
        colormap['colormap'] = Variable('uniform sampler2D u_colormap', tex)
        colormap['ncolors'] = nsignals
        self.shared_program.frag['get_color'] = colormap
        self._draw_mode = 'line_strip'
        self.set_gl_state('translucent', depth_test=False)

    @property
    def data(self):
        return self._data

    @data.setter
    def data(self, value):
        self._data = value
        self._buffer.set_subdata(value.reshape(-1, 1))
        self.update()

    @property
    def signal_scale(self):
        return self._y_transform['scale'].value

    @signal_scale.setter
    def signal_scale(self, value):
        self._y_transform['scale'].value = value
        self.update()

    def _prepare_draw(self, view=None):
        """This method is called immediately before each draw.

        The *view* argument indicates which view is about to be drawn.
        """
        pass

Signals = scene.visuals.create_visual_node(SignalsVisual)

if __name__ == '__main__':
    data = np.random.normal(size=(128, 1000)).astype(np.float32)

    pzcanvas = PanZoomCanvas(position=(400, 300), size=(800, 600),
                             title="PanZoomCanvas", vsync=False)
    visual = SignalsVisual(data)
    pzcanvas.add_visual('signal', visual)

    scanvas = scene.SceneCanvas(show=True, keys='interactive',
                                title="SceneCanvas", vsync=False)
    view = scanvas.central_widget.add_view('panzoom')
    svisual = Signals(data, parent=view.scene)
    view.camera.set_range([-0.9, 0.9], [-0.9, 0.9])

    import sys
    if sys.flags.interactive != 1:
        app.run()

max_stars_repo_name: chongxi/vispy

max_issues_repo_name: chongxi/vispy


text:
import numpy as np
from chainer0.function import Function
from chainer0.functions import sum

def MeanSquaredError(Function):

    def __call__(self, *inputs):
        x, t = inputs
        return mean_squared_error(x, t)

def mean_squared_error(x, t):
    diff = x - t
    total = sum(diff * diff)
    N = x.data.shape[0]
    return total / N


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
#from pylab import show as sh_show
from scipy.cluster.vq import vq
from scipy.cluster.vq import kmeans
import numpy as np
import pandas as pd
np.set_printoptions(precision=3)

base_path               = '/home/imademethink/prg05_scipy/'
file_raw_data           = base_path + 'temperature_data.csv'
temperature_date_df     = pd.read_csv(file_raw_data,
                                  header='infer',sep=',',index_col=False,
                                  dtype = 'object')
temperature_date_values = temperature_date_df.values

# get the temperature and day number of the year
temperature_date_values = np.append(
                                temperature_date_values[:,:-1],
                                np.arange(1,366).reshape(365,1),
                                axis=1)

# plotting raw values
plt.plot(temperature_date_values[:,-2])
plt.xticks(np.arange(0, temperature_date_values.size, 1.0))
plt.ylabel('original values')
plt.show()

# get columnwise data
c1               = pd.Series(temperature_date_values[:,2:3].ravel())
only_day_count   = pd.to_numeric(c1, errors='ignore')
c2               = pd.Series(temperature_date_values[:,1:2].ravel())
only_temperature = pd.to_numeric(c2, errors='ignore')

# merge column wise data
data = np.hstack((
        only_day_count.values.reshape(365,1),
        only_temperature.values.reshape(365,1)))
centroid_count      = 2
iterations_kmeans   = 1000
threshold_kmeans    = 0.01
# computing K-Means with K = 2 (2 clusters)
for x in range(0, 1):
    centroids,distortion = kmeans(
            data,
            centroid_count,
            iterations_kmeans,
            threshold_kmeans)
    print("centroids  =\n",centroids)
    print("distortion =\n",distortion)
# assign each sample to a cluster
idx,_ = vq(data,centroids)
print("idx  =\n",idx)

# some plotting using numpy's logical indexing
plt.plot(data[idx==0,0],data[idx==0,1],'ob',
         data[idx==1,0],data[idx==1,1],'or')
plt.plot(centroids[:,0],
         centroids[:,1],'sg',markersize=8)
plt.show()

# use this for 3 centroids when data= data6
plt.plot(data[idx==0,0],data[idx==0,1],'ob',
     data[idx==1,0],data[idx==1,1],'or',
     data[idx==2,0],data[idx==2,1],'og') # third cluster points
plt.plot(centroids[:,0],centroids[:,1],'sm',markersize=8)
plt.show()


text:
#
# Copyright (c) 2017-2018 cTuning foundation.
# See CK COPYRIGHT.txt for copyright details.
#
# SPDX-License-Identifier: BSD-3-Clause.
# See CK LICENSE.txt for licensing details.
#
#
# SqueezeNet v1.1 (signature pool 1/3/5)
# Implementation is based on https://github.com/avoroshilov/tf-squeezenet
#

import tensorflow as tf
import numpy as np
import scipy.io

WEIGHTS = {}

#-----------------------------------------------------------------------

def load_weights(data_path):
    global WEIGHTS

    weights_raw = scipy.io.loadmat(data_path)

    # Converting to needed type
    dtype = np.float32
    for name in weights_raw:
        WEIGHTS[name] = []
        # skipping '__version__', '__header__', '__globals__'
        if name[0:2] != '__':
            kernels, bias = weights_raw[name][0]
            WEIGHTS[name].append( kernels.astype(dtype) )
            WEIGHTS[name].append( bias.astype(dtype) )

#-----------------------------------------------------------------------

def get_mean_value():
  return np.array([104.006, 116.669, 122.679], dtype=np.float32)

#-----------------------------------------------------------------------

def get_weights_biases(layer_name):
    weights, biases = WEIGHTS[layer_name]
    biases = biases.reshape(-1)
    return (weights, biases)

#-----------------------------------------------------------------------

def inference(input_image):
    x = tf.cast(input_image, tf.float32)

    ### Feature extractor

    # conv1 cluster
    layer_name = 'conv1'
    weights, biases = get_weights_biases(layer_name)
    x = _conv_layer(layer_name + '_conv', x, weights, biases, padding='VALID', stride=(2, 2))
    x = _act_layer(layer_name + '_actv', x)
    x = _pool_layer('pool1_pool', x)

    # fire2 + fire3 clusters
    x = _fire_cluster('fire2', x)
    x = _fire_cluster('fire3', x)
    x = _pool_layer('pool3_pool', x)

    # fire4 + fire5 clusters
    x = _fire_cluster('fire4', x)
    x = _fire_cluster('fire5', x)
    x = _pool_layer('pool5_pool', x)

    # remainder (no pooling)
    x = _fire_cluster('fire6', x)
    x = _fire_cluster('fire7', x)
    x = _fire_cluster('fire8', x)
    x = _fire_cluster('fire9', x)

    ### Classifier

    # Dropout [use value of 50% when training]
    x = tf.nn.dropout(x, 1)

    # Fixed global avg pool/softmax classifier:
    # [227, 227, 3] -> 1000 classes
    layer_name = 'conv10'
    weights, biases = get_weights_biases(layer_name)
    x = _conv_layer(layer_name + '_conv', x, weights, biases)
    x = _act_layer(layer_name + '_actv', x)

    # Global Average Pooling
    x = tf.nn.avg_pool(x, ksize=(1, 13, 13, 1), strides=(1, 1, 1, 1), padding='VALID')

    x = tf.nn.softmax(x, name='prob')
    return x

#-----------------------------------------------------------------------

def get_image_scores(batch_results, image_index):
    '''
    Returns scores for specific image in batch.
    batch_results - results of evaluation of node returned by the 'inference' method of this model.
    image_index - index of image inside of batch feeded into 'inference'.
    '''
    return batch_results[image_index][0][0]

#-----------------------------------------------------------------------

def _fire_cluster(cluster_name, x):
    # central - squeeze
    layer_name = cluster_name + '/squeeze1x1'
    weights, biases = get_weights_biases(layer_name)
    x = _conv_layer(layer_name + '_conv', x, weights, biases, padding='VALID')
    x = _act_layer(layer_name + '_actv', x)

    # left - expand 1x1
    layer_name = cluster_name + '/expand1x1'
    weights, biases = get_weights_biases(layer_name)
    x_l = _conv_layer(layer_name + '_conv', x, weights, biases, padding='VALID')
    x_l = _act_layer(layer_name + '_actv', x_l)

    # right - expand 3x3
    layer_name = cluster_name + '/expand3x3'
    weights, biases = get_weights_biases(layer_name)
    x_r = _conv_layer(layer_name + '_conv', x, weights, biases, padding='SAME')
    x_r = _act_layer(layer_name + '_actv', x_r)

    # concatenate expand 1x1 (left) and expand 3x3 (right)
    return tf.concat([x_l, x_r], 3)

#-----------------------------------------------------------------------

def _conv_layer(layer_name, input, weights, bias, padding='SAME', stride=(1, 1)):
    conv = tf.nn.conv2d(input,
                        tf.constant(weights),
                        strides=(1, stride[0], stride[1], 1),
                        padding=padding,
                        name=layer_name)
    return tf.nn.bias_add(conv, bias)

#-----------------------------------------------------------------------

def _act_layer(layer_name, input):
    return tf.nn.relu(input, name=layer_name)

#-----------------------------------------------------------------------

def _pool_layer(layer_name, input, size=(3, 3), stride=(2, 2), padding='VALID'):
    return tf.nn.max_pool(input,
                          ksize=(1, size[0], size[1], 1),
                          strides=(1, stride[0], stride[1], 1),
                          padding=padding,
                          name=layer_name)

#-----------------------------------------------------------------------

max_stars_repo_name: TITAN-PyCompat/ck-tensorflow

max_issues_repo_name: TITAN-PyCompat/ck-tensorflow

max_forks_repo_name: TITAN-PyCompat/ck-tensorflow