text:
# -*- coding: utf-8 -*-
"""basicpLinearRegressionUSINGneuralnetworks.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1rHJ9vLfYdAtzOuPBPrrmB24yxVH5mBbI
"""

"""
always follow the following steps in partcular order

1) design your model , the input size , output size
2) construct your loss and your optimizer
3) training loop :
    1) forward pass
    2) backward pass - calculate the gradients
    3) update the weights

import torch
import torch.nn
import numpy as np
from sklearn import datasets

# importing the data that we need

x_train,y_train=datasets.make_regression(n_samples=100,n_features=1,n_targets=1)

print(x_train.shape,y_train.shape)

#we know that torch needs to be worked on with float data type
# lets see the type of input data and change it

print(y_train.dtype)

#its already in float
# but we need TORCH TYpe and not just np array

x=torch.from_numpy(x_train.astype(np.float32))

y=torch.from_numpy(y_train.astype(np.float32))
#now change the shape of the vector y to give 1000,1

y=y.view(y.shape[0],1)

# define the model
inpshape=x[0].shape
outshape=y[0].shape

print(inpshape,y[0].shape)

inpsize,inpshape=x.shape
outputshape=y.shape[1]
print(outputshape,inpsize,inpshape)

#model definition
model=torch.nn.Linear(inpshape,outputshape)

#loss and optimizer

loss=nn.MSELoss()
optimizer=torch.optim.SGD(model.parameters(),lr=0.1)

# training loop

for epoch in range(10000):
  #forward pass
  ypred=model(x)

  #backward
  lossval=loss(ypred,y)
  lossval.backward()
  #updating the weights
  optimizer.step()

  optimizer.zero_grad()

  if epoch%100==0:
    print("epoch  {}   loss   {} %".format(epoch,lossval*100))

#j truying to plot
predictions=model(x).detach()
import matplotlib.pyplot as plt
plt.plot(x,predictions,'b')
plt.plot(x,y,'ro')

max_stars_repo_name: sumukhasrivatsa/PyTorch-Learnings

max_issues_repo_name: sumukhasrivatsa/PyTorch-Learnings

max_forks_repo_name: sumukhasrivatsa/PyTorch-Learnings


text:
from __future__ import print_function

import networkx as nx

from . import graph
from . import resolve

def inspect_graph(import_graph):
    keys = set(x[0] for x in import_graph.graph.edges)
    for key in sorted(keys):
        k = import_graph.format(key)
        for _, value in sorted(import_graph.graph.edges([key])):
            v = import_graph.format(value)
            print('  %s -> %s' % (k, v))
        for value in sorted(import_graph.broken_deps[key]):
            print('  %s -> <%s>' % (k, value))

def format_file_node(import_graph, node, indent):
    """Prettyprint nodes based on their provenance."""
    f = import_graph.provenance[node]
    if isinstance(f, resolve.Direct):
        out = '+ ' + f.short_path
    elif isinstance(f, resolve.Local):
        out = '  ' + f.short_path
    elif isinstance(f, resolve.System):
        out = ':: ' + f.short_path
    elif isinstance(f, resolve.Builtin):
        out = '(%s)' % f.module_name
    else:
        out = '%r' % node
    return '  '*indent + out

def format_node(import_graph, node, indent):
    """Helper function for print_tree"""
    if isinstance(node, graph.NodeSet):
        ind = '  ' * indent
        out = [ind + 'cycle {'] + [
                format_file_node(import_graph, n, indent + 1)
                for n in node.nodes
        ] + [ind + '}']
        return '\n'.join(out)
    else:
        return format_file_node(import_graph, node, indent)

def print_tree(import_graph):
    def _print_tree(root, indent=0):
        if root in seen:
            return
        seen.add(root)
        print(format_node(import_graph, root, indent))
        for _, v in import_graph.graph.out_edges([root]):
            _print_tree(v, indent=indent+2)

    seen = set()
    for root in nx.topological_sort(import_graph.graph):
        if not import_graph.graph.in_edges([root]):
            _print_tree(root)

def print_topological_sort(import_graph):
    for node in nx.topological_sort(import_graph.graph):
        print(import_graph.format(node))

def formatted_deps_list(import_graph):
    out = []
    for node, deps in import_graph.deps_list():
        out.append('source: ' + import_graph.format(node))
        if deps:
            out.append('deps:')
            for dep in deps:
                out.append('  ' + import_graph.format(dep))
    return '\n'.join(out)

def print_unresolved_dependencies(import_graph):
    for imp in sorted(import_graph.get_all_unresolved()):
        print(' ', imp.name)

def print_unreadable_files(import_graph):
    for f in sorted(import_graph.unreadable_files):
        print(' ', f)

def maybe_show_unreadable(import_graph):
    """Only print an unreadable files section if nonempty."""
    if import_graph.unreadable_files:
        print()
        print('Unreadable files:')
        print_unreadable_files(import_graph)

ext: py

max_stars_repo_name: KGerring/importlab

max_issues_repo_name: KGerring/importlab

max_forks_repo_name: KGerring/importlab

reason: import networkx


text:
import time
import numpy as np

from fmodeling.rock_physics.Mediums.DEM import DEM
from fmodeling.rock_physics.Mediums.DEMSlb import DEM as DEMSlb

def main():
    # Matrix properties
    Km = 77.0  # GPa
    Gm = 32.0  # GPa
    rhom = 2.71  # g/cm3

    # Fluid properties
    Kf = 3.0  # GPa
    rhof = 1.0  # g/cm3

    # Porosity
    phimax = 1

    phi = 0.1
    alpha = 0.1

    # Inclusion properties
    # In this example a mixture of three inclusion types are used:
    # - 30% of 0.02 aspect ratio
    # - 50% of 0.15 aspect ratio
    # - 20% of 0.80 aspect ratio
    alphas = np.array([0.01, 0.15, 0.8])
    volumes = np.array([0.3, 0.5, 0.2]) * phimax

    time_point_1 = time.time()
    Km_DEM, Gm_DEM, phi_DEM = DEM(Km, Gm, np.array([0]), np.array([0]), np.array([alpha]), np.array([phi*phimax]))
    time_point_2 = time.time()
    Km_DEMSlb, Gm_DEMSlb = DEMSlb(Km, Gm, 0.0, 0.0, phi, alpha, phimax)
    time_point_3 = time.time()

    print(Km_DEM, Gm_DEM, phi_DEM)
    print(Km_DEMSlb, Gm_DEMSlb)

    print(f"Old calculation time: {time_point_2 - time_point_1}")
    print(f"New calculation time: {time_point_3 - time_point_2}")

if __name__ == '__main__':
    main()

max_stars_repo_name: lomonosouv/rpsi

max_issues_repo_name: lomonosouv/rpsi

max_forks_repo_name: lomonosouv/rpsi

reason: import numpy


text:
import emcee as mc
from pearce.emulator import OriginalRecipe, ExtraCrispy
from pearce.mocks import cat_dict
import numpy as np
from os import path
training_file = '/home/users/swmclau2/scratch/PearceRedMagicXiCosmo.hdf5'
a = 1.0
z = 1./a-1.0

fixed_params = {'z':z, 'r':24.06822623}
n_leaves, n_overlap = 10, 2
em_method = 'gp'
emu = OriginalRecipe(training_file, method = em_method, fixed_params=fixed_params, downsample_factor = 0.1)

# TODO downsample sampling?
def nll(p):
    emu._emulator.set_parameter_vector(p)

    ll = emu._emulator.lnlikelihood(emu.downsample_y, quiet=True)

    return -ll if np.isfinite(ll) else 1e25

def lnprior(theta):
    return -np.inf if np.any(np.logical_or(theta < -15, theta > 15)) else 0

def lnprob(theta, *args):
    lp = lnprior(theta)
    if not np.isfinite(lp):
        return -np.inf

    return lp - nll(theta, *args)

#p0 = emu._emulator.get_parameter_vector()

#p0 = np.array([  0.       ,  10.6161248,   1.8339794,   7.342365 ,  10.6371797,
     #    2.2441632,  13.8155106,  11.3512804,   3.1795786,   4.6846614,
     #             1.       ,   5.0188608,   3.7658774,  -1.5383083])

p0 = np.array([-12.0550382,   0.1054246,   0.2661017,   5.6407612,   0.2408568,   1.1295944,
   0.3643993,  11.5649985,   4.9071932,   4.7031938,   1.,         11.7621938,
     10.6279446,   0.,         10.6161248,   1.8339794,   7.342365   10.6371797,
        2.2441632,  13.8155106,  11.3512804   3.1795786,   4.6846614   1.,
           5.0188608,   3.7658774,  -1.5383083])

nwalkers = 100
nsteps = 2000
nburn = 0
num_params = p0.shape[0]#len(emu.get_param_names())+1
pos0 = p0+np.random.randn(nwalkers, num_params)
ncores = 16

savedir = '/home/users/swmclau2/scratch/'
chain_fname = path.join(savedir, '%d_walkers_%d_steps_cosmo_hyperparams.npy'%(nwalkers, nsteps))

pnames = ['amp']
pnames.extend(emu.get_param_names())
with open(chain_fname, 'w') as f:
    f.write('#' + '\t'.join(pnames)+'\t'+ '\t'.join(pnames)+'\tamp'+'\n')

sampler = mc.EnsembleSampler(nwalkers, num_params, lnprob, threads=ncores)
for result in sampler.sample(pos0, iterations = nsteps, storechain=False):

    with open(chain_fname, 'a') as f:
        np.savetxt(f, result[0])


text:
import torch
torch.backends.cudnn.benchmark=True
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
import numpy as np
from PIL import Image
from tqdm import tqdm
import time
import copy

import torchvision.models as models
import torchvision.transforms as transforms

def MultiClassCrossEntropy(logits, labels, T):
	# Ld = -1/N * sum(N) sum(C) softmax(label) * log(softmax(logit))
	labels = Variable(labels.data, requires_grad=False).cuda()
	outputs = torch.log_softmax(logits/T, dim=1)   # compute the log of softmax values
	labels = torch.softmax(labels/T, dim=1)
	# print('outputs: ', outputs)
	# print('labels: ', labels.shape)
	outputs = torch.sum(outputs * labels, dim=1, keepdim=False)
	outputs = -torch.mean(outputs, dim=0, keepdim=False)
	# print('OUT: ', outputs)
	return Variable(outputs.data, requires_grad=True).cuda()

def kaiming_normal_init(m):
	if isinstance(m, nn.Conv2d):
		nn.init.kaiming_normal_(m.weight, nonlinearity='relu')
	elif isinstance(m, nn.Linear):
		nn.init.kaiming_normal_(m.weight, nonlinearity='sigmoid')

class Model(nn.Module):
	def __init__(self, classes, classes_map, args):
		# Hyper Parameters
		self.init_lr = args.init_lr
		self.num_epochs = args.num_epochs
		self.batch_size = args.batch_size
		self.lower_rate_epoch = [int(0.7 * self.num_epochs), int(0.9 * self.num_epochs)] #hardcoded decay schedule
		self.lr_dec_factor = 10

		self.pretrained = False
		self.momentum = 0.9
		self.weight_decay = 0.0001
		# Constant to provide numerical stability while normalizing
		self.epsilon = 1e-16

		# Network architecture
		super(Model, self).__init__()
		self.model = models.resnet34(pretrained=self.pretrained)
		self.model.apply(kaiming_normal_init)

		num_features = self.model.fc.in_features
		self.model.fc = nn.Linear(num_features, classes, bias=False)
		self.fc = self.model.fc
		self.feature_extractor = nn.Sequential(*list(self.model.children())[:-1])
		self.feature_extractor = nn.DataParallel(self.feature_extractor)

		# n_classes is incremented before processing new data in an iteration
		# n_known is set to n_classes after all data for an iteration has been processed
		self.n_classes = 0
		self.n_known = 0
		self.classes_map = classes_map

	def forward(self, x):
		x = self.feature_extractor(x)
		x = x.view(x.size(0), -1)
		x = self.fc(x)
		return x

	def increment_classes(self, new_classes):
		"""Add n classes in the final fc layer"""
		n = len(new_classes)
		print('new classes: ', n)
		in_features = self.fc.in_features
		out_features = self.fc.out_features
		weight = self.fc.weight.data

		if self.n_known == 0:
			new_out_features = n
		else:
			new_out_features = out_features + n
		print('new out features: ', new_out_features)
		self.model.fc = nn.Linear(in_features, new_out_features, bias=False)
		self.fc = self.model.fc

		kaiming_normal_init(self.fc.weight)
		self.fc.weight.data[:out_features] = weight
		self.n_classes += n

	def classify(self, images):
		"""Classify images by softmax

		Args:
			x: input image batch
		Returns:
			preds: Tensor of size (batch_size,)
		"""
		_, preds = torch.max(torch.softmax(self.forward(images), dim=1), dim=1, keepdim=False)

		return preds

	def update(self, dataset, class_map, args):

		self.compute_means = True

		# Save a copy to compute distillation outputs
		prev_model = copy.deepcopy(self)
		prev_model.cuda()

		classes = list(set(dataset.train_labels))
		#print("Classes: ", classes)
		print('Known: ', self.n_known)
		if self.n_classes == 1 and self.n_known == 0:
			new_classes = [classes[i] for i in range(1,len(classes))]
		else:
			new_classes = [cl for cl in classes if class_map[cl] >= self.n_known]

		if len(new_classes) > 0:
			self.increment_classes(new_classes)
			self.cuda()

		loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size,
											   shuffle=True, num_workers=12)

		print("Batch Size (for n_classes classes) : ", len(dataset))
		optimizer = optim.SGD(self.parameters(), lr=self.init_lr, momentum = self.momentum, weight_decay=self.weight_decay)

		with tqdm(total=self.num_epochs) as pbar:
			for epoch in range(self.num_epochs):

				# Modify learning rate
				# if (epoch+1) in lower_rate_epoch:
				# 	self.lr = self.lr * 1.0/lr_dec_factor
				# 	for param_group in optimizer.param_groups:
				# 		param_group['lr'] = self.lr

				for i, (indices, images, labels) in enumerate(loader):
					seen_labels = []
					images = Variable(torch.FloatTensor(images)).cuda()
					seen_labels = torch.LongTensor([class_map[label] for label in labels.numpy()])
					labels = Variable(seen_labels).cuda()
					# indices = indices.cuda()

					optimizer.zero_grad()
					logits = self.forward(images)
					cls_loss = nn.CrossEntropyLoss()(logits, labels)
					if self.n_classes//len(new_classes) > 1:
						dist_target = prev_model.forward(images)
						logits_dist = logits[:,:-(self.n_classes-self.n_known)]
						dist_loss = MultiClassCrossEntropy(logits_dist, dist_target, 2)
						loss = dist_loss+cls_loss
					else:
						loss = cls_loss

					loss.backward()
					optimizer.step()

					if (i+1) % 1 == 0:
						tqdm.write('Epoch [%d/%d], Iter [%d/%d] Loss: %.4f'
							   %(epoch+1, self.num_epochs, i+1, np.ceil(len(dataset)/self.batch_size), loss.data))

				pbar.update(1)

max_stars_repo_name: ngailapdi/LWF

max_issues_repo_name: ngailapdi/LWF

max_forks_repo_name: ngailapdi/LWF


text:
import ast
import sys
import numpy as np

from parameters import *
from util import *

def read_record(file):
    with open(file) as f:
        return [ast.literal_eval(line) for line in f]

def read_shape_file(file, unkwn):
    shps = {}
    with open(file) as f:
        for i, line in enumerate(f):
            shps[ast.literal_eval(line)] = i
    shps[unkwn] = len(shps)
    return shps

def read_feat_file(file, unkwn):
    feats = {}
    with open(file) as f:
        for i, item in enumerate(f):
            feats[item.strip()] = i
    feats[unkwn] = len(feats)
    return feats

def read_embeddings(file):
    if 'pkl' in file:
        return load_bigger(file)
    embs = {}
    with open(file, encoding="ISO-8859-1") as f:
        for line in f:
            split = line.strip().split()
            if len(split) != 1025 and len(split) != 301:
                continue
            else:
                word, vect = split[0], [float(val) for val in split[1:]]
                embs[word] = vect
    return embs

def read_clusters(file):
    clusts = {}
    with open(file) as f:
        for line in f:
            split = line.strip().split()
            if len(split) == 2:
                wrd, clust = split
            else:
                continue
            if 'marlin' in file:
                clusts[wrd] = int(clust)
            elif 'brown' in file:
                clusts[wrd] = int(clust, 2)  # converts binary to int
    if 'brown' in file:
        clusts['<RARE>'] = max(clusts.values()) + 1
    return clusts

def pad_middle(sent: list, max_len: int):
    """ add padding elements (i.e. dummy word tokens) to fill the sentence to max_len """
    num_pads = max_len-len(sent)
    padding = num_pads * [None]
    if not before_e2:
        return padding + sent
    elif before_e2 == 1:
        return [sent[0]] + padding + sent[1:]
    elif before_e2 == 2:
        return sent[:-1] + padding + [sent[-1]]
    else:
        return sent + padding

if __name__ == '__main__':
    which_set = sys.argv[1]  # round
    if which_set == '0':
        which_set = ''
    which_record_train = 'record_train' + which_set + '.txt'
    which_record_test = 'record_test' + which_set + '.txt'
    records_and_outs = [(features_dir + which_record_train, models_dir + 'libLinearInput_train.txt'),
                        (features_dir + which_record_test, models_dir + 'libLinearInput_test.txt')]
    record_file = features_dir + 'record.txt'
    vocab_file = features_dir + 'vocab.txt'
    shapes_file = features_dir + 'shapes.txt'
    e1_context_file = features_dir + 'e1_context.txt'
    e2_context_file = features_dir + 'e2_context.txt'
    abstracts_file = features_dir + 'abstracts.txt'
    # word_embds_file = features_dir + 'abstracts-dblp-semeval2018.wcs.txt'  # origin
    # word_embds_file = features_dir + 'word2vec_abstracts.wcs.txt'  # acm + bdlp v10
    # word_embds_file = features_dir + 'word2vec_abstracts1.wcs.txt'  # dblp v5
    # word_embds_file = features_dir + 'dblp_abstracts.wcs.txt'  # dblp v10
    # word_embds_file = features_dir + 'fasttext_v5.txt'  # fasttext dblp v5
    # word_embds_file = features_dir + 'fasttext_acm.txt'  # fasttext dblp acm
    word_embds_file = features_dir + 'fasttext_v10.txt'  # fasttext dblp v10
    # word_embds_file = features_dir + 'bert_acm.txt'  # bert acm test
    # word_embds_file = features_dir + 'bert_v5.txt'  # bert dblp v5

    # cluster_file = features_dir + 'dblp_marlin_clusters_1000'
    cluster_file = features_dir + 'acm_marlin_clusters_1000'
    # word_embds_file = features_dir + 'acm_abstracts.wcs.txt'

    unknown = 'UNK'
    num_words = 0
    num_shapes = 0
    num_embeddings = 0
    num_clusters = 0
    num_e1_contexts = 0
    num_e2_contexts = 0
    num_abstracts = 0

    if fire_words:            # one-hot word 2963 dim
        words = read_feat_file(vocab_file, unknown)
        num_words = len(words)
    if fire_clusters:         # clusters 1000 dim
        clusters = read_clusters(cluster_file)
        num_clusters = max(clusters.values()) + 1
    if fire_shapes:           # shape 2^7=128 dim
        shapes = read_shape_file(shapes_file, unknown)
        num_shapes = len(shapes)
    if fire_embeddings:       # embedding 300 dim
        embeddings = read_embeddings(word_embds_file)
        num_embeddings = len(list(embeddings.values())[0])
    if fire_e1_context:
        e1_contexts = read_feat_file(e1_context_file, unknown)
        num_e1_contexts = len(e1_contexts)
    if fire_e2_context:
        e2_contexts = read_feat_file(e2_context_file, unknown)
        num_e2_contexts = len(e2_contexts)

    # with open(label_file) as labs:
    #     labels = {lab.strip(): i for i, lab in enumerate(labs, start=0)}
    labels = rela2id

    len_token_vec = num_words + num_clusters + num_shapes + num_embeddings
    feat_val = ':1.0'

    for record_file, out_file in records_and_outs:
        records = read_record(record_file)
        if 'train' in record_file:
            # rec -> tuple(create_record)
            # the max length of all sentences
            max_rel_length = max([len(rec[1]) for rec in records])
            print('creating training file...')

            feature_dim = len_token_vec * max_rel_length
            row_num = len(records)

            if not which_set:  # for pickle format output
                if fire_e1_context:
                    feature_dim += num_e1_contexts
                if fire_e2_context:
                    feature_dim += num_e2_contexts

                pickle_matrix = np.zeros(
                    (row_num + 1, feature_dim + 1))
                pickle_label = np.zeros((row_num + 1, ))

        elif 'test' in record_file:
            print('creating test file...')

            feature_dim = len_token_vec * max_rel_length
            row_num = len(records)

            if not which_set:  # for pickle format output
                if fire_e1_context:
                    feature_dim += num_e1_contexts
                if fire_e2_context:
                    feature_dim += num_e2_contexts

                pickle_matrix = np.zeros(
                    (row_num + 1, feature_dim + 1))
                pickle_label = np.zeros((row_num + 1, ))
        # print(row_num, feature_dim, num_e1_contexts, num_e2_contexts)

        with open(out_file, 'w+') as lib_out:
            for idx, rec in enumerate(records):
                sentence_feats = []
                current_relation = rec[4]
                sentence = rec[1]
                # make the len of sentence to max sentences len
                norm_sentence = pad_middle(sentence, max_rel_length)
                for i, token in enumerate(norm_sentence):
                    offset = i * len_token_vec
                    token_feats = []
                    if token:
                        if fire_words:
                            if token in words:
                                feat_pos = offset + words[token] + 1
                            else:
                                feat_pos = offset + words[unknown] + 1
                            token_feat = str(feat_pos) + feat_val
                            token_feats.append(token_feat)
                        if fire_clusters:
                            if token in clusters:
                                feat_pos = offset + \
                                    clusters[token] + num_words + 1
                            else:
                                feat_pos = offset + \
                                    clusters['<RARE>'] + num_words + 1

                            token_feat = str(feat_pos) + feat_val
                            token_feats.append(token_feat)
                        if fire_shapes:    # combination feature
                            shape_vec = [0, 0, 0, 0, 0, 0, 0]
                            if any(char.isupper() for char in token):
                                shape_vec[0] = 1
                            if ',' in token:
                                shape_vec[1] = 1
                            if any(char.isdigit() for char in token):
                                shape_vec[2] = 1
                            if i == 0 and token[0].isupper():
                                shape_vec[3] = 1
                            if token[0].islower():
                                shape_vec[4] = 1
                            if '_' in token:
                                shape_vec[5] = 1
                            if "'s" in token:
                                shape_vec[6] = 1
                            tup_vec = tuple(shape_vec)
                            feat_pos = offset + \
                                shapes[tup_vec] + num_clusters + num_words + 1
                            token_feat = str(feat_pos) + feat_val
                            token_feats.append(token_feat)
                        if fire_embeddings:
                            lowered = token.lower()
                            if lowered in embeddings:  # entity not in embedding
                                vec = embeddings[lowered]
                                token_feats += [str(offset + n + num_shapes + num_clusters + num_words + 1) + ':' +
                                                str(vec[n]) for n in range(num_embeddings)]
                    else:
                        unknown_word = None
                        unknown_shape = None
                        unknown_cluster = None
                        # not handling unknown embeddings because they don't have indices
                        if fire_words:
                            feat_pos = offset + words[unknown] + 1
                            unknown_word = str(feat_pos) + feat_val
                        if fire_clusters:
                            feat_pos = offset + \
                                clusters['<RARE>'] + num_words + 1
                            unknown_cluster = str(feat_pos) + feat_val
                        if fire_shapes:
                            feat_pos = offset + \
                                shapes[unknown] + num_clusters + num_words + 1
                            unknown_shape = str(feat_pos) + feat_val
                        token_feats = [unknown_word,
                                       unknown_cluster, unknown_shape]

                    sentence_feats += token_feats
                sent_offset = len(norm_sentence) * len_token_vec
                # print(sent_offset)
                if len(sentence) > 2:  # norm sentence not used to avoid padding elements
                    e1_context = sentence[1]
                    e2_context = sentence[-2]

                    if fire_e1_context and fire_e2_context:
                        e1_context_idx = e1_contexts[e1_context] if e1_context in e1_contexts else e1_contexts[unknown]
                        e2_context_idx = e2_contexts[e2_context] if e2_context in e2_contexts else e2_contexts[unknown]
                        e1_pos = sent_offset + e1_context_idx + 1
                        e2_pos = sent_offset + num_e1_contexts + e2_context_idx + 1
                        e1_feat = str(e1_pos) + feat_val
                        e2_feat = str(e2_pos) + feat_val
                        sentence_feats.append(e1_feat)
                        sentence_feats.append(e2_feat)
                    elif fire_e1_context:
                        e1_context_idx = e1_contexts[e1_context] if e1_context in e1_contexts else e1_contexts[unknown]
                        e1_pos = sent_offset + e1_context_idx + 1
                        e1_feat = str(e1_pos) + feat_val
                        sentence_feats.append(e1_feat)
                    elif fire_e2_context:
                        e2_context_idx = e2_contexts[e2_context] if e2_context in e2_contexts else e2_contexts[unknown]
                        e2_pos = sent_offset + e2_context_idx + 1
                        e2_feat = str(e2_pos) + feat_val
                        sentence_feats.append(e2_feat)

                # if 'train' in out_file:
                try:
                    lib_out.write(str(labels[current_relation]) + ' ')
                    lib_out.write(
                        ' '.join(i for i in sentence_feats if i) + '\n')
                except:
                    pass
                # else:
                #    lib_out.write('0 ')

                # output data into pickle
                if not which_set:
                    try:
                        pickle_label[idx] = labels[current_relation]
                        for feat in sentence_feats:
                            if feat:
                                feat_num, val = feat.split(':')
                                pickle_matrix[idx, int(feat_num)] = val
                    except:
                        pass

        if not which_set:  # for pickle format output
            if 'train' in record_file:
                print('dumping training pickle file...')
                dump_bigger([pickle_matrix, pickle_label],
                            '%strain_data.pkl' % models_dir)

            elif 'test' in record_file:
                print('dumping test pickle file...')
                dump_bigger([pickle_matrix, pickle_label],
                            '%stest_data.pkl' % models_dir)

max_stars_repo_name: pku-nlp-forfun/LightRel

max_issues_repo_name: pku-nlp-forfun/LightRel

max_forks_repo_name: pku-nlp-forfun/LightRel


text:
#!/usr/bin/env python
# Helper functions for translating costmaps

from __future__ import print_function, division

import numpy as np

from geometry_msgs.msg import Point

# Constants defined in move_base
CONST_NO_INFORMATION = 255
CONST_LETHAL_OBSTACLE = 254
CONST_INSCRIBED_INFLATED_OBSTACLE = 253
CONST_FREE_SPACE = 0

# Functions. Also defined in move_base
def make_footprint(radius, num_points):
    """Given a radius param and the number of points in the footprint polygon,
    make a footprint polygon. The footprint is relative to the center"""
    footprint = []
    for i in xrange(num_points):
        angle = i * 2 * np.pi / num_points
        footprint.append(Point(
            x=(radius * np.cos(angle)),
            y=(radius * np.sin(angle))
        ))
    return footprint

def make_cost_translation_func():
    """Values published in the costmap don't correspond to the parameter
    settings for obstacles; we need a translation function. This creates that
    function."""
    internal_table = {
        0: CONST_FREE_SPACE,
        -1: CONST_NO_INFORMATION,
        99: CONST_INSCRIBED_INFLATED_OBSTACLE,
        100: CONST_LETHAL_OBSTACLE,
    }
    func = lambda x: internal_table.get(x, int(1 + (251 * (x - 1)) / 97))
    return func

def get_map_coords(wx, wy, costmap):
    """Given coordinates in the world, figure out the indices in the costmap
    that correspond to them. Make sure that the frame_id of the coordinates are
    the same as that of the costmap!"""
    ox, oy, res = (costmap.info.origin.position.x,
                   costmap.info.origin.position.y,
                   costmap.info.resolution)
    size_x, size_y = costmap.info.width, costmap.info.height

    # Sanity check the inputs
    assert not (wx < ox or wy < oy), ("World coordinates {} not in map with origin {}"
                                      .format((wx, wy,), (ox, oy,)))

    # Calculate the map coordinate
    mx = (wx - ox) / res
    my = (wy - oy) / res

    # Sanity check the outputs
    assert (mx < size_x and my < size_y), ("Calculated coords {} larger than map size {}"
                                           .format((mx, my,), (size_x, size_y,)))

    # Return
    return (int(mx), int(my),)

def check_collision(x, y, footprint, costmap, cost_translation_func):
    """
    Assuming the robot is at coordinates (x, y) check to see if it is in
    collision based on its footprint polygon, the costmap, and a cost
    translation function to interpret the costmap values.
    """
    in_collision = False
    for point in footprint:
        px, py = x + point.x, y + point.y
        mx, my = get_map_coords(px, py, costmap)
        cost = cost_translation_func(costmap.data[my * costmap.info.width + mx])
        in_collision = in_collision or (
            cost in [CONST_NO_INFORMATION,
                     CONST_LETHAL_OBSTACLE,
                     CONST_INSCRIBED_INFLATED_OBSTACLE]
        )

    return in_collision


text:
#-*- coding:utf-8 -*-
#!/usr/bin/python
''' TensorFlow pipeline for small dataset
This code is for example, has not been used in ASR project

author:
zzw922cn

date:2017-08-06
'''

import sys
sys.path.append('../')
sys.dont_write_bytecode = True

import tensorflow as tf
from tensorflow.python.training import queue_runner_impl
from tensorflow.python.platform import gfile
from tensorflow.core.protobuf import saver_pb2
from tensorflow.python.training import saver as saver_lib
from tensorflow.python.platform import app
from tensorflow.python.platform import flags

import os
import time
import math
import numpy as np

flags.DEFINE_string("scale", "small", "specify your dataset scale")
flags.DEFINE_string("logdir", "/home/pony/github/data/inputpipeline/small", "specify the location to store log or model")
flags.DEFINE_integer("samples_num", 80, "specify your total number of samples")
flags.DEFINE_integer("time_length", 2, "specify max time length of sample")
flags.DEFINE_integer("feature_size", 2, "specify feature size of sample")
flags.DEFINE_integer("num_epochs", 100, "specify number of training epochs")
flags.DEFINE_integer("batch_size", 2, "specify batch size when training")
flags.DEFINE_integer("num_classes", 10, "specify number of output classes")
FLAGS = flags.FLAGS

if __name__ == '__main__':

  scale = FLAGS.scale
  logdir = FLAGS.logdir
  sn = FLAGS.samples_num
  fs = FLAGS.feature_size
  num_epochs = FLAGS.num_epochs
  batch_size = FLAGS.batch_size
  num_classes = FLAGS.num_classes
  num_batches = int(math.ceil(1.0*sn/batch_size))

  with tf.variable_scope('train-samples'):
    x = tf.constant(np.random.rand(sn, fs).astype(np.float32))

  with tf.variable_scope('train-labels'):
    indices = np.random.randint(0, num_classes, size=sn).astype(np.int32)
    y = tf.one_hot(indices, depth=num_classes,
                   on_value=1.0,
                   off_value=0.0,
                   axis=-1,
                   dtype=tf.float32)

  # dequeue ops
  with tf.variable_scope('InputProducer'):
    slice_x, slice_y = tf.train.slice_input_producer([x, y],
        num_epochs = num_epochs, seed=22,
        capacity=36, shuffle=True)

    batched_x, batched_y = tf.train.batch([slice_x, slice_y],
        batch_size=batch_size, dynamic_pad=False,
        allow_smaller_final_batch=True)

    batched_x = tf.layers.dense(batched_x, 2*fs)
    batched_x = tf.layers.dense(batched_x, num_classes)

  with tf.variable_scope('Loss'):
    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
        labels=batched_y, logits=batched_x))
    optimizer = tf.train.AdamOptimizer(0.1)
    train_op = optimizer.minimize(loss)
    tf.summary.scalar('Loss', loss)

  merged = tf.summary.merge_all()

  t1 = time.time()
  sess = tf.Session()
  checkpoint_path = os.path.join(logdir, scale+'_model')
  writer = tf.summary.FileWriter(logdir, sess.graph)
  sess.run([tf.global_variables_initializer(),tf.local_variables_initializer()])
  coord = tf.train.Coordinator()
  threads = queue_runner_impl.start_queue_runners(sess=sess)
  saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
  saver.save(sess, checkpoint_path)
  for i in range(num_batches*num_epochs):
    l, _, summary = sess.run([loss, train_op, merged])
    writer.add_summary(summary, i)
    print 'batch '+str(i+1)+'/'+str(num_batches*num_epochs)+'\tLoss:'+str(l)
  writer.close()
  coord.request_stop()
  coord.join(threads)
  print 'program takes time:'+str(time.time()-t1)


text:
# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: base_type
   :platform: Unix
   :synopsis: A module to load MRC image data.

.. moduleauthor:: Nicola Wadeson <scientificsoftware@diamond.ac.uk>

"""

import numpy as np

import savu.plugins.loaders.utils.mrc_header as header_format
from savu.data.data_structures.data_types.base_type import BaseType

class MRC(BaseType):

    def __init__(self, Data, filename, stats=None):
        self._data_obj = Data
        self.filename = filename
        super(MRC, self).__init__()

        self.yz_swapped = False
        header_size = 1024
        self.file = open(filename, 'rb')
        # get the header information and file pointer position of first data
        # entry
        header, first = self.__get_header(self.file, header_size)
        self.header_dict = self.__get_header_dict(header)
        self.format = self.__set_data_format(header)
        self.shape = self.__set_shape(header)
        if self.format['mode'] == 16:
            self.shape = self.shape + (3,)

        self.data = np.memmap(filename, dtype=self.format['dtype'], order='F',
                              mode='r', offset=first, shape=self.shape)

    def clone_data_args(self, args, kwargs, extras):
        args = ['self', 'filename']
        return args, kwargs, extras

    def __getitem__(self, idx):
        return self.data[idx]

    def __get_header(self, fd, size):
        rec_header_dtype = np.dtype(header_format.rec_header_dtd)
        assert rec_header_dtype.itemsize == size
        header = np.fromfile(fd, dtype=header_format.rec_header_dtd, count=1)
        # Seek header
        if header['next'] > 0:
            fd.seek(header['next'])  # ignore extended header
        return header, fd.tell()

    def __get_header_dict(self, header):
        header_dict = {}
        for name in header.dtype.names:
            header_dict[name] = header[name][0] if len(header[name]) == 1 \
                                else header[name]
        return header_dict

    def __set_data_format(self, header):
        mode = header['mode']
        # BitOrder: little or big endian
        bo = "<" if header['stamp'][0, 0] == 68 and \
             header['stamp'][0, 1] == 65 else "<"
        sign = "i1" if header['imodFlags'] == 1 else "u1"  # signed or unsigned
        dtype = [sign, "i2", "f",  "c4", "c8", None, "u2", None, None, None,
                 None, None, None, None, None, None, "u1"][mode]
        dsize = [1, 2, 4, 4, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3][mode]
        return {'mode': mode, 'bo': bo, 'sign': sign, 'dtype': bo+dtype,
                'dsize': dsize}

    def __set_shape(self, header):
        nx, ny, nz = header['nx'], header['ny'], header['nz']
        if not isinstance(nx, int):
            nx = nx[0]
            ny = ny[0]
            nz = nz[0]
        return (nx, ny, nz)

    def get_shape(self):
        return self.shape

max_stars_repo_name: malte-storm/Savu

max_issues_repo_name: malte-storm/Savu

max_forks_repo_name: malte-storm/Savu


text:
import os
import time
import math
import torch
import torch.distributed as dist
from torch.nn import Module
from torch.utils.data.distributed import DistributedSampler
from torch.utils.data.sampler import Sampler
import multiprocessing as mp
import numpy as np
from torch.distributed import get_world_size, get_rank

# def init_dist(backend='nccl',
#               master_ip='127.0.0.2',
#               port=29500):
#     if mp.get_start_method(allow_none=True) is None:
#         mp.set_start_method('spawn')

# #    if '[' in node_list:
# #        beg = node_list.find('[')
# #        pos1 = node_list.find('-', beg)
# #        if pos1 < 0:
# #            pos1 = 1000
# #        pos2 = node_list.find(',', beg)
# #        if pos2 < 0:
# #            pos2 = 1000
# #        node_list = node_list[:min(pos1, pos2)].replace('[', '')
# #    addr = node_list[8:].replace('-', '.')

#     os.environ['MASTER_ADDR'] = str(master_ip)
#     os.environ['MASTER_PORT'] = str(port)
#     rank = int(os.environ['RANK'])
#     world_size = int(os.environ['WORLD_SIZE'])
#     num_gpus = torch.cuda.device_count()
#     torch.cuda.set_device(rank % num_gpus)
#     device = torch.device("cuda")
#     dist.init_process_group(backend=backend)
#     return rank, world_size, device

def init_dist(backend='nccl', master_ip='10.10.17.21', port=29500):

   if mp.get_start_method(allow_none=True) is None:
       mp.set_start_method('spawn')

   # node_list = os.environ['SLURM_NODELIST']

   # if '[' in node_list:
   #     beg = node_list.find('[')
   #     pos1 = node_list.find('-', beg)
   #     if pos1 < 0:
   #         pos1 = 1000
   #     pos2 = node_list.find(',', beg)
   #     if pos2 < 0:
   #         pos2 = 1000
   #     node_list = node_list[:min(pos1, pos2)].replace('[', '')
   # addr = node_list[8:].replace('-', '.')

   # os.environ['MASTER_ADDR'] = addr
   os.environ['MASTER_ADDR'] = master_ip
   os.environ['MASTER_PORT'] = str(port)

   rank = int(os.environ['RANK'])
   world_size = int(os.environ['WORLD_SIZE'])

   num_gpus = torch.cuda.device_count()
   torch.cuda.set_device(rank % num_gpus)
   device = torch.device('cuda')

   dist.init_process_group(backend=backend)

   return rank, world_size, device

def reduce_gradients(model, sync=False):
    """ average gradients """
    for name, param in model.named_parameters():
        if param.requires_grad:
            dist.all_reduce(param.grad.detach())

def reduce_tensorgradients(tensor_list, sync=False):
    """ average gradients """
    for param in tensor_list:
        if param.requires_grad and param.grad is not None:
            dist.all_reduce(param.grad.detach())

def part_reduce_gradients(tensor_list, param_count, sync=False):
    """ average gradients """
    dist.all_reduce(param_count.detach())
    id = 0
    for param in tensor_list:
        if param.requires_grad:
            if param_count[id]!= 0:
                dist.all_reduce(param.grad.data)
                param.grad.div_(param_count[id])
            id += 1

def broadcast_params(model):
    """ broadcast model parameters """
    for name,p in model.state_dict().items():
        dist.broadcast(p, 0)

max_stars_repo_name: cwlacewe/SNAS-Series

max_issues_repo_name: cwlacewe/SNAS-Series

max_forks_repo_name: cwlacewe/SNAS-Series


text:
# -*- coding: utf-8 -*-'
# Creaetor: Huije Lee (https://github.com/huijelee)
# WikiCorpus (2018-10-24 enwiki): 58,860,232 lxml etree elements, 5,739,304 articles

import glob
import codecs
import time
import re
import json
import numpy as np
import pandas as pd
from collections import defaultdict
from sklearn import metrics
from gensim.models import word2vec

# initialization
MODEL_DIR = 'model/'
SOURCE_DIR = 'source/'

VOCAB_LIMIT = 100000
ANNOTATED_VOCAB_LIMIT = 10000
MODEL_NAME = 'glove' #'wiki'  # 'reddit' 'redditsmall'
BASE_WORD_NUM = '20' # '30'
WORD_EMBEDDING_NAME = "/source/glove.6B.300d.txt"
#WORD_EMBEDDING_NAME = MODEL_DIR + 'w2v_wiki_sg_300_neg5_it2.model'
MITIGATED_EMBEDDING_NAME = MODEL_DIR + "mitigated{}_".format(BASE_WORD_NUM) + MODEL_NAME + ".300d"
#MITIGATED_EMBEDDING_NAME = MODEL_DIR + 'my_embedding_{}{}'.format(MODEL_NAME, BASE_WORD_NUM)
MITIGATED_EMBEDDING_INFO = MITIGATED_EMBEDDING_NAME.rsplit('.', 1)[0] + ".info"
UNBALANCED_BASE_WORDS = False
RANDOM_BASE_WORDS = False
SAVED_MODEL = False # for polarity_induction_methods (skip learning)

# Option for training new embedding (base_embedding.py)
CONSIDER_GENDER = True
WIKI_DIR = 'D:/dataset/wiki/text_en/'
REDDIT_DIR = 'D:/dataset/reddit/'
MINIMUM_WINDOW_SIZE = 11
start_time = time.time()

def whattime():
    return time.time() - start_time

def load_my_model(fname):
    try:
        print('Loading My Model... in {0:.2f} seconds'.format(whattime()))
        my_model = word2vec.Word2VecKeyedVectors.load_word2vec_format(fname, binary=False)
        print(my_model)

    except IOError:
        print('No existed model. Training My Model... in {0:.2f} seconds'.format(time.time() - start_time))
        my_model = ''

    print('Success to load My Model... in {0:.2f} seconds'.format(time.time() - start_time))
    return my_model

# means predicted:X, target:y
def find_optimal_cutoff(predicted, target):
    fpr, tpr, threshold = metrics.roc_curve(target, predicted)
    i = np.arange(len(tpr))
    roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold': pd.Series(threshold, index=i)})
    roc_t = roc.ix[(roc.tf-0).abs().argsort()[:1]]

    return list(roc_t['threshold']).pop()

def load_sent_lexicon():
    lexicon_dict = dict()
    with codecs.open(SOURCE_DIR + 'opinion_lexicon.txt', 'r', encoding='utf-8', errors='ignore') as f:
        for i, line in enumerate(re.split('[\r\n]+', f.read())):
            tokens = line.split('\t')
            lexicon_dict[tokens[0]] = int(tokens[1])

    return lexicon_dict

def load_entity_lexicon():
    lexicon_dict = dict()
    lexicon_vocab_dict = dict()
    with codecs.open(SOURCE_DIR + 'wiki_vocabs_annotated.txt', 'r', encoding='utf-8', errors='ignore') as f:
        for i, line in enumerate(re.split('[\r\n]+', f.read())):
            if i > VOCAB_LIMIT:
                break
            tokens = line.split('\t')
            if i >= ANNOTATED_VOCAB_LIMIT and len(tokens) > 1:
                lexicon_dict[tokens[0]] = 0  # 'evaulate' method doesn't evaluate words with label 0.
                lexicon_vocab_dict[tokens[0]] = int(tokens[1])
            elif len(tokens) == 2:
                lexicon_dict[tokens[0]] = -1
                lexicon_vocab_dict[tokens[0]] = int(tokens[1])
            elif len(tokens) == 3 and tokens[2] == '1':
                lexicon_dict[tokens[0]] = 1
                lexicon_vocab_dict[tokens[0]] = int(tokens[1])
            else:
                continue

    return lexicon_dict, lexicon_vocab_dict

def load_analogy_pair(fname):
    ap_dict = defaultdict(list)
    with codecs.open(fname, 'r', encoding='utf-8', errors='ignore') as f:
        for i, line in enumerate(re.split('[\r\n]+', f.read())):
            if len(line.strip()) > 0:
                tokens = re.split(r'\t', line.strip())
                for token in re.split(r' ', tokens[1]):
                    if tokens[0] == 'pairs':
                        ap_dict[tokens[0]].append(tuple(re.split(r'/', token)))
                    else:
                        ap_dict[tokens[0]].append(token)

    return ap_dict['pairs'], ap_dict['neutral_words']

class Vocab(object):
    """
    A single vocabulary item, used internally e.g. for constructing binary trees
    (incl. both word leaves and inner nodes).
    Possible Fields:
        - count: how often the word occurred in the training sentences
        - index: the word's index in the embedding
    """

    def __init__(self, **kwargs):
        self.count = 0
        self.__dict__.update(kwargs)

    def __lt__(self, other):  # used for sorting in a priority queue
        return self.count < other.count

    def __str__(self):
        vals = ['%s:%r' % (key, self.__dict__[key]) for key in sorted(self.__dict__) if not key.startswith('_')]
        return "<" + ', '.join(vals) + ">"

class TwitterCorpus(object):
    def __init__(self, fname):
        self.fname = fname

    def __iter__(self):
        sentence, rest, max_sentence_length = [], '', 1000
        with codecs.open(self.fname, "r", encoding="utf-8", errors='ignore') as fin:
            while True:
                text = rest + fin.read(8192)
                if text == rest:  # EOF
                    sentence.extend(rest.split())
                    if sentence:
                        yield sentence
                    break
                last_token = text.rfind(' ')
                words, rest = (text[:last_token].split(), text[last_token:].strip()) if last_token >= 0 else \
                    ([], text)
                sentence.extend(words)
                while len(sentence) >= max_sentence_length:
                    yield sentence[:max_sentence_length]
                    sentence = sentence[max_sentence_length:]

class WikiCorpus(object):
    def __init__(self, only_eng=True):
        self.fnames = glob.glob(WIKI_DIR + '*/wiki_*')
        # self.fnames = glob.glob(WIKI_DIR + 'AA/wiki_0*')
        self.doc_count = 0
        self.line_count = 0
        self.token_count = 0
        self.only_eng = only_eng

    def __iter__(self):
        for fname in self.fnames:
            with codecs.open(fname, "r", encoding="utf-8", errors='ignore') as fin:
                docs = re.split('<.+>', fin.read())
                docs = [doc.strip() for doc in docs if len(doc.strip()) > 1]
                # self.doc_count += len(docs)
                for doc in docs:
                    lines = re.split('[\r\n]+', doc)
                    # self.line_count += len(lines)
                    for line in lines:
                        result = [token for token in re.split('\W', line) if token]
                        if self.only_eng:
                            result = [token for token in result if re.search(r'^[a-zA-Z][a-zA-Z0-9]{0,}$', token)]
                        # self.token_count += len(result)
                        if len(result) >= MINIMUM_WINDOW_SIZE:
                            yield result

    def __str__(self):
        return "WikiCorpus(doc=%d, line=%d, token=%d)" % (self.doc_count, self.line_count, self.token_count)

class RedditCorpus(object):
    def __init__(self, only_eng=True):
        self.fnames = glob.glob(REDDIT_DIR + 'RC_2015-0*')
        #self.fnames = glob.glob(REDDIT_DIR + 'RC_2015-01')
        self.doc_count = 0
        self.line_count = 0
        self.token_count = 0
        self.only_eng = only_eng

    def __iter__(self):
        for fname in self.fnames:
            with codecs.open(fname, "r", encoding="utf-8", errors='ignore') as fin:
                while True:
                    read_line = fin.readline()
                    if read_line:
                        comment_json = json.loads(read_line)
                        self.line_count += 1
                        try:
                            line = comment_json['body']
                        except Exception as e:
                            continue
                        result = [token for token in re.split('\W', line) if token]
                        if self.only_eng:
                            result = [token for token in result if re.search(r'^[a-zA-Z][a-zA-Z0-9]{0,}$', token)]
                        # self.token_count += len(result)
                        if len(result) >= MINIMUM_WINDOW_SIZE:
                            yield result
                    else:
                        break

    def __str__(self):
        return "RedditCorpus(doc=%d, line=%d, token=%d)" % (self.doc_count, self.line_count, self.token_count)

if __name__ == "__main__":
    for datastore in RedditCorpus():
        print(datastore)


text:
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functional tests for XLA Gather Op."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.compiler.tests import xla_test
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import variables
from tensorflow.python.platform import flags
from tensorflow.python.platform import test

FLAGS = flags.FLAGS

class GatherTest(xla_test.XLATestCase):

  def _buildParams(self, data, dtype):
    data = data.astype(dtype.as_numpy_dtype)
    # For complex types, adds an index-dependent imaginary component so we can
    # tell we got the right value.
    if dtype.is_complex:
      return data + 10j * data
    return data

  def testScalar1D(self):
    with self.test_session() as session, self.test_scope():
      data = np.array([0, 1, 2, 3, 7, 5])
      for dtype in self.all_tf_types:
        for indices in 4, [4], [1, 2, 2, 4, 5]:
          params_np = self._buildParams(data, dtype)
          params = array_ops.placeholder(dtype=dtype)
          indices_tf = constant_op.constant(indices)
          gather_t = array_ops.gather(params, indices_tf)
          gather_val = session.run(gather_t, feed_dict={params: params_np})
          np_val = params_np[indices]
          self.assertAllEqual(np_val, gather_val)

  def testScalar2D(self):
    with self.test_session() as session, self.test_scope():
      data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
                       [12, 13, 14]])
      for dtype in self.all_tf_types:
        for axis in 0, 1, -1:
          params_np = self._buildParams(data, dtype)
          params = array_ops.placeholder(dtype=dtype)
          indices = constant_op.constant(2)
          gather_t = array_ops.gather(params, indices, axis=axis)
          gather_val = session.run(gather_t, feed_dict={params: params_np})
          expected = np.take(params_np, 2, axis=axis)
          self.assertAllEqual(expected, gather_val)

  def testSimpleTwoD32(self):
    with self.test_session() as session, self.test_scope():
      data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
                       [12, 13, 14]])
      for dtype in self.all_tf_types:
        for axis in 0, 1, -1:
          params_np = self._buildParams(data, dtype)
          params = array_ops.placeholder(dtype=dtype)
          # The indices must be in bounds for any axis.
          indices = constant_op.constant([0, 1, 0, 2])
          gather_t = array_ops.gather(params, indices, axis=axis)
          gather_val = session.run(gather_t, feed_dict={params: params_np})
          expected = np.take(params_np, [0, 1, 0, 2], axis=axis)
          self.assertAllEqual(expected, gather_val)

  def testSimpleTwoD32_Int64Indices(self):
    if np.int64 not in self.int_types:
      return

    with self.test_session() as session, self.test_scope():
      data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11],
                       [12, 13, 14]])
      # The indices must be in bounds for any axis.
      indices_np = np.array([0, 1, 0, 2])
      for dtype in self.all_tf_types:
        for axis in 0, 1, -1:
          params_np = self._buildParams(data, dtype)
          params = array_ops.placeholder(dtype=dtype)
          indices = array_ops.placeholder(dtype=dtypes.int64)
          gather_t = array_ops.gather(params, indices, axis=axis)
          gather_val = session.run(
              gather_t, feed_dict={
                  params: params_np,
                  indices: indices_np
              })
          expected = np.take(params_np, [0, 1, 0, 2], axis=axis)
          self.assertAllEqual(expected, gather_val)

  def testHigherRank(self):
    """Check that scalar and empty indices shapes work as well."""
    shape = (2, 1, 3, 2)
    for indices_shape in (), (0,), (2, 0), (2, 3):
      for dtype in self.all_tf_types:
        for axis in 0, 1, 2, 3, -1, -2:
          params = self._buildParams(np.random.randn(*shape), dtype)
          indices = np.random.randint(shape[axis], size=indices_shape)
          with self.test_session() as sess, self.test_scope():
            tf_params = array_ops.placeholder(dtype=dtype)
            tf_indices = constant_op.constant(indices, dtype=dtypes.int32)
            gather = array_ops.gather(tf_params, tf_indices, axis=axis)
            gather_value = sess.run(gather, feed_dict={tf_params: params})
            gather_np = np.take(params, indices, axis=axis)
            self.assertAllEqual(gather_np, gather_value)

class GatherBenchmark(test.Benchmark):
  """Microbenchmarks for the gather op."""

  def _benchmarkGather(self, name, axis, gather_indices, use_xla_jit):

    def BuilderFn():
      inputs = variables.Variable(
          array_ops.zeros([100, 100, 10, 100, 50], dtype=dtypes.float32),
          dtype=dtypes.float32,
          name='input')
      indices = variables.Variable(
          gather_indices, dtype=dtypes.int32, name='indices')
      gather_t = array_ops.gather(inputs, indices, axis=axis)
      return '%s.axis%d' % (name, axis), [gather_t]

    xla_test.Benchmark(self, BuilderFn, use_xla_jit=use_xla_jit, device='cpu')

  def _benchmarkSliceGather(self, axis, use_xla_jit):
    """Benchmarks a gather op that's really a dynamic slice."""
    self._benchmarkGather('slice_gather', axis, [1], use_xla_jit)

  def _benchmarkNontrivialGather(self, axis, use_xla_jit):
    self._benchmarkGather('nontrivial_gather', axis, [9, 1, 0, 2] * 4,
                          use_xla_jit)

  def benchmarkSliceGatherAxis0(self):
    self._benchmarkSliceGather(axis=0, use_xla_jit=False)

  def benchmarkSliceGatherAxis0XLA(self):
    self._benchmarkSliceGather(axis=0, use_xla_jit=True)

  def benchmarkSliceGatherAxis1(self):
    self._benchmarkSliceGather(axis=1, use_xla_jit=False)

  def benchmarkSliceGatherAxis1XLA(self):
    self._benchmarkSliceGather(axis=1, use_xla_jit=True)

  def benchmarkSliceGatherAxis4(self):
    self._benchmarkSliceGather(axis=4, use_xla_jit=False)

  def benchmarkSliceGatherAxis4XLA(self):
    self._benchmarkSliceGather(axis=4, use_xla_jit=True)

  def benchmarkNontrivialGatherAxis0(self):
    self._benchmarkNontrivialGather(axis=0, use_xla_jit=False)

  def benchmarkNontrivialGatherAxis0XLA(self):
    self._benchmarkNontrivialGather(axis=0, use_xla_jit=True)

  def benchmarkNontrivialGatherAxis1(self):
    self._benchmarkNontrivialGather(axis=1, use_xla_jit=False)

  def benchmarkNontrivialGatherAxis1XLA(self):
    self._benchmarkNontrivialGather(axis=1, use_xla_jit=True)

  def benchmarkNontrivialGatherAxis4(self):
    self._benchmarkNontrivialGather(axis=4, use_xla_jit=False)

  def benchmarkNontrivialGatherAxis4XLA(self):
    self._benchmarkNontrivialGather(axis=4, use_xla_jit=True)

if __name__ == '__main__':
  test.main()

max_stars_repo_name: ryorda/tensorflow-viennacl

max_issues_repo_name: ryorda/tensorflow-viennacl

max_forks_repo_name: ryorda/tensorflow-viennacl


text:
#!/usr/bin/env python3
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#           http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Script to preprocess the data for BERT."""

import os
import sys
sys.path.insert(0, os.getcwd())

import argparse
import json
import numpy as np

from code.common import logging
from code.bert.tensorrt.helpers.data_processing import read_squad_json, convert_example_to_features
from code.bert.tensorrt.helpers.tokenization import BertTokenizer

def preprocess_bert(data_dir, model_dir, preprocessed_data_dir):

    max_seq_length = 384
    max_query_length = 64
    doc_stride = 128
    output_dir = os.path.join(preprocessed_data_dir, "squad_tokenized")
    os.makedirs(output_dir, exist_ok=True)

    logging.info("Creating tokenizer...")
    tokenizer = BertTokenizer(os.path.join(model_dir, "bert", "vocab.txt"))
    logging.info("Done creating tokenizer.")

    logging.info("Reading SQuAD examples...")
    eval_examples = read_squad_json(os.path.join(data_dir, "squad", "dev-v1.1.json"))
    logging.info("Done reading SQuAD examples.")

    logging.info("Converting examples to features...")
    eval_features = []
    for example in eval_examples:
        feature = convert_example_to_features(example.doc_tokens, example.question_text,
                                              tokenizer, max_seq_length, doc_stride, max_query_length)
        eval_features.extend(feature)
    logging.info("Done converting examples to features.")

    logging.info("Saving features...")
    eval_features_num = len(eval_features)
    input_ids = np.zeros((eval_features_num, max_seq_length), dtype=np.int32)
    input_mask = np.zeros((eval_features_num, max_seq_length), dtype=np.int32)
    segment_ids = np.zeros((eval_features_num, max_seq_length), dtype=np.int32)
    for idx, feature in enumerate(eval_features):
        print(f"Processing {idx}/{eval_features_num}...")
        input_ids[idx, :] = np.array(feature.input_ids, dtype=np.int32)
        input_mask[idx, :] = np.array(feature.input_mask, dtype=np.int32)
        segment_ids[idx, :] = np.array(feature.segment_ids, dtype=np.int32)
    np.save(os.path.join(output_dir, "input_ids.npy"), input_ids)
    np.save(os.path.join(output_dir, "input_mask.npy"), input_mask)
    np.save(os.path.join(output_dir, "segment_ids.npy"), segment_ids)
    logging.info("Done saving features.")

def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--data_dir", "-d",
        help="Directory containing the input data.",
        default="build/data"
    )
    parser.add_argument(
        "--model_dir", "-m",
        help="Directory containing the models.",
        default="build/models"
    )
    parser.add_argument(
        "--preprocessed_data_dir", "-o",
        help="Output directory for the preprocessed data.",
        default="build/preprocessed_data"
    )
    args = parser.parse_args()
    data_dir = args.data_dir
    model_dir = args.model_dir
    preprocessed_data_dir = args.preprocessed_data_dir

    preprocess_bert(data_dir, model_dir, preprocessed_data_dir)

    print("Done!")

if __name__ == '__main__':
    main()


text:
#!/usr/bin/python3
# -*- coding: utf-8 -*-

# Standard imports
import logging
import os
import threading
import time
import asyncio
from datetime import datetime
from multiprocessing.pool import ThreadPool

# Third party imports
import sortedcontainers
import numpy as np
from dotenv import load_dotenv

# Local imports
import src.client
import src.sql_queries
import src.helpers
from src.base_with_database_logger import BaseWithDatabaseAndLogger
from quant_sdk import Client
from src.helpers import DBMode

timeout_tag = "timeout"
order_sdk_tag = "order_sdk"
additional_info_tag = "additional_info"

class OrderStatus(BaseWithDatabaseAndLogger):
    def __init__(self, client=None, mode=DBMode.DEV, test=False, logger_instance=None, update_order_sleep=10):
        self.test: bool = test
        super().__init__(mode=mode, logger_wrapper=logger_instance, open_db_connection=False)
        self.counter: int = 1
        self.update_order_sleep = update_order_sleep
        self.max_threads_to_process_orders: int = 20
        self.storage_async_mutex = asyncio.Lock()
        self.storage_mutex = threading.Lock()
        self.__order_storage = sortedcontainers.SortedDict()
        self.state_mapper = {
            Client.ORDER_STATUS_CODE_OPEN: Client.ORDER_STATUS_OPEN,
            Client.ORDER_STATUS_CODE_CLOSED: Client.ORDER_STATUS_CLOSED,
            Client.ORDER_STATUS_CODE_FAILED: Client.ORDER_STATUS_FAILED,
            Client.ORDER_STATUS_CODE_PARTIALLY_FILLED: Client.ORDER_STATUS_PARTIALLY_FILLED
        }

        self.active = True
        if client:
            self.client = client
        else:
            self.client = Client(os.getenv("API_KEY_BLOCKSIZE"))

    @property
    def order_storage(self):
        return self.__order_storage

    async def cancel_order(self, order_id):
        async with self.storage_async_mutex:
            key_to_remove = None
            # search for key
            for key, order in self.order_storage.items():
                if order['order_id'] == order_id:
                    key_to_remove = key

            # remove key associated to the id
            if key_to_remove:
                self.client.cancel_order(order_id)

    def cancel_order_thread(self, order_id):
        with self.storage_mutex:
            key_to_remove = None
            # search for key
            for key, order in self.order_storage.items():
                if order['order_id'] == order_id:
                    key_to_remove = key

            # remove key associated to the id
            if key_to_remove:
                self.client.cancel_order(order_id)

    async def enqueue_order(self, order_struct):
        timeout = order_struct[timeout_tag]
        async with self.storage_async_mutex:
            self.order_storage[timeout] = order_struct

    def enqueue_order_thread(self, order_struct):
        timeout = order_struct[timeout_tag]
        with self.storage_mutex:
            self.order_storage[timeout] = order_struct

    async def update_open_orders(self):
        remove_keys = []
        async with self.storage_async_mutex:
            for key, order in self.order_storage.items():
                now = datetime.now()
                order_id = order[order_sdk_tag]['order']['order_id']
                if key > now:
                    self.client.cancel_order(order_id)

                order_data = self.client.order_status(order_id)
                order[order_sdk_tag] = order_data
                params = self.extract_order_details(order_data)
                status = params["status"]
                if status in [Client.ORDER_STATUS_CLOSED, Client.ORDER_STATUS_FAILED]:
                    self.update_order_log_entry(**params)
                    remove_keys.append(key)
                    print(f"[{self.counter}] updated order {params['order_id']}")

            for item in remove_keys:
                del self.order_storage[item]

    def update_open_orders_thread(self):
        remove_keys = []
        with self.storage_mutex:
            for key, order in self.order_storage.items():
                now = datetime.now()
                order_id = order[order_sdk_tag]['order']['order_id']
                if key > now:
                    self.client.cancel_order(order_id)

                order_data = self.client.order_status(order_id)
                order[order_sdk_tag] = order_data

                status = self.convert_status_code(order_data['aggregated_status'])
                if status in [Client.ORDER_STATUS_CLOSED, Client.ORDER_STATUS_FAILED]:
                    self.update_order_log_entry(order_data, order[additional_info_tag])
                    remove_keys.append(key)
                    self.logger_wrapper.logger.debug(f"[{self.counter}] updated order {order_data['orderid']}")

            for item in remove_keys:
                del self.order_storage[item]

    def update_order(self, order_id: str):
        order_data = self.client.order_status(order_id)
        params = self.extract_order_details(order_data)
        status = params["status"]
        if status in [Client.ORDER_STATUS_CLOSED, Client.ORDER_STATUS_FAILED]:
            self.update_order_log_entry(**params)
            print(f"[{self.counter}] updated order {params['order_id']}")
            self.counter += 1

    def extract_order_details(self, order_status_dict, additional_info):
        order_id = order_status_dict['orderid']
        status = self.convert_status_code(order_status_dict['aggregated_status'])
        filled_quantity = order_status_dict['trade_status'][0]['trade']['trade_quantity']
        price_executed = order_status_dict['trade_status'][0]['status_report']['executed_price']
        fee = order_status_dict['trade_status'][0]['status_report']['fees']
        fee_currency = order_status_dict['trade_status'][0]['status_report']['fee_currency']

        return {
            "order_id": order_id,
            "status": status,
            "filled_quantity": filled_quantity,
            "price_executed": price_executed,
            "fee": fee,
            "fee_currency": fee_currency,
            "algo_id": additional_info["algo_id"],
            "exchange_id": additional_info["exchange_id"]
        }

    def convert_status_code(self, code):
        return self.state_mapper[code]

    def disable_activity(self):
        self.active = False

    async def resolve_orders(self):
        while self.active:
            try:
                await self.update_open_orders()
                await asyncio.sleep(self.update_order_sleep)
            except Exception as ex:
                self.logger_wrapper.logger.error(f"Error", exc_info=True)

    def resolve_orders_thread(self):
        while self.active:
            try:
                self.update_open_orders_thread()
                time.sleep(self.update_order_sleep)
            except Exception as ex:
                self.logger_wrapper.logger.error(f"Error", exc_info=True)

    def resolve_orders_threaded(self):
        return self.resolve_orders_thread()

    def update_order_log_entry(self, order_data, additional_info):
        query2 = src.sql_queries.insert_into_order_log(additional_info['algo_id'], additional_info['exchange_id'], order_data, self.logger_wrapper.logger)
        self.db_connector.execute_dml(query2)

if __name__ == '__main__':
    logger = logging.getLogger()
    orders = OrderStatus(client=src.client.test_sdk_client.TestClient(logger))
    asyncio.get_event_loop().run_until_complete(orders.resolve_orders())


text:
from pyunitree.base._handler import RobotHandler
from pyunitree.interfaces.gazeboInterface import GazeboInterface
from pyunitree.robots.a1.constants import STAND_ANGLES,INIT_ANGLES
import numpy as np
import time
import math

UPDATE_RATE = 1000

commsInterface = GazeboInterface(UPDATE_RATE)

robotHandler = RobotHandler(UPDATE_RATE)

def build_robot(handler, transmitter, receiver):
    handler.set_transmitter(transmitter)
    handler.set_receiver(receiver)
    robot = handler
    return robot

robot = build_robot(robotHandler,commsInterface.send, commsInterface.receive)

robot.start()

desired_angles = np.array(STAND_ANGLES)
robot.move_to(desired_angles)

initial_time = robot.state.time
currentTime = 0

while robot.state.time - initial_time < 5:

    desired_angles = np.array(STAND_ANGLES)*(1 + 0.25 * math.sin(3*currentTime))
    robot.set_angles(desired_angles)
    currentTime = time.time()

robot.move_to_init()

commsInterface.stop()
robot.stop()

max_stars_repo_name: RumblingTurtle/unitreepy

max_issues_repo_name: RumblingTurtle/unitreepy

max_forks_repo_name: RumblingTurtle/unitreepy


text:
# -*- coding: utf-8 -*-
import numpy as np
import loader
from sklearn.preprocessing import MinMaxScaler, StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.linear_model import LogisticRegression, SGDClassifier

train_inputs, train_output= loader.load_data_sklearn()
X_train, X_test, y_train, y_test = train_test_split(train_inputs, train_output, test_size=0.2, random_state=50000)

scaler = StandardScaler()
scaler.fit(train_inputs)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

# clf = SGDClassifier(shuffle=True)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

lr = LogisticRegression(multi_class="ovr")
lr.fit(X_train, y_train)
y_predict = lr.predict(X_test)
print("准确率：", lr.score(X_test, y_test))

# print("召回率：", classification_report(y_test, y_predict))

# bagging = BaggingClassifier(KNeighborsClassifier(),  max_samples=0.5, max_features=0.5)
# bagging.fit(X_train, y_train)
# y_predict = bagging.predict(X_test)
# print("准确率：", bagging.score(X_test, y_test))

# clf = RandomForestClassifier(n_estimators=10, max_depth=None,  min_samples_split=2, random_state=0)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

# clf = DecisionTreeClassifier(max_depth=None, min_samples_split=2, random_state=0)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

# clf = ExtraTreesClassifier(n_estimators=10, max_depth=None, min_samples_split=2, random_state=0)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

# clf = AdaBoostClassifier(n_estimators=100)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

# clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=5, random_state=0)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

# clf = HistGradientBoostingClassifier(max_iter=100)
# clf = clf.fit(X_train, y_train)
# y_predict = clf.predict(X_test)
# print("准确率：", clf.score(X_test, y_test))

# est = make_pipeline(StandardScaler(), SGDClassifier())
# est.fit(X_train, y_train)
# y_predict = est.predict(X_test)
# print("准确率：", est.score(X_test, y_test))

max_stars_repo_licenses: Unlicense MIT

max_issues_repo_licenses: Unlicense MIT

max_forks_repo_licenses: Unlicense MIT


text:
import numpy as np
from numpy.linalg import norm
import matplotlib.pyplot as plt
from importlib import reload
from sympy import *
from scipy.optimize import brentq, minimize_scalar
from scipy.integrate import solve_ivp, romberg, quad
import gravispy.geom as geom
from basic_units import radians
wrap = geom.wrap
unwrap = geom.unwrap
metric = geom.metric
lensing = geom.lensing
Ray = geom.Ray
Plane = geom.Plane
Sphere = geom.Sphere
t, r, th, ph, r0, l0, l_P, l, M = symbols('t r theta phi r0 l0 l_P l M', positive=True)
s = np.linspace(0,2*np.pi,1000)
args = (30, 3*np.pi/4)
rO, theta = args
l_R = 1/2/sqrt(2)

S = metric.Schwarzschild(1, [t, r, th, ph], timelike=False, lambdify_modules='numpy')
SM = metric.Schwarzschild(M, [t, r, th, ph], timelike=False, lambdify_modules='numpy')
B = metric.BarriolaVilenkin(1/3.7, [t, r, th, ph], timelike=False, lambdify_modules='numpy')
E = metric.EllisWormhole(1, [t, r, th, ph], timelike=False, lambdify_modules='numpy')

Sf2 = S.radial_factor(generator=True)
Rf2 = S.angular_factor(generator=True)
S2 = SM.radial_factor()
R2 = SM.angular_factor()
P = sqrt(R2.subs({r:r0})*S2 / (R2*(R2-R2.subs({r:r0})*sin(th)**2))) * sin(th)
Pf = lambdify(r, P.subs({r0:30, th:theta, M:1}))

def T_impact_func(r, rO, theta):
    return (Rf2(r)-Rf2(rO)*np.sin(theta)**2) / (Sf2(r)*Rf2(r))

r_P = brentq(T_impact_func, 3, rO, args=args)
l_P = 1/np.sqrt(2)/r_P
q1 = 2*l_P*(1-2*np.sqrt(2)*l_P)
q2 = 1-6*np.sqrt(2)*l_P
q3 = 2*np.sqrt(2)
T_P = lambda q: np.sqrt(2)*(l_P-q)**2/np.sqrt(q1*q-q2*q**2-q3*q**3)

def T_Pf(r):
    l = 1/np.sqrt(2)/r
    return T_P(l_P-l)

def T_phi_func(rval, rO, theta):
    num = Rf2(rO)*Sf2(rval)
    den = Rf2(rval) * (Rf2(rval)-Rf2(rO)*np.sin(theta)**2)
    return np.sin(theta) * np.sqrt(num/den)

def T_lens(ths, rO=30, rS=None):
    if not rS:
        rS = 1e+4*rO
    return np.fromiter(lensing.static_spherical_lens(ths, rO, rS, S), np.float32)

def T_sc_lens(ths, rO=30, rS=None):
    if not rS:
        rS = 1e+4*rO
    return np.fromiter(lensing.schwarzschild_lens(ths, rO, rS, S), np.float32)

def T_bv_lens(ths, rO=30, rS=None, general=False):
    if not rS:
        rS = rO/.77
    if not general:
        return np.fromiter(lensing.barriola_vilenkin_lens(ths,rO,rS,B), np.float32)
    else:
        return np.fromiter(lensing.static_spherical_lens(ths,rO,rS,B), np.float32)

def T_ew_lens(ths, rO=30, rS=None):
    if not rS:
        rS = 1e+4*rO
    return np.fromiter(lensing.ellis_wormhole_lens(ths,rO,rS,E), np.float32)

def T_thin_lens(ths, rO=30, rS=None):
    if not rS:
        rS = 1e+4*rO
    return np.fromiter(lensing.schwarzschild_thin_lens(ths,rO,rS,S), np.float32)

def T_trivial_lens(ths, rO=30, rS=None):
    if not rS:
        rS = 1e+4*rO
    return np.fromiter(lensing.trivial_lens(ths, rO, rS), np.float32)

def delta1(rO):
    inf = minimize_scalar(lambda r: np.sqrt(Rf2(r)/Rf2(rO)), method='bounded', bounds=(rO, 1e+4*rO))
    if inf['fun'] <= 1:
        return np.arcsin(inf['fun'])
    else:
        return np.pi/2
def delta2(rO):
    inf = minimize_scalar(lambda r: np.sqrt(Rf2(r)/Rf2(rO)), method='bounded', bounds=(0, rO))
    if inf['fun'] <= 1:
        return np.arcsin(inf['fun'])
    else:
        return np.pi/2

init_printing(num_columns=150)
testS1 = T_sc_lens(unwrap(s),10)
testS2 = T_lens(unwrap(s),10)
testS3 = T_thin_lens(unwrap(s),10)

dd = np.abs(np.diff(unwrap(testS2+np.pi)))
arg = (np.argwhere(dd>5).flatten() + 1)[:2]
testS2[slice(*arg)] = np.nan

plt.xlabel(r'Observation Angle, $\Theta$')
plt.ylabel(r'Source Angular Position, $\Phi$')
plt.plot(s*radians,unwrap(testS1+np.pi),label='Explicit Lens')
plt.plot(s*radians,unwrap(testS2+np.pi),alpha=.7,label='General Lens')
plt.plot(s*radians,unwrap(testS3+np.pi),'--k',alpha=.7,label='Thin Lens')
plt.xlim(0*radians,2*(np.pi*radians))
plt.ylim(-np.pi*radians,np.pi*radians)
plt.title(r'Schwarzschild Lensing Methods ($R_O = 10M$)')
plt.legend()
plt.savefig('sc_lensing.png')

max_stars_repo_name: mvtea/sgl

max_issues_repo_name: mvtea/sgl

max_forks_repo_name: mvtea/sgl

reason: import numpy,from numpy,from scipy,from sympy


text:
import os

import hydra
import torch
import clip
import numpy as np
from PIL import Image
from tqdm import tqdm

def run_eval(
    clip_model: str,
    generated_img_folder: str,
    generated_img_file_template: str,
    generated_img_start: int,
    generated_img_end: int,
    ground_truth_folder: str,
    ground_truth_file_template: str,
    ground_truth_start: int,
    ground_truth_end: int,
    text_prompt: str,
):
    # metric
    cos_sim = torch.nn.CosineSimilarity()

    generated_img_files = [
        generated_img_file_template.format(i)
        for i in range(generated_img_start, generated_img_end + 1)
    ]
    for file in generated_img_files:
        try:
            assert os.path.exists(os.path.join(generated_img_folder, file))
        except:
            print(os.path.join(generated_img_folder, file))
            exit()
    ground_truth_files = [
        ground_truth_file_template.format(i)
        for i in range(ground_truth_start, ground_truth_end + 1)
    ]
    for file in ground_truth_files:
        assert os.path.exists(os.path.join(ground_truth_folder, file))

    device = "cuda" if torch.cuda.is_available() else "cpu"
    model, preprocess = clip.load(clip_model, device=device)

    # ground truth embeddings
    ground_truth_embds = []
    for file in tqdm(ground_truth_files):
        img = (
            preprocess(Image.open(os.path.join(ground_truth_folder, file)))
            .unsqueeze(0)
            .to(device)
        )
        with torch.no_grad():
            ground_truth_embds.append(model.encode_image(img))
    ground_truth_embds = torch.concat(ground_truth_embds, dim=0)

    # generated imgs embeddings
    generated_embds = []
    for file in tqdm(generated_img_files):
        img = (
            preprocess(Image.open(os.path.join(generated_img_folder, file)))
            .unsqueeze(0)
            .to(device)
        )
        with torch.no_grad():
            generated_embds.append(model.encode_image(img))
    generated_embds = torch.concat(generated_embds, dim=0)

    assert ground_truth_embds.shape[1] == generated_embds.shape[1]

    # text embedding
    text = clip.tokenize([text_prompt]).to(device)
    with torch.no_grad():
        text_embd = model.encode_text(text)

    # score ground truth
    with torch.no_grad():
        gt_score = cos_sim(ground_truth_embds, text_embd)
    gt_score = gt_score.cpu().numpy()

    # score generated
    with torch.no_grad():
        generated_score = cos_sim(generated_embds, text_embd)
    generated_score = generated_score.cpu().numpy()

    print(f"\n Ground-truth score mean: {np.mean(gt_score)}, std: {np.std(gt_score)}\n Generated score mean: {np.mean(generated_score)}, std: {np.std(generated_score)}")

@hydra.main(config_path="./eval_cfg", config_name="conf")
def main(cfg):
    run_eval(**cfg.eval)

if __name__ == "__main__":
    main()

max_stars_repo_name: yunfanjiang/clip-nerf

max_issues_repo_name: yunfanjiang/clip-nerf

max_forks_repo_name: yunfanjiang/clip-nerf


text:
import os
import h5py
import re
import random
import numpy as np
from argparse import ArgumentParser
from collections import Counter
from typing import Iterable, Dict, Any

TRAIN_FRAC = 0.85
VALID_FRAC = 0.15
CHUNK_SIZE = 1000
SEQ_LENGTH = 23

LINE_REGEX = re.compile(r"'(.*)',([0-9]+)")
SPLIT_REGEX = re.compile(r'\\n')

def iterate_dataset(path: str) -> Iterable[Any]:
    with open(path, 'r') as fin:
        is_header = True

        for line in fin:
            line = line.strip()

            if is_header:
                is_header = (line != '@data')
            elif len(line) > 0:
                match = LINE_REGEX.match(line)
                features = match.group(1)
                label = int(match.group(2)) - 1

                feature_vec_strings = list(filter(lambda t: len(t) > 0, SPLIT_REGEX.split(features)))
                feature_vectors = np.array([list(map(int, vec.split(','))) for vec in feature_vec_strings])  # [10, 23]

                feature_vectors = np.ascontiguousarray(np.transpose(feature_vectors))  # [23, 10]

                yield feature_vectors, label

def write_dataset(path: str, output_folder: str, series: str):
    # Create the data writers
    if series == 'train':
        inputs = dict(train=[], validation=[])
        output = dict(train=[], validation=[])

        label_counters = {
            'train': Counter(),
            'validation': Counter()
        }
    else:
        inputs = dict(test=[])
        output = dict(test=[])

        label_counters = {
            'test': Counter()
        }

    # Iterate over the dataset
    for index, sample in enumerate(iterate_dataset(path=path)):

        # Select the partition
        if series == 'train':
            if random.random() < TRAIN_FRAC:
                partition = 'train'
            else:
                partition = 'validation'
        else:
            partition = 'test'

        sample_inputs, sample_output = sample

        inputs[partition].append(np.expand_dims(sample_inputs, axis=0))
        output[partition].append(sample_output)

        label_counters[partition][sample_output] += 1

        if (index + 1) % CHUNK_SIZE == 0:
            print('Completed {0} samples.'.format(index + 1), end='\r')
    print()

    print(label_counters)

    for fold in inputs.keys():

        if len(inputs[fold]) == 0:
            print('WARNING: Empty fold {0}'.format(fold))
            continue

        partition_inputs = np.vstack(inputs[fold])  # [N, T, D]
        partition_output = np.vstack(output[fold]).reshape(-1)  # [N]

        partition_folder = os.path.join(output_folder, fold)

        if not os.path.exists(partition_folder):
            os.mkdir(partition_folder)

        print(partition_inputs.shape)

        with h5py.File(os.path.join(partition_folder, 'data.h5'), 'w') as fout:
            input_dataset = fout.create_dataset('inputs', partition_inputs.shape, dtype='i')
            input_dataset.write_direct(partition_inputs)

            output_dataset = fout.create_dataset('output', partition_output.shape, dtype='i')
            output_dataset.write_direct(partition_output)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--input-folder', type=str, required=True)
    parser.add_argument('--output-folder', type=str, required=True)
    parser.add_argument('--dataset-name', type=str, required=True)
    args = parser.parse_args()

    # Set the random seed for reproducible results
    random.seed(42)

    # Create the output folder
    if not os.path.exists(args.output_folder):
        os.mkdir(args.output_folder)

    train_path = os.path.join(args.input_folder, '{0}_TRAIN.arff'.format(args.dataset_name))
    write_dataset(train_path, args.output_folder, series='train')

    test_path = os.path.join(args.input_folder, '{0}_TEST.arff'.format(args.dataset_name))
    write_dataset(test_path, args.output_folder, series='test')

max_stars_repo_name: tejaskannan/adaptive-sensor-security

max_issues_repo_name: tejaskannan/adaptive-sensor-security

max_forks_repo_name: tejaskannan/adaptive-sensor-security


text:
import sys, os, time, datetime
import json
import itertools, functools
from math import floor
import concurrent.futures
from generate_simulation_cases import generate_file_name, calc_parameter_product_length
from e2ep import run_with_setting_file, run
from importlib import import_module
import numpy as np
import matplotlib.pyplot as plt

def main():
    """
    parameter_file_name = sys.argv[1]
    parameter_module_name = parameter_file_name.split("/")[1].split(".")[0]
    parameter_module = import_module(parameter_module_name)
    """

    directory_root = '../data_generalizd_fidelity1/'
    fidelity_raw_bellpair_ = 0.8
    layer2_target_fidelity_ = 0.99
    layer3_target_fidelity_ = 0.99
    layer4_target_fidelity_ = 0.99
    p_op_int_node_  = 0.0001
    p_mem_int_node_ = 0.0001
    p_op_end_node_  = 0
    p_mem_end_node_ = 0

    # [i for i in range(100, 2001, 100)]

    purification_at_int_nodes_ = False

    num_nodes = [i for i in range(2, 257)]

    false_e2e_final_int_node, false_e2e_final_end_node, false_e2e_final_total_node = collect_num_node(directory_root, fidelity_raw_bellpair_, layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_, num_nodes, purification_at_int_nodes_)

    purification_at_int_nodes_ = True

    true_e2e_final_int_node, true_e2e_final_end_node, true_e2e_final_total_node = collect_num_node(directory_root, fidelity_raw_bellpair_, layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_, num_nodes, purification_at_int_nodes_)

    print(len(num_nodes))
    print(len(false_e2e_final_int_node))
    print(false_e2e_final_total_node)

    fig, ax = plt.subplots(figsize=(10,10),dpi=200)
    ax.set_yscale('log')
    ax.plot(num_nodes, false_e2e_final_int_node, label='Block. Time of Int. Node with only E2E Pur.', ls='solid', marker='o', color='blue')
    ax.plot(num_nodes, false_e2e_final_end_node, label='Block. Time of End Node with only E2E Pur.', ls='solid', marker='v', color='blue')
    ax.plot(num_nodes, false_e2e_final_total_node, label='Block. Time of Both Nodes with only E2E Pur.', ls='solid', marker='s', color='blue')
    ax.plot(num_nodes, true_e2e_final_int_node, label='Block. Time of Int. Node with non-E2E Pur.', ls='dashed', marker='o', color='red')
    ax.plot(num_nodes, true_e2e_final_end_node, label='Block. Time of End Node with non-E2E Pur.', ls='dashed', marker='v', color='red')
    ax.plot(num_nodes, true_e2e_final_total_node, label='Block. Time of Both Nodes with non-E2E Pur.', ls='dashed', marker='s', color='red')
    ax.set_xlabel('Number of Nodes')
    ax.set_ylabel('Consuming Blocking Time')
    title = "l2:%f, l3:%f, l4:%f, \np_op_int_node:%.1e, p_mem_int_node:%.1e, \np_op_end_node:%.1e, p_mem_end_node:%.1e"%(layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_)
    ax.set_title(title)
    ax.legend()
    plt.show()
    fig.savefig(directory_root + title + ".pdf")

def collect_num_node(directory_root, fidelity_raw_bellpair_, layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_, num_nodes, purification_at_int_nodes_):
    e2e_final_int_node = []
    e2e_final_end_node = []
    e2e_final_total_node = []
    for num_node_ in num_nodes:
        directory, file_output, file_input = generate_file_name(directory_root, fidelity_raw_bellpair_, layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_, num_node_, purification_at_int_nodes_)
        with open(file_output, 'r') as fd:
            json_dict = json.load(fd)
            if json_dict["success"]:
                e2e_final_int_node.append(json_dict["e2e_final"]["bt_int_node"])
                e2e_final_end_node.append(json_dict["e2e_final"]["bt_end_node"])
                e2e_final_total_node.append(json_dict["e2e_final"]["bt_int_node"] + json_dict["e2e_final"]["bt_end_node"])
            else:
                e2e_final_int_node.append(None)
                e2e_final_end_node.append(None)
                e2e_final_total_node.append(None)
    return e2e_final_int_node, e2e_final_end_node, e2e_final_total_node

def collect_results(cases, data):
    # For purification_at_int_nodes,
    # 0 for False
    # 1 for True
    for case in cases:
        (length, directory_root), fidelity_raw_bellpair_, layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_, num_node_, purification_at_int_nodes_ = case

        directory, file_output, file_input = generate_file_name(directory_root, fidelity_raw_bellpair_, layer2_target_fidelity_, layer3_target_fidelity_, layer4_target_fidelity_, p_op_int_node_, p_mem_int_node_, p_op_end_node_, p_mem_end_node_, num_node_, purification_at_int_nodes_)

        if purification_at_int_nodes_:
            purification_at_int_nodes__ = 1
        else:
            purification_at_int_nodes__ = 0

        data[fidelity_raw_bellpair_][layer2_target_fidelity_][layer3_target_fidelity_][layer4_target_fidelity_][p_op_int_node_][p_mem_int_node_][p_op_end_node_][p_mem_end_node_][num_node_][purification_at_int_nodes__] = file_output

if __name__ == "__main__":
    main()


text:
"""
Set partition and set covering in cpmpy.

Example from the Swedish book
Lundgren, Roennqvist, Vaebrand
'Optimeringslaera' (translation: 'Optimization theory'),
page 408.

* Set partition:
  We want to minimize the cost of the alternatives which covers all the
  objects, i.e. all objects must be choosen. The requirement is than an
  object may be selected _exactly_ once.

  Note: This is 1-based representation

  Alternative        Cost        Object
  1                  19           1,6
  2                  16           2,6,8
  3                  18           1,4,7
  4                  13           2,3,5
  5                  15           2,5
  6                  19           2,3
  7                  15           2,3,4
  8                  17           4,5,8
  9                  16           3,6,8
  10                 15           1,6,7

  The problem has a unique solution of z = 49 where alternatives
    3, 5, and 9
  is selected.

* Set covering:
  If we, however, allow that an object is selected _more than one time_,
  then the solution is z = 45 (i.e. less cost than the first problem),
  and the alternatives
    4, 8, and 10
  is selected, where object 5 is selected twice (alt. 4 and 8).
  It's an unique solution as well.

This cpmpy model was written by Hakan Kjellerstrand (hakank@gmail.com)
See also my cpmpy page: http://hakank.org/cpmpy/

"""
from cpmpy import *
import cpmpy.solvers
import numpy as np
from cpmpy_hakank import *

def set_covering4(set_partition=1):

  #
  # data
  #
  num_alternatives = 10
  num_objects = 8

  # costs for the alternatives
  costs = [19, 16, 18, 13, 15, 19, 15, 17, 16, 15]

  # the alternatives, and their objects
  a = [
      # 1 2 3 4 5 6 7 8    the objects
      [1, 0, 0, 0, 0, 1, 0, 0],  # alternative 1
      [0, 1, 0, 0, 0, 1, 0, 1],  # alternative 2
      [1, 0, 0, 1, 0, 0, 1, 0],  # alternative 3
      [0, 1, 1, 0, 1, 0, 0, 0],  # alternative 4
      [0, 1, 0, 0, 1, 0, 0, 0],  # alternative 5
      [0, 1, 1, 0, 0, 0, 0, 0],  # alternative 6
      [0, 1, 1, 1, 0, 0, 0, 0],  # alternative 7
      [0, 0, 0, 1, 1, 0, 0, 1],  # alternative 8
      [0, 0, 1, 0, 0, 1, 0, 1],  # alternative 9
      [1, 0, 0, 0, 0, 1, 1, 0]  # alternative 10
  ]

  # For slighly neater encodings using numpy below
  at = np.array(a).transpose()

  #
  # declare variables
  #
  x = boolvar(shape=num_alternatives,name="x")
  z = intvar(0,100,name="z")

  model = Model(minimize=z)

  #
  # constraints
  #

  # sum the cost of the choosen alternative,
  # to be minimized
  model += [z == sum(x*costs)]

  #
  for j in range(num_objects):
    if set_partition == 1:
      # model += [ sum([x[i] * a[i][j] for i in range(num_alternatives)]) == 1]
      model += [ sum(x * at[j]) == 1]
    else:
      # model += [sum([x[i] * a[i][j] for i in range(num_alternatives)]) >= 1]
      model += [ sum(x * at[j]) >= 1]

  ss = CPM_ortools(model)

  alternatives = np.array(range(num_alternatives))+1
  if ss.solve():
    print("z:", z.value())
    # print("selected alternatives:", [i + 1 for i in range(num_alternatives) if x[i].value() == 1])
    print("selected alternatives:", alternatives[x.value() == 1])

print("Set partition:")
set_covering4(1)

print("\nSet covering:")
set_covering4(0)

max_stars_repo_name: tias/hakank

max_issues_repo_name: tias/hakank

max_forks_repo_name: tias/hakank