text:
import numpy as np
import pandas as pd
import scipy.stats as ss

from .data_processing import get_filter_expr, data_filter_keys, get_type_to_measures
from .diff_exp import fdrcorrection
from .envir import CIRRO_SERVE, CIRRO_MAX_WORKERS

executor = None

def submit_job(database_api, dataset_api, email, dataset, job_name, job_type, params):
    global executor
    if executor is None:
        from concurrent.futures.thread import ThreadPoolExecutor
        import os

        if os.environ.get(CIRRO_SERVE) != 'true':
            max_workers = 1
        else:
            max_workers = int(os.environ.get(CIRRO_MAX_WORKERS, '2'))
        executor = ThreadPoolExecutor(max_workers=max_workers)
    job_id = database_api.create_job(email=email, dataset_id=dataset['id'], job_name=job_name, job_type=job_type,
                                     params=params)
    # run_job(database_api, dataset_api, email, job_id, job_type, dataset, params)
    executor.submit(run_job, database_api, dataset_api, email, job_id, job_type, dataset, params)
    return job_id

def get_mask(dataset_api, dataset, data_filter):
    measures, dimensions, basis_list = data_filter_keys(data_filter)
    keys = get_type_to_measures(measures)
    keys['obs'] = list(dimensions)
    keys['basis'] = basis_list
    df = dataset_api.read_dataset(keys=keys, dataset=dataset)
    return get_filter_expr(df, data_filter)

def run_job(database_api, dataset_api, email, job_id, job_type, dataset, params):
    database_api.update_job(email=email, job_id=job_id, status='running', result=None)
    schema = dataset_api.schema(dataset)
    var_names = schema['var']
    nfeatures = len(var_names)
    mask1 = get_mask(dataset_api, dataset, params['filter'])
    batch_size = 1000  # TODO
    percent_expressed1 = np.zeros(nfeatures)
    percent_expressed2 = np.zeros(nfeatures)
    scores = np.zeros(nfeatures)
    pvals = np.ones(nfeatures)
    is_sparse = None
    v1_size = mask1.sum()

    if job_type == 'de':
        avg_log2FC = np.zeros(nfeatures)
        mask2 = get_mask(dataset_api, dataset, params['filter2'])
        v2_size = mask2.sum()

    elif job_type == 'corr':
        df = dataset_api.read_dataset(keys=dict(X=[params['ref']]), dataset=dataset)
        df = df[mask1]
        ref = df[params['ref']]
        is_sparse = hasattr(ref, 'sparse')
        if is_sparse:
            ref = ref.sparse.to_dense()
        is_pearson = False  # params.get('method', 'pearson') == 'pearson'
        if not is_pearson:
            ref_expressed = (ref != 0)
            ref_not_expressed = ~ref_expressed
    index = 0
    for i in range(0, nfeatures, batch_size):
        start = i
        end = min(nfeatures, start + batch_size)
        features = var_names[start:end]
        df = dataset_api.read_dataset(keys=dict(X=features), dataset=dataset)
        df1 = df[mask1]
        if job_type == 'de':
            df2 = df[mask2]
            for feature in features:
                v1 = df1[feature]
                v2 = df2[feature]

                if is_sparse is None:
                    is_sparse = hasattr(v1, 'sparse')
                if is_sparse:
                    v1 = v1.sparse.to_dense()
                    v2 = v2.sparse.to_dense()
                avg_log2FC[index] = np.log2(np.expm1(v1).mean() + 1) - np.log2(np.expm1(v2).mean() + 1)
                percent_expressed1[index] = 100 * ((v1 != 0).sum() / v1_size)
                percent_expressed2[index] = 100 * ((v2 != 0).sum() / v2_size)
                try:
                    scores[index], pvals[index] = ss.mannwhitneyu(v1, v2, alternative="two-sided", use_continuity=False)
                except ValueError:  # All numbers are identical
                    pass
                index += 1
        elif job_type == 'corr':
            for feature in features:
                v1 = df1[feature]
                if is_sparse:
                    v1 = v1.sparse.to_dense()
                try:
                    if is_pearson:
                        scores[index], pvals[index] = ss.pearsonr(ref, v1)
                    else:
                        #              gene1_exp, gene1_not_exp
                        # gene2_exp
                        # gene2_not_exp

                        a = (v1[ref_expressed] != 0).sum()
                        b = (v1[ref_not_expressed] != 0).sum()
                        c = (v1[ref_expressed] == 0).sum()
                        d = (v1[ref_not_expressed] == 0).sum()
                        scores[index], pvals[index] = ss.fisher_exact(
                            [[a, b], [c, d]])
                except ValueError:  # All numbers are identical
                    pass
                index += 1
        database_api.update_job(email=email, job_id=job_id, status='running {:.0f}%'.format(100 * index / nfeatures),
                                result=None)
    pvals[np.isnan(pvals)] = 1
    pvals = fdrcorrection(pvals)
    if job_type == 'de':
        result_df = pd.DataFrame(
            data={'index': var_names, 'comparison:pvals_adj': pvals, 'comparison:avg_log2FC': avg_log2FC,
                  'comparison:scores': scores, 'comparison:percent_expressed1': percent_expressed1,
                  'comparison:percent_expressed2': percent_expressed2})
        result = dict(groups=['comparison'],
                      fields=['pvals_adj', 'avg_log2FC', 'scores', 'percent_expressed1', 'percent_expressed2'],
                      data=result_df.to_dict(orient='records'))
    elif job_type == 'corr':
        result_df = pd.DataFrame(
            data={'index': var_names, 'selection:pvals_adj': pvals, 'selection:scores': scores})
        result = dict(groups=['selection'], fields=['pvals_adj', 'scores'], data=result_df.to_dict(orient='records'))
    database_api.update_job(email=email, job_id=job_id, status='complete', result=result)

max_stars_repo_name: jkanche/cirrocumulus

max_issues_repo_name: jkanche/cirrocumulus

max_forks_repo_name: jkanche/cirrocumulus


text:
import os
import time
import random
import argparse
import numpy as np

import torch
import torch.distributed as dist
import torch.backends.cudnn as cudnn

# os.environ['TORCH_HOME']='~/.cache/torch/'
os.environ['TORCH_HOME']='/home/data/equation/submit_template_for_AI_challenger_sea6/ckpt/'

def reduce_tensor(tensor, n):
    rt = tensor.clone()
    dist.all_reduce(rt, op=dist.ReduceOp.SUM)
    return rt

from utils import MyCustomDataset, get_architecture, Input_diversity

parser = argparse.ArgumentParser(description='Benchmarking ImageNet models under transfer-based unrestricted attacks')
parser.add_argument('--batch_size', type=int, default=10, metavar='N', help='batch size for attack (default: 30)')
parser.add_argument('--input_path', type=str, default="data/images", help='The path of adversarial examples to evaluate on the benchmark')
parser.add_argument('--mode', type=str, default="nearest")
parser.add_argument('--prob', default=0.7, type=float,help='input diversity prob')
parser.add_argument('--scale', default=0.1, type=float,help='input diversity scale')
parser.add_argument('--distributed', action='store_true',help='Use multi-processing distributed training to launch')
parser.add_argument("--local_rank", type=int, help='local rank for DistributedDataParallel')

NUM_CLASSES = 1000

MODEL_NAME_DICT = {
    0: "ens_adv_inception_resnet_v2",
    1: "Fast_AT",
    2: "Free_AT",
    3: "AdvResnet152",
    4: "Resnext101-DenoiseAll",
    5: "Resnet152-DenoiseAll",
    6: "RVT-Tiny",
    7: "DeepAugment_AugMix",
    8: "tf_efficientnet_l2_ns_475",
    9: "swin_large_patch4_window12_384",
}

def init_seeds(seed=0, cuda_deterministic=True):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    # Speed-reproducibility tradeoff https://pytorch.org/docs/stable/notes/randomness.html
    if cuda_deterministic:  # slower, more reproducible
        cudnn.deterministic = True
        cudnn.benchmark = False
    else:  # faster, less reproducible
        cudnn.deterministic = False
        cudnn.benchmark = True

def main(args):
    init_seeds(cuda_deterministic=True)

    # Target model
    target_id_list = list(range(0, 10))
    if not args.distributed or args.local_rank == 0:
        print("Target id list: {}".format(target_id_list))

    Target_model_list = []
    for idx in target_id_list:
        temp_model = get_architecture(model_name=MODEL_NAME_DICT[idx]).cuda().eval()
        temp_model = Input_diversity(temp_model, args, num_classes=NUM_CLASSES, prob=args.prob, mode=args.mode, diversity_scale=args.scale)
        Target_model_list.append(temp_model)

    loader = MyCustomDataset(img_path=args.input_path)

    if args.distributed:
        sampler = torch.utils.data.distributed.DistributedSampler(loader)
    else:
        sampler = torch.utils.data.SequentialSampler(loader)

    attack_loader = torch.utils.data.DataLoader(dataset=loader,
                                                batch_size=args.batch_size,
                                                shuffle=False,
                                                sampler=sampler, num_workers=8, pin_memory=True)

    target_err_total = torch.zeros((10, )).cuda()

    count = 0
    time_start = time.time()
    if not args.distributed or args.local_rank == 0:
        print("Starting time counting: {}\n".format(time_start))

    for (img, label, img_name) in attack_loader:
        img, label = img.cuda(), label.cuda()
        batch_szie = img.shape[0]

        count += batch_szie

        target_err_now = []
        with torch.no_grad():
            for idx in range(0, 10):
                out_target = Target_model_list[idx](img, diversity=False).view(batch_szie, -1)
                mask = (out_target.data.max(1)[1] != label.data)
                err_target = mask.float().sum()
                target_err_now.append(err_target.item())
                target_err_total[idx] += err_target

        print("img_id: {}, Attacked: {}, Batch size: {}, Target error: {}".format(img_name, count, batch_szie, target_err_now))

    if args.distributed:
        target_err_total = reduce_tensor(target_err_total.data, args.world_size)

    torch.cuda.synchronize()

    time_end = time.time()

    if not args.distributed or args.local_rank == 0:
        print('time cost', time_end-time_start, 's')
        print("Target Success total: ", target_err_total)
        print("Benchmarking Accuracy: ", [(1000 - item.item()) / 1000 for item in target_err_total])

if __name__ == "__main__":
    opt = parser.parse_args()

    if opt.distributed:
        torch.cuda.set_device(opt.local_rank)
        torch.distributed.init_process_group(backend='nccl', init_method='env://')
        opt.world_size = dist.get_world_size()
        opt.batch_size = int(opt.batch_size / opt.world_size)

    if not opt.distributed or opt.local_rank == 0:
        print(opt)
    main(opt)

ext: py

max_stars_repo_name: Equationliu/GA-Attack

max_issues_repo_name: Equationliu/GA-Attack

max_forks_repo_name: Equationliu/GA-Attack

reason: import numpy


text:
'''
Copyright (c) Facebook, Inc. and its affiliates.
All rights reserved.

This source code is licensed under the license found in the
LICENSE file in the root directory of this source tree.
'''

import torch
from torch.nn import functional as F
import os
import sys
import copy
import argparse
from tqdm import tqdm
import pickle
from pytorch_transformers import *

import numpy as np
import random

from wsd_models.util import *

parser = argparse.ArgumentParser(description='BERT Frozen Probing Model for WSD')
parser.add_argument('--rand_seed', type=int, default=42)
parser.add_argument('--silent', action='store_true',
	help='Flag to supress training progress bar for each epoch')
parser.add_argument('--lr', type=float, default=0.0001)
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--bsz', type=int, default=128)
parser.add_argument('--ckpt', type=str, required=True,
	help='filepath at which to save best probing model (on dev set)')
parser.add_argument('--encoder-name', type=str, default='bert-base',
	choices=['bert-base', 'bert-large', 'roberta-base', 'roberta-large'])
parser.add_argument('--kshot', type=int, default=-1,
	help='if set to k (1+), will filter training data to only have up to k examples per sense')
parser.add_argument('--data-path', type=str, required=True,
	help='Location of top-level directory for the Unified WSD Framework')

parser.add_argument('--eval', action='store_true',
	help='Flag to set script to evaluate probe (rather than train)')
parser.add_argument('--split', type=str, default='semeval2007',
	choices=['semeval2007', 'senseval2', 'senseval3', 'semeval2013', 'semeval2015', 'ALL', 'all-test'],
	help='Which evaluation split on which to evaluate probe')

def wn_keys(data):
	keys = []
	for sent in data:
		for form, lemma, pos, inst, _ in sent:
			if inst != -1:
				key = generate_key(lemma, pos)
				keys.append(key)
	return keys

def batchify(data, bsz=1):
	print('Batching data with bsz={}...'.format(bsz))
	batched_data = []
	for i in range(0, len(data), bsz):
		if i+bsz < len(data): d_arr = data[i:i+bsz]
		else: d_arr = data[i:] #get remainder examples
		batched_ids = torch.cat([ids for ids, _, _, _ in d_arr], dim=0)
		batched_masks = torch.stack([mask for _, mask, _, _ in d_arr], dim=0)
		batched_insts = [inst for _, _, inst, _ in d_arr]
		batched_labels = torch.cat([label for _, _, _, label in d_arr], dim=0)
		batched_data.append((batched_ids, batched_masks, batched_insts, batched_labels))
	return batched_data

#takes in text data, tensorizes it for BERT, runs though BERT,
#filters out the context words (not labeled), and averages
#the representation(s) for words/phrases to be disambiguated
#output is tuples of (input tensor prepared for linear probing model,
#instance numbers (for dataset), tensor of label indexes)
def preprocess(tokenizer, context_model, text_data, label_space, label_map):
	processed_examples = []
	output_masks = []
	instances = []
	label_indexes = []

	#tensorize data
	for sent in tqdm(text_data):
		sent_ids = [torch.tensor([tokenizer.encode(tokenizer.cls_token)])] #aka sos token, returns a list with single index
		bert_mask = [-1]
		for idx, (word, lemma, pos, inst, label) in enumerate(sent):
			word_ids = torch.tensor([tokenizer.encode(word.lower())])
			sent_ids.append(word_ids)

			if inst != -1:
				#masking for averaging of bert outputs
				bert_mask.extend([idx]*word_ids.size(-1))

				#tracking instance for sense-labeled word
				instances.append(inst)

				#adding label tensor for senes-labeled word
				if label in label_space:
					label_indexes.append(torch.tensor([label_space.index(label)]))
				else:
					label_indexes.append(torch.tensor([label_space.index('n/a')]))

				#adding appropriate label space for sense-labeled word (we only use this for wsd task)
				key = generate_key(lemma, pos)
				if key in label_map:
					l_space = label_map[key]
					o_mask = torch.zeros(len(label_space))
					for l in l_space: o_mask[l] = 1
					output_masks.append(o_mask)
				else:
					output_masks.append(torch.ones(len(label_space))) #let this predict whatever -- should not use this (default to backoff for unseen forms)

			else:
				bert_mask.extend([-1]*word_ids.size(-1))

		#add eos token
		sent_ids.append(torch.tensor([tokenizer.encode(tokenizer.sep_token)])) #aka eos token
		bert_mask.append(-1)

		sent_ids = torch.cat(sent_ids, dim=-1)

		#run inputs through frozen bert
		sent_ids = sent_ids.cuda()

		with torch.no_grad():
			output = context_model(sent_ids)[0].squeeze().cpu()

		#average outputs for subword units in same word/phrase, drop unlabeled words
		combined_outputs = process_encoder_outputs(output, bert_mask)
		processed_examples.extend(combined_outputs)

	#package preprocessed data together + return
	data = list(zip(processed_examples, output_masks, instances, label_indexes))
	return data

def _train(train_data, probe, optim, criterion, bsz=1, silent=False):
	if not silent: train_data = tqdm(train_data)
	for input_ids, output_mask, _, label in train_data:
		input_ids = input_ids.cuda()
		output_mask = output_mask.cuda()
		label = label.cuda()

		optim.zero_grad()
		output = probe(input_ids)
		#mask to candidate senses for target word
		output = torch.mul(output, output_mask)
		#set masked out items to -inf to get proper probabilities over the candidate senses
		output[output == 0] = float('-inf')

		output = F.softmax(output, dim=-1)
		loss = criterion(output, label)

		batch_sz = loss.size(0)
		loss = loss.sum()/batch_sz
		loss.backward()
		optim.step()

	return probe, optim

def _eval(eval_data, probe, label_space):
	eval_preds = []
	for input_ids, output_mask, inst, _ in eval_data:
		input_ids = input_ids.cuda()
		output_mask = output_mask.cuda()

		#run example through model
		with torch.no_grad():
			output = probe(input_ids)
			#mask to candidate senses for target word
			output = torch.mul(output, output_mask)
			#set masked out items to -inf to get proper probabilities over the candidate senses
			output[output == 0] = float('-inf')
			output = F.softmax(output, dim=-1)

		#get predicted label
		pred_id = output.topk(1, dim=-1)[1].squeeze().item()
		pred_label = label_space[pred_id]
		eval_preds.append((inst[0], pred_label))
	return eval_preds

def _eval_with_backoff(eval_data, probe, label_space, wn_senses, coverage, keys):
	eval_preds = []
	for key, (input_ids, output_mask, inst, _) in zip(keys, eval_data):
		input_ids = input_ids.cuda()
		output_mask = output_mask.cuda()

		if key in coverage:
			#run example through model
			with torch.no_grad():
				output = probe(input_ids)
				output = torch.mul(output, output_mask)
				#set masked out items to -inf to get proper probabilities over the candidate senses
				output[output == 0] = float('-inf')

				output = F.softmax(output, dim=-1)
			#get predicted label
			pred_id = output.topk(1, dim=-1)[1].squeeze().item()
			pred_label = label_space[pred_id]
			eval_preds.append((inst[0], pred_label))
		#backoff to wsd for lemma+pos
		else:
			#this is ws1 for given key
			pred_label = wn_senses[key][0]
			eval_preds.append((inst[0], pred_label))

	return eval_preds

def train_probe(args):
	lr = args.lr
	bsz = args.bsz

	#create passed in ckpt dir if doesn't exist
	if not os.path.exists(args.ckpt): os.mkdir(args.ckpt)

	'''
	LOAD PRETRAINED BERT MODEL
	'''

	#model loading code based on pytorch_transformers README example
	tokenizer = load_tokenizer(args.encoder_name)
	pretrained_model, output_dim = load_pretrained_model(args.encoder_name)
	pretrained_model = pretrained_model.cuda()

	'''
	LOADING IN TRAINING AND EVAL DATA
	'''
	print('Loading data + preprocessing...')
	sys.stdout.flush()
	#loading WSD (semcor) data + convert to supersenses
	train_path = os.path.join(args.data_path, 'Training_Corpora/SemCor/')
	train_data = load_data(train_path, 'semcor')

	#filter train data for k-shot learning
	if args.kshot > 0:
		train_data = filter_k_examples(train_data, args.kshot)

	task_labels, label_map = get_label_space(train_data)
	print('num labels = {} + 1 unknown label'.format(len(task_labels)-1))

	train_data = preprocess(tokenizer, pretrained_model, train_data, task_labels, label_map)
	train_data = batchify(train_data, bsz=args.bsz)

	num_epochs = args.epochs
	if args.kshot > 0:
		NUM_STEPS = 176600 #hard coded for fair comparision with full model on default num. of epochs
		num_batches = len(train_data)
		num_epochs = NUM_STEPS//num_batches #recalculate number of epochs
		overflow_steps = NUM_STEPS%num_batches #num steps in last overflow epoch (if there is one, otherwise 0)
		t_total = NUM_STEPS #manually set number of steps for lr schedule
		if overflow_steps > 0: num_epochs+=1 #add extra epoch for overflow steps
		print('Overriding args.epochs and training for {} epochs...'.format(epochs))

	#loading eval data & convert to supersense tags
	#dev set = semeval2007
	semeval2007_path = os.path.join(args.data_path, 'Evaluation_Datasets/semeval2007/')
	semeval2007_data = load_data(semeval2007_path, 'semeval2007')
	semeval2007_data = preprocess(tokenizer, pretrained_model, semeval2007_data, task_labels, label_map)
	semeval2007_data = batchify(semeval2007_data, bsz=1)

	'''
	SET UP PROBING MODEL FOR TASK
	'''

	#probing model = projection layer to label space, loss function, and optimizer
	probe = torch.nn.Linear(output_dim, len(task_labels))
	probe = probe.cuda()
	criterion = torch.nn.CrossEntropyLoss(reduction='none')
	optim = torch.optim.Adam(probe.parameters(), lr=lr)

	'''
	TRAIN PROBING MODEL ON SEMCOR DATA
	'''

	best_dev_f1 = 0.
	print('Training probe...')
	sys.stdout.flush()
	for epoch in range(1, num_epochs+1):
		#train on full dataset
		probe_optim = _train(train_data, probe, optim, criterion, bsz=bsz, silent=args.silent)

		#eval probe on dev set (semeval2007)
		eval_preds = _eval(semeval2007_data, probe, task_labels)

		#generate predictions file
		pred_filepath = os.path.join(args.ckpt, 'tmp_predictions.txt')
		with open(pred_filepath, 'w') as f:
			for inst, prediction in eval_preds:
				f.write('{} {}\n'.format(inst, prediction))

		#run predictions through scorer
		gold_filepath = os.path.join(args.data_path, 'Evaluation_Datasets/semeval2007/semeval2007.gold.key.txt')
		scorer_path = os.path.join(args.data_path, 'Evaluation_Datasets')
		_, _, dev_f1 = evaluate_output(scorer_path, gold_filepath, pred_filepath)
		print('Dev f1 after {} epochs = {}'.format(epoch, dev_f1))
		sys.stdout.flush()

		if dev_f1 >= best_dev_f1:
			print('updating best model at epoch {}...'.format(epoch))
			sys.stdout.flush()
			best_dev_f1 = dev_f1
			#save to file if best probe so far on dev set
			probe_fname = os.path.join(args.ckpt, 'best_model.ckpt')
			with open(probe_fname, 'wb') as f:
				torch.save(probe.state_dict(), f)
		sys.stdout.flush()

		#shuffle train data after every epoch
		random.shuffle(train_data)

	return

def evaluate_probe(args):
	print('Evaluating WSD probe on {}...'.format(args.split))

	'''
	LOAD TOKENIZER + BERT MODEL
	'''
	tokenizer = load_tokenizer(args.encoder_name)
	pretrained_model, output_dim = load_pretrained_model(args.encoder_name)
	pretrained_model = pretrained_model.cuda()

	'''
	GET LABEL SPACE
	'''
	train_path = os.path.join(args.data_path, 'Training_Corpora/SemCor/')
	train_data = load_data(train_path, 'semcor')
	task_labels, label_map = get_label_space(train_data)
	#for backoff eval
	train_keys = wn_keys(train_data)
	coverage = set(train_keys)

	'''
	LOAD TRAINED PROBE
	'''
	probe = torch.nn.Linear(output_dim, len(task_labels))
	probe_path = os.path.join(args.ckpt, 'best_model.ckpt')
	probe.load_state_dict(torch.load(probe_path))
	probe = probe.cuda()

	'''
	LOAD EVAL SET
	'''
	eval_path = os.path.join(args.data_path, 'Evaluation_Datasets/{}/'.format(args.split))
	eval_data = load_data(eval_path, args.split)
	#for backoff
	eval_keys = wn_keys(eval_data)
	eval_data = preprocess(tokenizer, pretrained_model, eval_data, task_labels, label_map)
	eval_data = batchify(eval_data, bsz=1)

	'''
	EVALUATE PROBE w/o backoff
	'''
	eval_preds = _eval(eval_data, probe, task_labels)

	#generate predictions file
	pred_filepath = os.path.join(args.ckpt, './{}_predictions.txt'.format(args.split))
	with open(pred_filepath, 'w') as f:
		for inst, prediction in eval_preds:
			f.write('{} {}\n'.format(inst, prediction))

	#run predictions through scorer
	gold_filepath = os.path.join(eval_path, '{}.gold.key.txt'.format(args.split))
	scorer_path = os.path.join(args.data_path, 'Evaluation_Datasets')
	p, r, f1 = evaluate_output(scorer_path, gold_filepath, pred_filepath)
	print('f1 of WSD probe on {} test set = {}'.format(args.split, f1))

	'''
	EVALUATE PROBE with backoff
	'''
	wn_path = os.path.join(args.data_path, 'Data_Validation/candidatesWN30.txt')
	wn_senses = load_wn_senses(wn_path)
	eval_preds = _eval_with_backoff(eval_data, probe, task_labels, wn_senses, coverage, eval_keys)

	#generate predictions file
	pred_filepath = os.path.join(args.ckpt, './{}_backoff_predictions.txt'.format(args.split))
	with open(pred_filepath, 'w') as f:
		for inst, prediction in eval_preds:
			f.write('{} {}\n'.format(inst, prediction))

	#run predictions through scorer
	gold_filepath = os.path.join(eval_path, '{}.gold.key.txt'.format(args.split))
	scorer_path = os.path.join(args.data_path, 'Evaluation_Datasets')
	p, r, f1 = evaluate_output(scorer_path, gold_filepath, pred_filepath)
	print('f1 of BERT probe (with backoff) = {}'.format(f1))

	return

if __name__ == "__main__":
	if not torch.cuda.is_available():
		print("Need available GPU(s) to run this model...")
		quit()

	args = parser.parse_args()
	print(args)

	#set random seeds
	torch.manual_seed(args.rand_seed)
	os.environ['PYTHONHASHSEED'] = str(args.rand_seed)

	torch.cuda.manual_seed(args.rand_seed)
	torch.cuda.manual_seed_all(args.rand_seed)

	np.random.seed(args.rand_seed)
	random.seed(args.rand_seed)
	torch.backends.cudnn.benchmark = False
	torch.backends.cudnn.deterministic=True

	if args.eval:
		evaluate_probe(args)
	else:
		train_probe(args)


text:
# -*- coding: utf-8 -*-
"""
Created on Tue Dec 10 20:29:06 2019

@author: User
"""

# General packages for text reading
#import requests
import pandas as pd
import os

# Packages for text preprocessing
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.util import ngrams
from nltk import bigrams
import itertools
import collections
import re

#import seaborn as sns
import pickle
import random
from collections import Counter

# Visualization
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
import nltk
import networkx as nx

def remove_sw(words,sw_list):
    return [word for word in words if word not in sw_list]

scriptPath = 'C:/Users/User/Desktop/texto_conpes'

# Absolute working directory
# Necessary for relative call inside the next loops
os.chdir(scriptPath)

# Connect to postgres
import psycopg2

try:
    connection = psycopg2.connect(user = "postgres",
                                  password = "D1iv8Mv2a135nrW7kYB1",
                                  host = "nps-demo-instance.cmbnhcfpuqng.us-east-2.rds.amazonaws.com",
                                  port = "5432",
                                  database = "Team28_FP")

    cursor = connection.cursor()
    # Print PostgreSQL Connection properties
    print ( connection.get_dsn_parameters(),"\n")
    sql_query = """
    SELECT
    c.dt_publicacion,
    tc.numero,
    c.titulo,
    tp.description,
    tc.texto,
    tc.textolimpio,
    p.presidente
    FROM
    CONPES C
    JOIN tipoconpes tp on c.idtipoconpes = tp.idtipoconpes
    JOIN PRESIDENTES P
    ON C.DT_PUBLICACION >= P.INICIO AND C.DT_PUBLICACION <= P.FIN
    JOIN textconpes tc ON tc.idtipoconpes = tp.idtipoconpes AND tc.numero = c.numero
    ORDER BY TC.NUMERO DESC
"""

    df_conpes = pd.read_sql(sql_query , connection)

    # Print PostgreSQL version
    cursor.execute("SELECT version();")
    postgreSQL_select_Query = "select * from textconpes"
    #cursor.execute(postgreSQL_select_Query)
    cursor.execute(sql_query)
    record = cursor.fetchone()
    conpes_text_2 = cursor.fetchall()
    print("You are connected to - ", record,"\n")

except (Exception, psycopg2.Error) as error :
    print ("Error while connecting to PostgreSQL", error)
finally:
    #closing database connection.
        if(connection):
            cursor.close()
            connection.close()
            print("PostgreSQL connection is closed")

#------------------------------------------------------
#Complete database with Duques Conpes
try:
    connection = psycopg2.connect(user = "postgres",
                                  password = "D1iv8Mv2a135nrW7kYB1",
                                  host = "nps-demo-instance.cmbnhcfpuqng.us-east-2.rds.amazonaws.com",
                                  port = "5432",
                                  database = "Team28_FP")

    cursor = connection.cursor()
    # Print PostgreSQL Connection properties
    print ( connection.get_dsn_parameters(),"\n")

    #df_conpes = pd.read_sql(sql_query , connection)

    # Print PostgreSQL version
    cursor.execute("SELECT version();")
    postgreSQL_select_Query = "select * from textconpes"
    cursor.execute(postgreSQL_select_Query)
    #cursor.execute(sql_query)
    record = cursor.fetchone()
    conpes_text_3 = cursor.fetchall()
    print("You are connected to - ", record,"\n")

except (Exception, psycopg2.Error) as error :
    print ("Error while connecting to PostgreSQL", error)
finally:
    #closing database connection.
        if(connection):
            cursor.close()
            connection.close()
            print("PostgreSQL connection is closed")

conpes_text = pd.DataFrame(conpes_text_2)

# Reset colnames
conpes_text.columns = ['fecha','numero','titulo','tipo','texto','textolimpio','presidente']

conpes_text.set_index('fecha', inplace=True)
conpes_text.index = pd.to_datetime(conpes_text.index)

test = conpes_text.resample('Y', kind='period', convention='start').agg({'texto': lambda x: ' '.join(x)})

# Datos abiertos

conteo_1 = []
for index, row in test.iterrows():
    temp = row['texto'].count('datos abiertos')
    conteo_1.append(temp)

sample = {'year':test.index, 'count':conteo_1}

datos_abiertos = pd.DataFrame(sample)

no_she = [0]
no_she = no_she * 52
no_she[46] = 26.71
no_she[47] = 32.38
no_she[48] = 45.39
no_she[49] = 51.65
no_she[50] = 80
no_she

sample_2 = {'year':test.index, 'index':no_she}
datos_abiertos_2 = pd.DataFrame(sample_2)

datos_abiertos = datos_abiertos.set_index(datos_abiertos.year)
datos_abiertos_2 = datos_abiertos_2.set_index(datos_abiertos_2.year)

fig, ax = plt.subplots()
datos_abiertos.plot(ax=ax)
datos_abiertos_2.plot(ax=ax)
plt.show()
plt.title('Open Data Barometer vs ocurrences of "datos abiertos" in CONPES documents ')

#############################################3

conteo_2= []
for index, row in test.iterrows():
    temp = row['texto'].count('desigualdad')
    conteo_2.append(temp)

sample_3 = {'year':test.index, 'count':conteo_2}
desigualdad = pd.DataFrame(sample_3)
desigualdad = desigualdad.tail(9)

no_she_2 = [27.5 , 30.1, 26.6, 25.7, 24.1, 23.7, 22.6, 22.9, 22.4]

sample_4 = {'year':desigualdad.year, 'index':no_she_2}
desigualdad_2 = pd.DataFrame(sample_4)

desigualdad = desigualdad.set_index(desigualdad.year)
desigualdad_2 = desigualdad_2.set_index(desigualdad_2.year)

fig, ax = plt.subplots()
desigualdad.plot(ax=ax)
desigualdad_2.plot(ax=ax)
plt.show()
plt.title('Human Inequality Index vs ocurrences of "desigualdad" in CONPES documents ')

#############################################

conteo_3= []
for index, row in test.iterrows():
    temp = row['texto'].count('medio ambiente')
    conteo_3.append(temp)

sample_5 = {'year':test.index, 'count':conteo_3}
internet = pd.DataFrame(sample_5)
internet = internet.tail(9)
internet = internet.head(7)

no_she_3 = [1.3*100,1.4*100,1.4*100,		1.6*100,		1.7*100,		1.6*100,		1.8*100]

sample_6 = {'year':internet.year, 'index':no_she_3}
internet_2 = pd.DataFrame(sample_6)

internet = internet.set_index(internet.year)
internet_2 = internet_2.set_index(internet_2.year)

fig, ax = plt.subplots()
internet.plot(ax=ax)
internet_2.plot(ax=ax)
plt.show()
plt.title('Gender inequality vs ocurrences of "género" in CONPES documents ')

pd.DataFrame(conteo_3).plot()

reason: import numpy,import networkx


text:
import torch
import copy
import pprint
import warnings
import numpy as np
from functools import reduce
from numbers import Number
from typing import Any, List, Tuple, Union, Iterator, Optional

# Disable pickle warning related to torch, since it has been removed
# on torch master branch. See Pull Request #39003 for details:
# https://github.com/pytorch/pytorch/pull/39003
warnings.filterwarnings(
    "ignore", message="pickle support for Storage will be removed in 1.5.")

def _is_batch_set(data: Any) -> bool:
    if isinstance(data, (list, tuple)):
        if len(data) > 0 and isinstance(data[0], (dict, Batch)):
            return True
    elif isinstance(data, np.ndarray):
        if isinstance(data.item(0), (dict, Batch)):
            return True
    return False

def _valid_bounds(length: int, index: Union[
        slice, int, np.integer, np.ndarray, List[int]]) -> bool:
    if isinstance(index, (int, np.integer)):
        return -length <= index and index < length
    elif isinstance(index, (list, np.ndarray)):
        return _valid_bounds(length, np.min(index)) and \
            _valid_bounds(length, np.max(index))
    elif isinstance(index, slice):
        if index.start is not None:
            start_valid = _valid_bounds(length, index.start)
        else:
            start_valid = True
        if index.stop is not None:
            stop_valid = _valid_bounds(length, index.stop - 1)
        else:
            stop_valid = True
        return start_valid and stop_valid

def _create_value(inst: Any, size: int) -> Union['Batch', np.ndarray]:
    if isinstance(inst, np.ndarray):
        return np.full((size, *inst.shape),
                       fill_value=None if inst.dtype == np.object else 0,
                       dtype=inst.dtype)
    elif isinstance(inst, torch.Tensor):
        return torch.full((size, *inst.shape),
                          fill_value=None if inst.dtype == np.object else 0,
                          device=inst.device,
                          dtype=inst.dtype)
    elif isinstance(inst, (dict, Batch)):
        zero_batch = Batch()
        for key, val in inst.items():
            zero_batch.__dict__[key] = _create_value(val, size)
        return zero_batch
    elif isinstance(inst, (np.generic, Number)):
        return _create_value(np.asarray(inst), size)
    else:  # fall back to np.object
        return np.array([None for _ in range(size)])

class Batch:
    """Tianshou provides :class:`~tianshou.data.Batch` as the internal data
    structure to pass any kind of data to other methods, for example, a
    collector gives a :class:`~tianshou.data.Batch` to policy for learning.
    Here is the usage:
    ::

        >>> import numpy as np
        >>> from tianshou.data import Batch
        >>> data = Batch(a=4, b=[5, 5], c='2312312')
        >>> data.b
        [5, 5]
        >>> data.b = np.array([3, 4, 5])
        >>> print(data)
        Batch(
            a: 4,
            b: array([3, 4, 5]),
            c: '2312312',
        )

    In short, you can define a :class:`Batch` with any key-value pair. The
    current implementation of Tianshou typically use 7 reserved keys in
    :class:`~tianshou.data.Batch`:

    * ``obs`` the observation of step :math:`t` ;
    * ``act`` the action of step :math:`t` ;
    * ``rew`` the reward of step :math:`t` ;
    * ``done`` the done flag of step :math:`t` ;
    * ``obs_next`` the observation of step :math:`t+1` ;
    * ``info`` the info of step :math:`t` (in ``gym.Env``, the ``env.step()``\
        function return 4 arguments, and the last one is ``info``);
    * ``policy`` the data computed by policy in step :math:`t`;

    :class:`~tianshou.data.Batch` object can be initialized using wide variety
    of arguments, starting with the key/value pairs or dictionary, but also
    list and Numpy arrays of :class:`dict` or Batch instances. In which case,
    each element is considered as an individual sample and get stacked
    together:
    ::

        >>> import numpy as np
        >>> from tianshou.data import Batch
        >>> data = Batch([{'a': {'b': [0.0, "info"]}}])
        >>> print(data[0])
        Batch(
            a: Batch(
                b: array(['0.0', 'info'], dtype='<U32'),
            ),
        )

    :class:`~tianshou.data.Batch` has the same API as a native Python
    :class:`dict`. In this regard, one can access to stored data using string
    key, or iterate over stored data:
    ::

        >>> from tianshou.data import Batch
        >>> data = Batch(a=4, b=[5, 5])
        >>> print(data["a"])
        4
        >>> for key, value in data.items():
        >>>     print(f"{key}: {value}")
        a: 4
        b: [5, 5]

    :class:`~tianshou.data.Batch` is also reproduce partially the Numpy API for
    arrays. You can access or iterate over the individual samples, if any:
    ::

        >>> import numpy as np
        >>> from tianshou.data import Batch
        >>> data = Batch(a=np.array([[0.0, 2.0], [1.0, 3.0]]), b=[5, -5])
        >>> print(data[0])
        Batch(
            a: np.array([0.0, 2.0])
            b: 5
        )
        >>> for sample in data:
        >>>     print(sample.a)
        [0.0, 2.0]
        [1.0, 3.0]

    Similarly, one can also perform simple algebra on it, and stack, split or
    concatenate multiple instances:
    ::

        >>> import numpy as np
        >>> from tianshou.data import Batch
        >>> data_1 = Batch(a=np.array([0.0, 2.0]), b=5)
        >>> data_2 = Batch(a=np.array([1.0, 3.0]), b=-5)
        >>> data = Batch.stack((data_1, data_2))
        >>> print(data)
        Batch(
            b: array([ 5, -5]),
            a: array([[0., 2.],
                      [1., 3.]]),
        )
        >>> print(np.mean(data))
        Batch(
            b: 0.0,
            a: array([0.5, 2.5]),
        )
        >>> data_split = list(data.split(1, False))
        >>> print(list(data.split(1, False)))
        [Batch(
            b: [5],
            a: array([[0., 2.]]),
        ),
        Batch(
            b: [-5],
            a: array([[1., 3.]]),
        )]
        >>> data_cat = Batch.cat(data_split)
        >>> print(data_cat)
        Batch(
            b: array([ 5, -5]),
            a: array([[0., 2.],
                      [1., 3.]]),
        )

    Note that stacking of inconsistent data is also supported. In which case,
    None is added in list or :class:`np.ndarray` of objects, 0 otherwise.
    ::

        >>> import numpy as np
        >>> from tianshou.data import Batch
        >>> data_1 = Batch(a=np.array([0.0, 2.0]))
        >>> data_2 = Batch(a=np.array([1.0, 3.0]), b='done')
        >>> data = Batch.stack((data_1, data_2))
        >>> print(data)
        Batch(
            a: array([[0., 2.],
                      [1., 3.]]),
            b: array([None, 'done'], dtype=object),
        )

    :meth:`~tianshou.data.Batch.size` and :meth:`~tianshou.data.Batch.__len__`
    methods are also provided to respectively get the length and the size of
    a :class:`Batch` instance. It mimics the Numpy API for Numpy arrays, which
    means that getting the length of a scalar Batch raises an exception, while
    the size is 1. The size is only 0 if empty. Note that the size and length
    are the identical if multiple samples are stored:
    ::

        >>> import numpy as np
        >>> from tianshou.data import Batch
        >>> data = Batch(a=[5., 4.], b=np.zeros((2, 3, 4)))
        >>> data.size
        2
        >>> len(data)
        2
        >>> data[0].size
        1
        >>> len(data[0])
        TypeError: Object of type 'Batch' has no len()

    Convenience helpers are available to convert in-place the stored data into
    Numpy arrays or Torch tensors.

    Finally, note that :class:`~tianshou.data.Batch` instance are serializable
    and therefore Pickle compatible. This is especially important for
    distributed sampling.
    """

    def __init__(self,
                 batch_dict: Optional[Union[
                     dict, 'Batch', Tuple[Union[dict, 'Batch']],
                     List[Union[dict, 'Batch']], np.ndarray]] = None,
                 **kwargs) -> None:
        if _is_batch_set(batch_dict):
            self.stack_(batch_dict)
        elif isinstance(batch_dict, (dict, Batch)):
            for k, v in batch_dict.items():
                if isinstance(v, dict) or _is_batch_set(v):
                    self.__dict__[k] = Batch(v)
                else:
                    self.__dict__[k] = v
        if len(kwargs) > 0:
            self.__init__(kwargs)

    def __getstate__(self):
        """Pickling interface. Only the actual data are serialized
        for both efficiency and simplicity.
        """
        state = {}
        for k, v in self.items():
            if isinstance(v, Batch):
                v = v.__getstate__()
            state[k] = v
        return state

    def __setstate__(self, state):
        """Unpickling interface. At this point, self is an empty Batch
        instance that has not been initialized, so it can safely be
        initialized by the pickle state.
        """
        self.__init__(**state)

    def __getitem__(self, index: Union[
            str, slice, int, np.integer, np.ndarray, List[int]]) -> 'Batch':
        """Return self[index]."""
        if isinstance(index, str):
            return self.__dict__[index]

        if not _valid_bounds(len(self), index):
            raise IndexError(
                f"Index {index} out of bounds for Batch of len {len(self)}.")
        else:
            b = Batch()
            is_index_scalar = isinstance(index, (int, np.integer)) or \
                (isinstance(index, np.ndarray) and index.ndim == 0)
            for k, v in self.items():
                if isinstance(v, Batch) and len(v.__dict__) == 0:
                    b.__dict__[k] = Batch()
                elif is_index_scalar or not isinstance(v, list):
                    b.__dict__[k] = v[index]
                else:
                    b.__dict__[k] = [v[i] for i in index]
            return b

    def __setitem__(self, index: Union[
                        str, slice, int, np.integer, np.ndarray, List[int]],
                    value: Any) -> None:
        """Assign value to self[index]."""
        if isinstance(index, str):
            self.__dict__[index] = value
            return
        if not isinstance(value, (dict, Batch)):
            raise TypeError("Batch does not supported value type "
                            f"{type(value)} for item assignment.")
        if not set(value.keys()).issubset(self.__dict__.keys()):
            raise KeyError(
                "Creating keys is not supported by item assignment.")
        for key, val in self.items():
            try:
                self.__dict__[key][index] = value[key]
            except KeyError:
                if isinstance(val, Batch):
                    self.__dict__[key][index] = Batch()
                elif isinstance(val, np.ndarray) and \
                        val.dtype == np.integer:
                    # Fallback for np.array of integer,
                    # since neither None or nan is supported.
                    self.__dict__[key][index] = 0
                else:
                    self.__dict__[key][index] = None

    def __iadd__(self, other: Union['Batch', Number]):
        """Algebraic addition with another :class:`~tianshou.data.Batch`
        instance in-place."""
        if isinstance(other, Batch):
            for (k, r), v in zip(self.__dict__.items(),
                                 other.__dict__.values()):
                if r is None:
                    continue
                elif isinstance(r, list):
                    self.__dict__[k] = [r_ + v_ for r_, v_ in zip(r, v)]
                else:
                    self.__dict__[k] += v
            return self
        elif isinstance(other, Number):
            for k, r in self.items():
                if r is None:
                    continue
                elif isinstance(r, list):
                    self.__dict__[k] = [r_ + other for r_ in r]
                else:
                    self.__dict__[k] += other
            return self
        else:
            raise TypeError("Only addition of Batch or number is supported.")

    def __add__(self, other: Union['Batch', Number]):
        """Algebraic addition with another :class:`~tianshou.data.Batch`
        instance out-of-place."""
        return copy.deepcopy(self).__iadd__(other)

    def __imul__(self, val: Number):
        """Algebraic multiplication with a scalar value in-place."""
        assert isinstance(val, Number), \
            "Only multiplication by a number is supported."
        for k in self.__dict__.keys():
            self.__dict__[k] *= val
        return self

    def __mul__(self, val: Number):
        """Algebraic multiplication with a scalar value out-of-place."""
        return copy.deepcopy(self).__imul__(val)

    def __itruediv__(self, val: Number):
        """Algebraic division wibyth a scalar value in-place."""
        assert isinstance(val, Number), \
            "Only division by a number is supported."
        for k in self.__dict__.keys():
            self.__dict__[k] /= val
        return self

    def __truediv__(self, val: Number):
        """Algebraic division wibyth a scalar value out-of-place."""
        return copy.deepcopy(self).__itruediv__(val)

    def __repr__(self) -> str:
        """Return str(self)."""
        s = self.__class__.__name__ + '(\n'
        flag = False
        for k, v in self.items():
            rpl = '\n' + ' ' * (6 + len(k))
            obj = pprint.pformat(v).replace('\n', rpl)
            s += f'    {k}: {obj},\n'
            flag = True
        if flag:
            s += ')'
        else:
            s = self.__class__.__name__ + '()'
        return s

    def keys(self) -> List[str]:
        """Return self.keys()."""
        return self.__dict__.keys()

    def values(self) -> List[Any]:
        """Return self.values()."""
        return self.__dict__.values()

    def items(self) -> List[Tuple[str, Any]]:
        """Return self.items()."""
        return self.__dict__.items()

    def get(self, k: str, d: Optional[Any] = None) -> Union['Batch', Any]:
        """Return self[k] if k in self else d. d defaults to None."""
        return self.__dict__.get(k, d)

    def to_numpy(self) -> None:
        """Change all torch.Tensor to numpy.ndarray. This is an in-place
        operation.
        """
        for k, v in self.items():
            if isinstance(v, torch.Tensor):
                self.__dict__[k] = v.detach().cpu().numpy()
            elif isinstance(v, Batch):
                v.to_numpy()

    def to_torch(self,
                 dtype: Optional[torch.dtype] = None,
                 device: Union[str, int, torch.device] = 'cpu'
                 ) -> None:
        """Change all numpy.ndarray to torch.Tensor. This is an in-place
        operation.
        """
        if not isinstance(device, torch.device):
            device = torch.device(device)

        for k, v in self.items():
            if isinstance(v, (np.generic, np.ndarray)):
                v = torch.from_numpy(v).to(device)
                if dtype is not None:
                    v = v.type(dtype)
                self.__dict__[k] = v
            if isinstance(v, torch.Tensor):
                if dtype is not None and v.dtype != dtype:
                    must_update_tensor = True
                elif v.device.type != device.type:
                    must_update_tensor = True
                elif device.index is not None and \
                        device.index != v.device.index:
                    must_update_tensor = True
                else:
                    must_update_tensor = False
                if must_update_tensor:
                    if dtype is not None:
                        v = v.type(dtype)
                    self.__dict__[k] = v.to(device)
            elif isinstance(v, Batch):
                v.to_torch(dtype, device)

    def append(self, batch: 'Batch') -> None:
        warnings.warn('Method append will be removed soon, please use '
                      ':meth:`~tianshou.data.Batch.cat`')
        return self.cat_(batch)

    def cat_(self, batch: 'Batch') -> None:
        """Concatenate a :class:`~tianshou.data.Batch` object into
        current batch.
        """
        assert isinstance(batch, Batch), \
            'Only Batch is allowed to be concatenated in-place!'
        for k, v in batch.items():
            if v is None:
                continue
            if not hasattr(self, k) or self.__dict__[k] is None:
                self.__dict__[k] = copy.deepcopy(v)
            elif isinstance(v, np.ndarray) and v.ndim > 0:
                self.__dict__[k] = np.concatenate([self.__dict__[k], v])
            elif isinstance(v, torch.Tensor):
                self.__dict__[k] = torch.cat([self.__dict__[k], v])
            elif isinstance(v, list):
                self.__dict__[k] += copy.deepcopy(v)
            elif isinstance(v, Batch):
                self.__dict__[k].cat_(v)
            else:
                s = 'No support for method "cat" with type '\
                    f'{type(v)} in class Batch.'
                raise TypeError(s)

    @classmethod
    def cat(cls, batches: List['Batch']) -> 'Batch':
        """Concatenate a :class:`~tianshou.data.Batch` object into a
        single new batch.
        """
        batch = cls()
        for batch_ in batches:
            batch.cat_(batch_)
        return batch

    def stack_(self,
               batches: List[Union[dict, 'Batch']],
               axis: int = 0) -> None:
        """Stack a :class:`~tianshou.data.Batch` object i into current
        batch.
        """
        if len(self.__dict__) > 0:
            batches = [self] + list(batches)
        keys_map = list(map(lambda e: set(e.keys()), batches))
        keys_shared = set.intersection(*keys_map)
        values_shared = [
            [e[k] for e in batches] for k in keys_shared]
        for k, v in zip(keys_shared, values_shared):
            if isinstance(v[0], (dict, Batch)):
                self.__dict__[k] = Batch.stack(v, axis)
            elif isinstance(v[0], torch.Tensor):
                self.__dict__[k] = torch.stack(v, axis)
            else:
                self.__dict__[k] = np.stack(v, axis)
        keys_partial = reduce(set.symmetric_difference, keys_map)
        for k in keys_partial:
            for i, e in enumerate(batches):
                val = e.get(k, None)
                if val is not None:
                    try:
                        self.__dict__[k][i] = val
                    except KeyError:
                        self.__dict__[k] = \
                            _create_value(val, len(batches))
                        self.__dict__[k][i] = val

    @staticmethod
    def stack(batches: List['Batch'], axis: int = 0) -> 'Batch':
        """Stack a :class:`~tianshou.data.Batch` object into a
        single new batch.
        """
        batch = Batch()
        batch.stack_(batches, axis)
        return batch

    def __len__(self) -> int:
        """Return len(self)."""
        r = []
        for v in self.__dict__.values():
            if isinstance(v, Batch) and len(v.__dict__) == 0:
                continue
            elif hasattr(v, '__len__') and (not isinstance(
                    v, (np.ndarray, torch.Tensor)) or v.ndim > 0):
                r.append(len(v))
            else:
                raise TypeError("Object of type 'Batch' has no len()")
        if len(r) == 0:
            raise TypeError("Object of type 'Batch' has no len()")
        return min(r)

    @property
    def size(self) -> int:
        """Return self.size."""
        if len(self.__dict__.keys()) == 0:
            return 0
        else:
            r = []
            for v in self.__dict__.values():
                if isinstance(v, Batch):
                    r.append(v.size)
                elif hasattr(v, '__len__') and (not isinstance(
                        v, (np.ndarray, torch.Tensor)) or v.ndim > 0):
                    r.append(len(v))
                else:
                    r.append(1)
            return min(r) if len(r) > 0 else 0

    def split(self, size: Optional[int] = None,
              shuffle: bool = True) -> Iterator['Batch']:
        """Split whole data into multiple small batch.

        :param int size: if it is ``None``, it does not split the data batch;
            otherwise it will divide the data batch with the given size.
            Default to ``None``.
        :param bool shuffle: randomly shuffle the entire data batch if it is
            ``True``, otherwise remain in the same. Default to ``True``.
        """
        length = len(self)
        if size is None:
            size = length
        if shuffle:
            indices = np.random.permutation(length)
        else:
            indices = np.arange(length)
        for idx in np.arange(0, length, size):
            yield self[indices[idx:(idx + size)]]

max_stars_repo_name: dvont/tianshou

max_issues_repo_name: dvont/tianshou

max_forks_repo_name: dvont/tianshou


text:
#!/usr/bin/env python
# coding: utf-8

# In[30]:

import json
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import ast

# # Data

# In[31]:

tweets_data_path = 'twitter_data.txt'

tweets_data = []
tweets_file = open(tweets_data_path, "r")
for line in tweets_file:
    try:
        tweet = json.loads(line)
        tweets_data.append(tweet)
    except:
        continue

# In[32]:

len(tweets_data)

# In[33]:

tweets_data

# In[34]:

tweets = pd.DataFrame(tweets_data,columns=['text','lang','extended_tweet','timestamp_ms'])
tweets.head()

# In[35]:

tweets["extended_tweet"] = tweets["extended_tweet"].apply(lambda x : ast.literal_eval(x) if type(x) is str else np.nan)

# In[36]:

tweets["complete_tweets"] = tweets.apply(lambda x: x['text'] if pd.isnull(x['extended_tweet']) else x['extended_tweet']["full_text"], axis=1)

# In[37]:

tweets.head()

# In[38]:

tweets.drop(["text", "lang", "extended_tweet"], 1, inplace=True)

# In[39]:

tweets.head()

# In[40]:

tweets.dropna(inplace=True)

# # Preprocessing

# In[41]:

import nltk
from nltk.corpus import stopwords
from textblob import Word, TextBlob

# In[42]:

nltk.download("stopwords")
nltk.download("wordnet")
stop_words = stopwords.words("english")
custom_stopwords = ["RT"]

# In[43]:

def preprocess_tweets(tweet, custom_stopwords):
    preprocessed_tweet = tweet
    preprocessed_tweet.replace('[^\w\s]', '')
    preprocessed_tweet = " ".join(Word(word).lemmatize() for word in preprocessed_tweet.split() if  word not in custom_stopwords)
    return preprocessed_tweet

# In[44]:

tweets["processed_tweet"] = tweets["complete_tweets"].apply(lambda tweet : preprocess_tweets(tweet, custom_stopwords))

# In[45]:

tweets.head()

# In[46]:

trump_refs = ["DonaldTrump", "Donald Trump", "Donald", "Trump", "trump"]
biden_refs = ["JoeBiden", "Joe Biden", "Joe", "Biden", "biden"]

# In[47]:

def identify_subject(tweet, refs):
    flag = 0
    for ref in refs:
        if tweet.find(ref) != -1:
            flag = 1
    return flag

# In[48]:

tweets["trump"] = tweets["complete_tweets"].apply(lambda tweet : identify_subject(tweet, trump_refs))

# In[49]:

tweets["biden"] = tweets["complete_tweets"].apply(lambda tweet : identify_subject(tweet, biden_refs))

# In[50]:

tweets.dropna(inplace=True)

# In[51]:

tweets.shape

# # Sentiment analysis

# In[52]:

tweets["polarity"] = tweets["processed_tweet"].apply(lambda tweet : TextBlob(tweet).sentiment[0] if tweet != np.nan else np.nan)
tweets["subjectivity"] = tweets["processed_tweet"].apply(lambda tweet : TextBlob(tweet).sentiment[1] if tweet != np.nan else np.nan)

# In[53]:

tweets.head()

# # Visualize

# In[54]:

biden = tweets[tweets["biden"] == 1.0][["timestamp_ms", "polarity"]]
biden = biden.sort_values(by="timestamp_ms", ascending=True)
biden["mean_polarity"] = biden.polarity.rolling(100, min_periods = 3).mean()

# In[55]:

trump = tweets[tweets["trump"] == 1.0][["timestamp_ms", "polarity"]]
trump = trump.sort_values(by="timestamp_ms", ascending=True)
trump["mean_polarity"] = trump.polarity.rolling(100, min_periods = 3).mean()

# In[56]:

fig, axes = plt.subplots(2, 1, figsize=(15, 10))
axes[0].plot(biden["timestamp_ms"][20:], biden["mean_polarity"][20:], "b")
axes[0].set_title("\n".join(["Biden 10 Tweet Moving Average Polarity"]))
axes[1].plot(trump["timestamp_ms"][20:], trump["mean_polarity"][20:], "r")
axes[1].set_title("\n".join(["Trump 10 Tweet Moving Average Polarity"]))
fig.suptitle("\n".join(["Presidential Analysis"]), y=0.98)

# In[ ]:


text:
import numpy as np
import argparse
import os
import csv
import librosa
import uuid

from helpers import str2bool

def generate_mixed_files(audio_files, audio_data, classes, n_files, output_folder, length, max_event, overlap, wn_ratio,
                         sampling_rate):
    r"""Generates mixed audio files from the input audio files. The generated files are saved in the output folder,
        with a '.csv' file keeping track of the classes present in each mix. The sources include in the mix are saved
        in a folder having the same name as the mix.

        Assumption: the length of the mix should be greater than the length of the audio events to put in the mix
    Args:
        audio_files (lst): List of the audio files (eg ["Alert01.wav", ...]
        audio_data (lst): List of numpy arrays storing the audio data for each file
        classes (lst): List of str: the audio event classes present in the audio_files
        n_files (int): Number of mixed files to generate
        output_folder (str): Path to the output folder for the generated files (folder is created if not-existing)
        length (float): Length of the mixed audio files
        max_event (int): Maximal number of events to include in a mixed file
        overlap (bool): Whether or not the audio events can overlap in the remixed files
        wn_ratio (float): Energy ratio between white noise and audio signal
        sampling_rate (int): Sampling rate for the mixed audio files
    """

    header = ["filename"] + classes  # header of the csv file

    total_n_events = len(audio_files)
    mixed_audio_length = int(np.ceil(sampling_rate * length))
    # Write the csv file containing the weak label of the mixed segments
    with open(os.path.join(output_folder, "weak_labels.csv"), 'w', newline='') as label_file:
        label_writer = csv.writer(label_file)
        label_writer.writerow(header)

        # Generate the mixed files
        for _ in range(n_files):
            mixed_audio = np.zeros(mixed_audio_length, dtype=np.float)
            n_events = np.random.randint(low=1, high=max_event + 1)  # number of audio to include in this mixed file
            events_idx = np.random.choice(total_n_events, n_events)  # the audio files to include

            ground_truth_events = np.zeros((len(classes), mixed_audio_length), dtype=np.float)

            if not overlap:
                # The audio files can not overlap in the mix. We have two possibilities: if the audio files to
                # include last more than the required length of the mix, then we include as much of the audio as we
                # can, and simply cut the remaining. If all audio can fit in the mix, then we randomly draw silences
                # duration to separate the audio in the mix
                cum_length = np.sum([audio_data[i].shape[0] for i in events_idx])
                if cum_length > mixed_audio_length:
                    start = 0
                    for idx_idx, idx in enumerate(events_idx):
                        # If audio_data[idx] fits in the mix: copy it entirely
                        if mixed_audio_length - start >= audio_data[idx].shape[0]:
                            mixed_audio[start:start + audio_data[idx].shape[0]] = audio_data[idx]
                            for class_idx, class_name in enumerate(classes):
                                if audio_files[idx].startswith(class_name):
                                    ground_truth_events[class_idx][start:start + audio_data[idx].shape[0]] += \
                                        audio_data[idx]
                            start += audio_data[idx].shape[0]
                        else:  # If not: select a random part of the audio event to copy in the mix.
                            event_start_time = np.random.randint(
                                audio_data[idx].shape[0] - (mixed_audio_length - start))
                            mixed_audio[start:] = audio_data[idx][event_start_time:
                                                                  event_start_time + mixed_audio_length - start]
                            for class_idx, class_name in enumerate(classes):
                                if audio_files[idx].startswith(class_name):
                                    ground_truth_events[class_idx][start:] += audio_data[idx][event_start_time:
                                                                                event_start_time +
                                                                                mixed_audio_length -
                                                                                start]
                            # No more events will fit in this mix: remove the rest of the labels.
                            events_idx = events_idx[:idx_idx + 1]
                            break
                else:
                    # randomly chose a total of silence amount to include between the files, in the available length.
                    total_silence_between_files = np.random.randint(mixed_audio_length - cum_length)
                    silences_length = np.floor(  # Draw silences duration such that they sum up to the total amount.
                        np.random.dirichlet(np.ones(n_events)) * total_silence_between_files).astype(int)
                    start = 0
                    for i, idx in enumerate(events_idx):
                        mixed_audio[start + silences_length[i]: start + silences_length[i] + audio_data[idx].shape[0]] \
                            = audio_data[idx]
                        for class_idx, class_name in enumerate(classes):
                            if audio_files[idx].startswith(class_name):
                                ground_truth_events[class_idx][start + silences_length[i]:
                                                               start + silences_length[i] + audio_data[idx].shape[0]] \
                                    += audio_data[idx]
                        start += silences_length[i] + audio_data[idx].shape[0]
            else:
                # Overlap is authorized, so we simply draw random start times and copy the entire audio files in the mix
                for idx in events_idx:
                    event_length = np.amin([audio_data[idx].shape[0], mixed_audio_length - 1])  # clip to length of mix
                    start_time = np.random.randint(mixed_audio_length - event_length)
                    mixed_audio[start_time: start_time + event_length] += audio_data[idx][:event_length]
                    for class_idx, class_name in enumerate(classes):
                        if audio_files[idx].startswith(class_name):
                            ground_truth_events[class_idx][start_time: start_time + event_length] \
                                += audio_data[idx][:event_length]

            # Add white noise
            noise = np.random.normal(0.0, 1.0, mixed_audio_length)  # scale noise so that the energy ratio is 'wn_ratio'
            mixed_audio += np.sqrt(wn_ratio * np.sum(mixed_audio ** 2) / np.sum(noise ** 2)) * noise
            # Save file
            name = uuid.uuid1().hex
            librosa.output.write_wav(os.path.join(output_folder, name + '.wav'), mixed_audio, sampling_rate, norm=True)
            # write ground truth events
            os.makedirs(os.path.join(output_folder, name))
            for class_idx, class_name in enumerate(classes):
                librosa.output.write_wav(os.path.join(output_folder, name, class_name + '.wav'),
                                         ground_truth_events[class_idx] / np.abs(mixed_audio).max(),  # normalize
                                         sampling_rate, norm=False)
            # save file labels
            labels = [0] * len(classes)
            for idx in events_idx:
                for i, class_name in enumerate(classes):
                    if audio_files[idx].startswith(class_name):
                        labels[i] = 1
            label_writer.writerow([name + '.wav'] + [str(label) for label in labels])

def main():
    r"""DCASE 2013 proposed a competition to detect and classify "office noises". This script implements a procedure
        to create audio mixture from separated audio events sound files.

        The script splits the available audio events in training, testing and validation sets. Then audio mixtures
        are made from the events for each set.

        (Subtask 1 of Event Detection problem at: http://c4dm.eecs.qmul.ac.uk/sceneseventschallenge/description.html)

    """
    parser = argparse.ArgumentParser(allow_abbrev=False,
                                     description="Generate audio files by mixing audio events of the DCASE 2013 sound "
                                                 "event detection dataset (task 2). A csv file is also generated "
                                                 "containing the weak labels for the generated segments")
    parser.add_argument("-l", "--length", type=float, default=10,
                        help="Length (in seconds) of the mixed audio files to generate.")
    parser.add_argument("-M", "--max_event", type=float, default=4,
                        help="Maximal number of audio event per mixed file.")
    parser.add_argument("-o", "--overlap", type=str2bool, default=False,
                        help="Controls if the audio events can overlap in the generated mixed files. Default: False.")
    parser.add_argument("-r", "--white_noise_ratio", type=float, default=0.1,
                        help="Ratio between the energy of the clean mixed audio, "
                             "and the energy of the white noise to apply as background")
    parser.add_argument("-sr", "--sampling_rate", type=int, default=16000,
                        help="The sampling rate to use for the generated mixed files.")
    parser.add_argument("-N", "--n_files", type=int, default=2000,
                        help="Number of mixed file to generate. Default: 2000")
    parser.add_argument("-p_tr", "--training_percentage", type=float, default=0.8,
                        help="The percentage of the files to use to generate the mixes for the training set. "
                             "(validation percentage is deduced from training and development percentages)")
    parser.add_argument("-p_dev", "--development_percentage", type=float, default=0.1,
                        help="The percentage of the files to use to generate the mixes for the development set. "
                             "(validation percentage is deduced from training and development percentages")
    parser.add_argument("-df", "--DCASE_2013_stereo_data_folder", type=str, required=True,
                        help="Path to the folder 'singlesounds_stereo' provided as part of the DCASE 2013 dataset, "
                             "containing the audio files to mix")
    parser.add_argument("-of", "--output_folder", type=str, required=True,
                        help="Path to the folder where the mixed files will be saved.")

    args = vars(parser.parse_args())

    # Check if the output folder exists, if not creates it, otherwise inform user and stop execution
    for set_name in ["training", "development", "validation"]:
        if not os.path.exists(os.path.join(args["output_folder"], set_name)):
            os.makedirs(os.path.join(args["output_folder"], set_name))
        else:
            raise ValueError('Output folders already exist !')

    # The 16 classes present in the DCASE2013 sound event detection data set.
    classes = ["alert", "clearthroat", "cough", "doorslam", "drawer", "keyboard", "keys", "knock", "laughter", "mouse",
               "pageturn", "pendrop", "phone", "printer", "speech", "switch"]

    # train, development and validation split
    tr_audio_files = []
    dev_audio_files = []
    test_audio_files = []

    tr_audio_data = []
    dev_audio_data = []
    test_audio_data = []

    all_audio_files = [f for f in os.listdir(args["DCASE_2013_stereo_data_folder"])
                       if os.path.isfile(os.path.join(args["DCASE_2013_stereo_data_folder"], f))
                       and f.endswith(".wav")]

    for a_class in classes:
        # Load the DCASE2013 dataset audio files of the class 'a_class' from disk
        audio_files = [f for f in all_audio_files if f.startswith(a_class)]
        audio_data = [librosa.core.load(os.path.join(args["DCASE_2013_stereo_data_folder"], f),
                                        sr=args["sampling_rate"])[0]
                      for f in audio_files]

        # Split the files in training, development and validation set
        permutation = np.arange(len(audio_files))
        np.random.shuffle(permutation)  # random permutation to shuffle the data
        n_tr = int(np.floor(args["training_percentage"] * len(audio_files)))
        n_dev = int(np.floor(args["development_percentage"] * len(audio_files)))

        tr_audio_files.extend([audio_files[i] for i in permutation[:n_tr]])
        dev_audio_files.extend([audio_files[i] for i in permutation[n_tr: n_tr + n_dev]])
        test_audio_files.extend([audio_files[i] for i in permutation[n_tr + n_dev:]])

        tr_audio_data.extend([audio_data[i] for i in permutation[:n_tr]])
        dev_audio_data.extend([audio_data[i] for i in permutation[n_tr: n_tr + n_dev]])
        test_audio_data.extend([audio_data[i] for i in permutation[n_tr + n_dev:]])

    generate_mixed_files(tr_audio_files, tr_audio_data, classes,
                         int(np.floor(args["n_files"] * args["training_percentage"])),
                         os.path.join(args["output_folder"], "training"), args["length"],
                         args["max_event"], args["overlap"], args["white_noise_ratio"], args["sampling_rate"])

    generate_mixed_files(dev_audio_files, dev_audio_data, classes,
                         int(np.floor(args["n_files"] * args["development_percentage"])),
                         os.path.join(args["output_folder"], "development"), args["length"],
                         args["max_event"], args["overlap"], args["white_noise_ratio"], args["sampling_rate"])

    generate_mixed_files(test_audio_files, test_audio_data, classes,
                         int(np.ceil(args["n_files"]
                                     * (1 - args["training_percentage"] - args["development_percentage"]))),
                         os.path.join(args["output_folder"], "validation"), args["length"],
                         args["max_event"], args["overlap"], args["white_noise_ratio"], args["sampling_rate"])

if __name__ == '__main__':
    main()


text:
from skimage.io import imread
import matplotlib.pyplot as plt
from skimage.color import rgb2gray
import scipy.ndimage as ndi
from terraingrid import TerrainGrid
import numpy as np
import cv2 as cv
from scipy.ndimage.filters import gaussian_filter
rd0 = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37ez2.tif"
rd1 = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37fz1.tif"
rasdf = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37hn2.tif"
ehamr = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_25dn1.tif"
r2 = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_37fz2.tif"
path = r"C:\Users\siddh\Documents\DSMS\R_25GN1\r_25gn1.tif"
rotterdam = r"D:\Documents\School\2019-20\ISEF 2020\HighProcessed\r_51bz2.tif"

a = TerrainGrid(rd0, (1,1), 1)
a.show(-5, 50)

reason: import numpy,import scipy,from scipy


text:
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pytest
import numpy as np

from nomad.datamodel import EntryArchive
from phonopyparser import PhonopyParser

def approx(value, abs=0, rel=1e-6):
    return pytest.approx(value, abs=abs, rel=rel)

@pytest.fixture(scope='module')
def parser():
    return PhonopyParser()

def test_basic(parser):
    archive = EntryArchive()
    parser.parse('tests/data/Ge/phonopy-FHI-aims-displacement-01/control.in', archive, None)

    # need to assert values, no unbiased reference
    sec_thermo = archive.section_run[0].section_frame_sequence[0].section_thermodynamical_properties[0]
    assert len(sec_thermo.thermodynamical_property_temperature) == 11
    assert len(sec_thermo.thermodynamical_property_heat_capacity_C_v) == 11
    assert len(sec_thermo.vibrational_free_energy_at_constant_volume) == 11

    assert archive.section_run[0].section_method[0].x_phonopy_displacement.magnitude == 1e-12
    sec_scc = archive.section_run[0].section_single_configuration_calculation[0]
    assert np.shape(sec_scc.hessian_matrix) == (8, 8, 3, 3)
    assert np.shape(sec_scc.section_dos[0].dos_values) == (1, 201)
    assert len(sec_scc.section_k_band[0].section_k_band_segment) == 10

def test_vasp(parser):
    archive = EntryArchive()
    parser.parse('tests/data/vasp/phonopy.yaml', archive, None)

max_stars_repo_name: nomad-coe/nomad-parser-phonopy

max_issues_repo_name: nomad-coe/nomad-parser-phonopy

max_forks_repo_name: nomad-coe/nomad-parser-phonopy


text:
import networkx as nx
import matplotlib.pyplot as plt
import graph
import math
import threading, queue
import random
from time import time

class Node :

    def __init__(self ,key):
        self.key = key
        self.child = []

def search_conexions(node,G):
    nodes_connected = []
    for edge in G.edges:
        if edge[0] == node:
            nodes_connected.append(edge[1])
        elif edge[1] == node:
            nodes_connected.append(edge[0])

    return nodes_connected

def printNodeLevelWise(root):
    if root is None:
        return

    queue = []
    queue.append(root)

    while(len(queue) >0):

        n = len(queue)
        while(n > 0) :

            p = queue[0]
            queue.pop(0)
            print (p.key)

            for index, value in enumerate(p.child):
                queue.append(value)

            n -= 1
        print()

def buscar_nodo(camino,raiz):

    root_temp = raiz

    if len(camino) == 1:
        return raiz

    i=1
    while i < len(camino):
        e=0
        while e < len(root_temp.child):
            if camino[i] == root_temp.child[e]:
                root_temp = root_temp.child[e]

            e +=1

        i += 1

    return root_temp

def generate_tree(G,nodo_inicial):

    print("...arbol de estados generando")
    print("demora en minutos por mas cantidad de conexiones")

    path = []
    level = []

    root = Node(nodo_inicial)

    path.append(root)
    level.append(root)

    root_temp = root

    while len(path) != 0:

        if len(level) == 0:
            break

        ### Verfica conexiones de nodo actual ####

        list_conexions = search_conexions(level.pop().key,G)

        ### Verifica que existan conexiones ###

        if len(list_conexions) != 0:

            ### Si hay mas nodos se borran los nodos conectados que estan presentes en el camino y asi evitar bucles ###

            if len(path)-1 != 0:

                ### Evita que cuente el nodo actual y asi evite ciclo del mismo nodo ###

                for i in range(0,len(path)):  ### len(path)-1 antes
                    for nodo in list_conexions:
                        if nodo == path[i].key:
                            list_conexions.remove(nodo)

            ### Si no hay nodos conectados para insertar ###

            if len(list_conexions) == 0:
                """
                for i in range(0,len(path)):
                    print("camino fuera bucle: ",path[i].key)
                print()
                """

                level.append(path.pop())

                ### retorna nodo padre del nodo actual ###

                root_temp = buscar_nodo(path,root)

                subida = False

                while True:
                    """
                    print("root: ",root_temp.key)
                    for i in path:
                        print("camino element: ",i.key)
                    print()"""

                    for i in range(0,len(root_temp.child)):
                        ### Evalua que el nodo padre solo tenga un hijo para buscar padre de ese nodo ###

                        if i+1 == len(root_temp.child):
                            subida = True
                            break

                        ### Evalua que el nodo padre tenga mas hijos para trabajar con nodo hijo siguiente ###

                        elif root_temp.child[i] == level[0]:

                            path.append(root_temp.child[i+1])
                            level.pop()
                            level.append(root_temp.child[i+1])
                            root_temp = root_temp.child[i+1]
                            subida = False
                            break

                    ### Se busca un nuevo nodo padre ###

                    if subida == True:

                        level.pop()
                        level.append(path.pop())

                        if len(path) == 0:
                            subida = False
                            break

                        root_temp = buscar_nodo(path,root)

                    ### Ya se trabaja con el nodo con el flujo normal ###

                    elif subida == False:
                        break

            ### Si hay nodos conectados para insertar ###

            else:

                for nodo in list_conexions:
                    root_temp.child.append(Node(nodo))

                path.append(root_temp.child[0])
                level.append(root_temp.child[0])

                root_temp = root_temp.child[0]

    return root

def busqueda_amplitud_arbol(G,num_nodes,position_node,nodo_inicial,nodo_objetivo):

    start_time = time()

    tree_root = generate_tree(G,nodo_inicial)
    camino_found = 0
    print("...buscando el mejor camino")
    root_tmp = tree_root

    print_queue = []

    queue = []
    camino = []
    queue.append([tree_root.key,camino,tree_root])
    print_queue.append([tree_root.key,camino])

    while True:
        #print("cola caminos: ",print_queue)

        if len(queue) == 0:
            camino_found= False
            break

        elif queue[0][0] == nodo_objetivo:
            queue[0][1].append(nodo_objetivo)
            camino_found = queue[0][1]
            break

        else:

            path = []
            if len(queue[0][1]) == 0:
                path.append(queue[0][0])

            else:
                path = queue[0][1].copy()
                path.append(queue[0][0])

            root_tmp = queue[0][2]

            queue.pop(0)
            print_queue.pop(0)

            for i in range(0,len(root_tmp.child)):
                queue.append([root_tmp.child[i].key,path,root_tmp.child[i]])
                print_queue.append([root_tmp.child[i].key,path])

    elapsed_time = time()- start_time
    print("Elapsed time for BA: %.10f seconds." % elapsed_time)
    print()

    if camino_found == False:
        print("No se encontro camino")
        print()

    else:
        print("Camino encontrado busqueda amplitud: ",camino_found)
        print()

        camino_edges = []
        j=0
        for i in range(0,len(camino_found)):
            j += 1
            edge_tmp = (camino_found[i],camino_found[j])
            camino_edges.append(edge_tmp)

            if j==len(camino_found)-1:
                break

        nx.draw_networkx_nodes(G,position_node)
        nx.draw_networkx_nodes(G,position_node,nodelist = camino_found,node_color="green")
        nx.draw_networkx_edges(G,position_node)
        nx.draw_networkx_edges(G,position_node,edgelist=camino_edges,edge_color="r")

        labels = {}
        for i in range(0,num_nodes):
            labels[i] = i

        nx.draw_networkx_labels(G,position_node,labels,font_size=12)

        plt.show()

def busqueda_amplitud_sin_arbol(G,num_nodes,position_node,nodo_inicial,nodo_objetivo):

    start_time = time()

    tree_root = nodo_inicial
    camino_found = 0
    print("...buscando el mejor camino")
    root_tmp = tree_root

    print_queue = []

    queue = []
    camino = []
    queue.append([tree_root,camino])
    print_queue.append([tree_root,camino])

    while True:
        #print("cola caminos: ",print_queue)

        if len(queue) == 0:
            camino_found= False
            break

        elif queue[0][0] == nodo_objetivo:
            queue[0][1].append(nodo_objetivo)
            camino_found = queue[0][1]
            break

        else:

            path = []
            if len(queue[0][1]) == 0:
                path.append(queue[0][0])

            else:
                path = queue[0][1].copy()
                path.append(queue[0][0])

            root_tmp = queue[0][0]
            list_children = search_conexions(root_tmp,G)

            for j in path:
                for i in list_children:
                    if i == j:
                        list_children.remove(i)

            queue.pop(0)
            print_queue.pop(0)

            for i in range(0,len(list_children)):
                queue.append([list_children[i],path])
                print_queue.append([list_children[i],path])

    elapsed_time = time()- start_time
    print("Elapsed time for BA: %.10f seconds." % elapsed_time)
    print()

    if camino_found == False:
        print("No se encontro camino")
        print()

    else:
        print("Camino encontrado busqueda amplitud: ",camino_found)
        print()

        camino_edges = []
        j=0
        for i in range(0,len(camino_found)):
            j += 1
            edge_tmp = (camino_found[i],camino_found[j])
            camino_edges.append(edge_tmp)

            if j==len(camino_found)-1:
                break

        nx.draw_networkx_nodes(G,position_node)
        nx.draw_networkx_nodes(G,position_node,nodelist = camino_found,node_color="green")
        nx.draw_networkx_edges(G,position_node)
        nx.draw_networkx_edges(G,position_node,edgelist=camino_edges,edge_color="r")

        labels = {}
        for i in range(0,num_nodes):
            labels[i] = i

        nx.draw_networkx_labels(G,position_node,labels,font_size=12)

        plt.show()

def get_key(val,dist_neighbors):
    for key, value in dist_neighbors.items():
         if val == value:
             return key
    return "key doesn't exist"

def hill_climbing(Grafo,n_nodos,nodo_ini,nodo_objetivo):

    start_time = time()

    distancia_nodo_objetivo={}

    for i in range(0,n_nodos):
        if i != n_nodos:
            distancia_nodo_objetivo[i]=(round(math.sqrt(((Grafo.nodes[i]['pos'][1]-Grafo.nodes[nodo_objetivo]['pos'][1])**2)+((Grafo.nodes[nodo_objetivo]['pos'][0]-Grafo.nodes[i]['pos'][0])**2)),2))

    nodo_inicial = nodo_ini
    nodo_final = nodo_objetivo
    nodo_cam = Grafo.nodes[nodo_inicial]
    i = 0
    path = []
    path.append(nodo_inicial)
    while nodo_cam['value'] != nodo_final:
        dist_neighbors = {}
        menor = 0
        key = 0
        hijo = 0
        for edges in Grafo.edges([nodo_cam['value']]):
            hijo = edges[1]
            if (Grafo.nodes[hijo]['path'] != True):
                dist_neighbors[hijo]=distancia_nodo_objetivo[hijo]
                if (hijo == nodo_final or distancia_nodo_objetivo[hijo] == 0 ):
                    menor = distancia_nodo_objetivo[hijo]
                    break
                if menor == 0:
                    menor = distancia_nodo_objetivo[hijo]
                if distancia_nodo_objetivo[hijo] < menor and distancia_nodo_objetivo[hijo] != 0 and nodo_cam['path']!= True:
                    menor = distancia_nodo_objetivo[hijo]
        key = get_key(menor,dist_neighbors)
        if key == "key doesn't exist":
            print ("Camino no encontrado")
            path_find = False
            break
        else:
            path_find = True
        path.append(key)
        nodo_cam['path']=True
        nodo_cam = Grafo.nodes[key]
        Grafo.nodes[nodo_inicial]['path'] = True

    elapsed_time = time()- start_time
    print("Elapsed time for Hill-Climbing: %.10f seconds." % elapsed_time)
    print()

    Grafo.nodes[nodo_final]['path'] = True
    color_map = []
    for i in range (0,n_nodos):
        if Grafo.nodes[i]['path'] == True :
            color_map.append('red')
        else:
            color_map.append('green')

    if path_find:
        print ("Camino encontrado hill-climbing: ",path)
        pos=nx.get_node_attributes(Grafo,'pos')
        nx.draw(Grafo,pos, node_color = color_map,with_labels=True)
        plt.show()

max_stars_repo_name: dennisMSF/IA

max_issues_repo_name: dennisMSF/IA

max_forks_repo_name: dennisMSF/IA

reason: import networkx


text:
from os import walk, getcwd, path
import numpy as np
from customTypes import *
from shutil import copy2

def loadConfig(fname = "config.txt"):
	if not path.exists(fname):
		f = open(fname,"w",encoding='utf-8')
		initConfigFile(f)
		print("No config file detected. Created one. Please fill inn fields.")
		exit()
		return 0

	with open(fname,'r',encoding='utf-8') as f:
		# Get path data from config file
		configs = CONFIG()
		for line in f:
			if line.startswith("Target folder  :"):
				configs.pdfFolder = line[16:-1]
			elif line.startswith("Temp folder    :"):
				configs.temp_folder = line[16:-1]
			elif line.startswith("Frontpage path :"):
				configs.frontpage = line[16:-1]
			elif line.startswith("Output filename:"):
				configs.outfile = line[16:-1]
			elif line.startswith("Field separator:"):
				configs.field_sep = line[16:-1]
			elif line.startswith("Comment between:"):
				configs.comment_sep = line[16:-1]
			elif line.startswith("Delete temp after completion (y/n):"):
				configs.delTemp = line[35:36]
	return configs

def initConfigFile(f):
	f.write("----------------Paths----------------\n")
	f.write("Target folder  :/example docs/\n")
	f.write("Temp folder    :/tex/\n")
	f.write("Frontpage path :/tex/frontpage.tex\n")
	f.write("Output filename:/Notes.pdf\n\n")
	f.write("--------------Settings--------------\n")
	f.write("Field separator:_\n")
	f.write("Comment between:^\n")
	f.write("Delete temp after completion (y/n):n\n\n")
	f.write("-------------Filenames--------------\n")
	f.write("File scraping profiles\n")
	f.write("Lectures     :L_date_number_name_'comment'\n")
	f.write("Problem set  :PS_date_number\n")
	f.write("Exam relevant:ER_date_comment\n")
	f.close()

def filesLoader(filepath = "/target", fieldSeparator = "_", commentSeparator = "'"):
	filepath = getcwd() + filepath
	rawFileList = []

	for (dirpath, dirnames, filenames) in walk(filepath):
		rawFileList.extend(filenames)
		break

	for files in range(len(rawFileList)):
		if not rawFileList[files].endswith(".pdf"):
			del rawFileList[files]

	formattedList = []
	noFiles = -1
	if len(rawFileList):
		for files in range(len(rawFileList)):
			noFiles = noFiles + 1
			newFile = filenameSplitter(rawFileList[files], noFiles, fieldSeparator, commentSeparator)
			if newFile != 0:
				formattedList.append(newFile)
		return formattedList
	else:
		print("No PDF files found in folder " + filepath)
		exit()
		return 0

def filenameSplitter(fnString, fid, fieldSeparator = "_", commentSeparator = "'"):
	fnList = fnString.split(fieldSeparator)
	formatted = PDF()

	# Lecture case:
	try:
		if fnList[0] == "L":
			formatted.fname   = fnString
			formatted.fid 	  = 'f' + str(fid)
			formatted.type    = fnList[0]
			formatted.date    = fnList[1]
			formatted.time	  = fnList[2]
			formatted.num     = fnList[3]
			formatted.name    = fnList[4].split(commentSeparator)[1]
			try:
				formatted.comment = fnList[5].split(commentSeparator)[1]
			except IndexError:
				pass

		# Problem set case:
		elif fnList[0] == "PS":
			formatted.fname = fnString
			formatted.fid   = 'f' + str(fid)
			formatted.type  = fnList[0]
			formatted.date  = fnList[1]
			formatted.time  = fnList[2]
			formatted.num   = fnList[3]
			if formatted.num.endswith(".pdf"):
				formatted.num = formatted.num[:-4]
			try:
				formatted.comment = fnList[4].split(commentSeparator)[1]
			except IndexError:
				pass

		# Exam relevant case:
		elif fnList[0] == "ER":
			formatted.fname   = fnString
			formatted.fid 	  = 'f' + str(fid)
			formatted.type    = fnList[0]
			formatted.date    = fnList[1]
			formatted.time	  = fnList[2]
			formatted.comment = fnList[3].split(commentSeparator)[1]

			# If not implemented file syntax:
		else:
			print("Filename:", fnString, "splitting failed.")
			print("Naming convention described in config.txt not used.")
			formatted = 0

	except IndexError:
		print("Filename scraping of file",fnString,"failed. Please use convention specified in config.txt")

	return formatted

def copyPDFsToTemp(files, source = '/target/', dest = '/tex/'):
	for file in files:
		sourcePath = getcwd() + source + file.fname
		destPath = getcwd() + dest + file.fid + ".pdf"
		copy2(sourcePath,destPath)

max_issues_repo_name: hdieset/Clever-PDF-merger

max_forks_repo_name: hdieset/Clever-PDF-merger


text:
import numpy as np
from summariser.querier.pairwise_uncertainty_querier import PairUncQuerier
from scipy.stats import norm

class ExpectedImprovementQuerier(PairUncQuerier):
    '''
    Compare the current best summary against the summary with the largest expected improvement
    over current best expected summary score.

    If the pair has already been seen, choose the next summary.

    If the best summary has already been compared against all others, do a random exploration
    step and replace the best summary in the pair with a random summary.

    The approach is basically the expected improvement method tested here: Sequential Preference-Based Optimization,
    Ian Dewancker, Jakob Bauer, Michael McCourt, Bayesian Deep Learning workshop at NIPS 2017.

    The approach implemented below is the related to "dueling Thompson sampling" and "copeland expected improvement"
    from González, J., Dai, Z., Damianou, A.,
    & Lawrence, N. D. (2017, July). Preferential Bayesian Optimization. In International Conference on Machine Learning
    (pp. 1282-1291).
    The difference to DTS: it uses expected improvement over the underlying preference function instead of thompson
    sampling from a pairwise difference function.
    Difference to CEI: we consider only the expected improvement over the underlying preference function scores
    as a first selection step, then compare to the best, rather than choosing pairs.

    '''

    def _get_candidates(self):
        f = self.reward_learner.get_rewards()

        # consider only the top ranked items.
        num = 20**2
        candidate_idxs = np.argsort(f)[-num:]

        return candidate_idxs

    def _compute_pairwise_scores(self, f, Cov):

        best_idx = np.argmax(f)
        f_best = f[best_idx]

        sigma = Cov[best_idx, best_idx] + np.diag(Cov) - Cov[best_idx, :] - Cov[:, best_idx]
        sigma[best_idx] = 1 # avoid the invalid value errors -- the value of u should be 0

        # for all candidates, compute u = (mu - f_best) / sigma
        u = (f - f_best) / np.sqrt(sigma) # mean improvement. Similar to preference likelihood, but that adds in 2
        # due to labelling noise
        cdf_u = norm.cdf(u) # probability of improvement
        pdf_u = norm.pdf(u) #

        E_improvement = np.sqrt(sigma) * (u * cdf_u + pdf_u)  # the sqrt was added back in in October 2020 after it was
        # accidentally removed in???
        E_improvement[best_idx] = -np.inf

        # make it back into a matrix
        E_imp_mat = np.zeros((f.size, f.size))
        E_imp_mat[best_idx, :] = E_improvement

        return E_imp_mat

reason: import numpy,from scipy


text:
import numpy as np
import gym
import time

from tensorflow.keras.models import Sequential, Model
from tensorflow.keras import Input
from tensorflow.keras.layers import Dense, Activation, Flatten,LSTM, BatchNormalization
from tensorflow.keras.optimizers import Adam

from rl.agents.dqn import DQNAgent
from rl.policy import BoltzmannQPolicy
from rl.memory import SequentialMemory

from env_centrifuge_continuous import CentrifugeEnv

#----------------------------------------------
# パラメータ
#----------------------------------------------

window_haba = 8

#----------------------------------------------

#----------------------------------------------
# 環境
#----------------------------------------------
env = CentrifugeEnv()
# env = DummyVecEnv([lambda: env])

ENV_NAME = 'Centrifuge-v0'

#----------------------------------------------

# Get the environment and extract the number of actions.
# env = gym.make(ENV_NAME)
# np.random.seed(123)
# env.seed(123)
nb_actions = env.action_space.n
input_tensor =  (window_haba,) + env.observation_space.shape
# Next, we build a very simple model.
model = Sequential()
model.add(LSTM(8, input_shape=input_tensor))
# model.add(LSTM(8, input_shape=(window_haba,) + env.observation_space.shape))
model.add(BatchNormalization())
# model.add(Dense(16))
# model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dense(nb_actions))
model.add(Activation('linear'))

# print(env.observation_space.shape)
# input_dim = (window_haba,) + env.observation_space.shape
# input_dim = (4,5)
# print("input_dim :",input_dim)
# input_tensor = Input(shape = window_haba,env.observation_space.shape)
# input_tensor = Input(input_dim)

# x = LSTM(16)(input_tensor)
# x = Flatten(x)
# x = Dense(16,activation = 'relu')(x)
# x = Dense(16,activation = 'relu')(x)
# out_put = Dense(n_actions,activation = 'linear')

# lstm_model = Model(input_tensor,out_put)

print(model.summary())

# Finally, we configure and compile our agent. You can use every built-in tensorflow.keras optimizer and
# even the metrics!
# memory = SequentialMemory(limit=60000, window_length=1)
memory = SequentialMemory(limit=60000, window_length=window_haba)
policy = BoltzmannQPolicy()
dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=10,
               target_model_update=1e-2, policy=policy)
dqn.compile(Adam(learning_rate=1e-3), metrics=['mae'])

start = time.time()
# Okay, now it's time to learn something! We visualize the training here for show, but this
# slows down training quite a lot. You can always safely abort the training prematurely using
# Ctrl + C.
dqn.fit(env, nb_steps=6000, visualize=False, nb_max_episode_steps=60, verbose=2)

elapsed_time = time.time() - start
# After training is done, we save the final weights.
dqn.save_weights('dqn_{ENV_NAME}_weights.h5f', overwrite=True)

# Finally, evaluate our algorithm for 5 episodes.
dqn.test(env, nb_episodes=5, visualize=True)
print ("elapsed_time:{0}".format(elapsed_time) + "[sec]")

max_stars_repo_name: Delapre/baby-steps-of-rl-ja

max_issues_repo_name: Delapre/baby-steps-of-rl-ja

max_forks_repo_name: Delapre/baby-steps-of-rl-ja