text:
import argparse
import ast
import datetime
import logging
import math
import os
import random
import sys
import time

import numpy as np
import torch
import torch.nn.functional as nnf
import torch.optim as optim
from sklearn.preprocessing import StandardScaler
from tensorboardX import SummaryWriter
from torch.optim.lr_scheduler import MultiStepLR
from tqdm import tqdm

from engine.datasets import create_tough_dataset
from engine.models import create_model

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def get_cli_args():
    parser = argparse.ArgumentParser()

    # Optimization arguments
    parser.add_argument('--wd', default=5.0e-4, type=float, help='Weight decay')
    parser.add_argument('--lr', default=1e-3, type=float, help='Initial learning rate')
    parser.add_argument('--lr_decay', default=0.2, type=float, help='Multiplicative factor used on learning rate at `lr_steps`')
    parser.add_argument('--lr_steps', default='[100,150]', help='List of epochs where the learning rate is decreased by `lr_decay`')
    parser.add_argument('--momentum', default=0.9, type=float, help='Momentum for sgd')
    parser.add_argument('--epochs', default=151, type=int, help='Number of epochs to train. If <=0, only testing will be done.')
    parser.add_argument('--batch_size', default=16, type=int, help='Batch size')
    parser.add_argument('--batch_parts', default=1, type=int, help='Batch can be evaluated sequentially in multiple shards, >=1, very useful in low memory settings, though computation is not strictly equivalent due to batch normalization runnning statistics.')
    parser.add_argument('--optim', default='adam', help='Optimizer: sgd|adam')
    parser.add_argument('--max_train_samples', default=2500, type=int, help='Max train samples per epoch (good for large datasets)')
    parser.add_argument('--max_test_samples', default=100, type=int, help='Max test samples per epoch (good for large datasets)')

    # Experiment arguments
    parser.add_argument('--device', default='cuda:0', type=str)
    parser.add_argument('--nworkers', default=3, type=int, help='Num subprocesses to use for data loading. 0 means that the data will be loaded in the main process')
    parser.add_argument('--test_nth_epoch', default=1, type=int, help='Test each n-th epoch during training')
    parser.add_argument('--resume', default='', help='Loads a previously saved model.')

    # Dataset
    parser.add_argument('--output_dir', default='results', help='Directory to store results')
    parser.add_argument('--cvfold', default=0, type=int, help='Fold left-out for testing in leave-one-out setting')
    parser.add_argument('--num_folds', default=5, type=int, help='Num folds')
    parser.add_argument('--augm_rot', default=1, type=int, help='Training augmentation: Bool, random rotation')
    parser.add_argument('--augm_mirror_prob', default=0, type=float, help='Training augmentation: Probability of mirroring about axes')
    parser.add_argument('--augm_sampling_dist', default=2.0, type=float, help='Training augmentation: Max distance from fpocket centers')
    parser.add_argument('--augm_decoy_prob', default=0.1, type=float, help='Training augmentation: Probability of negative decoy')
    parser.add_argument('--patch_size', default=24, type=int, help='Patch size for training')
    parser.add_argument('--input_normalization', default=1, type=int, help='Bool: whether to normalize input statistics')
    parser.add_argument('--db_exclude_vertex', default='', type=str, help='Whether to exclude Vertex dataset proteins in the training fold: (|seqclust|uniprot|pdb)')
    parser.add_argument('--db_exclude_prospeccts', default='', type=str, help='Whether to exclude Prospeccts dataset proteins in the training fold: (|seqclust|uniprot|pdb)')
    parser.add_argument('--db_split_strategy', default='seqclust', help="pdb_folds|uniprot_folds|seqclust|none")
    parser.add_argument('--db_preprocessing', default=0, type=int, help='Bool: whether to run preprocessing for the dataset')
    parser.add_argument('--db_size_limit', default=0, type=int, help='Artification restriction of database size, either on # pdbs (>0) or # pairs (<0)')

    # Model
    parser.add_argument('--model_config', default='se_16_16_16_16_7_3_2_batch_1,se_32_32_32_32_3_1_1_batch_1,se_48_48_48_48_3_1_2_batch_1,se_64_64_64_64_3_0_1_batch_1,se_256_0_0_0_3_0_2_batch_1,r,b,c_128_1', help='Defines the model as a sequence of layers.')
    parser.add_argument('--seed', default=1, type=int, help='Seed for random initialisation')
    parser.add_argument('--l2_normed_descriptors', default=1, type=int, help='L2-normalize descriptors/network outputs')
    parser.add_argument('--loss_margin', default=1.0, type=float, help='Margin in hinge losses')
    parser.add_argument('--stability_loss_weight', default=1.0, type=float, help='Weight of augmentation invariance loss')
    parser.add_argument('--stability_loss_squared', default=0, type=int, help='Augmentation invariance loss distances squared (1) or not (0)')

    args = parser.parse_args()
    args.start_epoch = 0
    args.lr_steps = ast.literal_eval(args.lr_steps)
    args.output_dir = args.output_dir.replace('TTTT', datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
    assert args.batch_size % args.batch_parts == 0
    return args

def main():
    args = get_cli_args()
    print('Will save to ' + args.output_dir)
    if not os.path.exists(args.output_dir):
        os.makedirs(args.output_dir)
    with open(os.path.join(args.output_dir, 'cmdline.txt'), 'w') as f:
        f.write(" ".join(["'"+a+"'" if (len(a) == 0 or a[0] != '-') else a for a in sys.argv]))

    set_seed(args.seed)
    device = torch.device(args.device)
    writer = SummaryWriter(args.output_dir)

    train_dataset, test_dataset = create_tough_dataset(
        args, fold_nr=args.cvfold, n_folds=args.num_folds, seed=args.seed,
        exclude_Vertex_from_train=args.db_exclude_vertex, exclude_Prospeccts_from_train=args.db_exclude_prospeccts
    )
    logger.info('Train set size: %d, test set size: %d', len(train_dataset), len(test_dataset))

    # Create model and optimizer (or resume pre-existing)
    if args.resume != '':
        if args.resume == 'RESUME':
            args.resume = args.output_dir + '/model.pth.tar'
        model, optimizer, scheduler = resume(args, train_dataset, device)
    else:
        model = create_model(args, train_dataset, device)
        if args.input_normalization:
            model.set_input_scaler(estimate_scaler(args, train_dataset, nsamples=200))
        optimizer = create_optimizer(args, model)
        scheduler = MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_decay)

    ############
    def train():
        model.train()

        loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size//args.batch_parts,
                                             num_workers=args.nworkers, shuffle=True, drop_last=True,
                                             worker_init_fn=set_worker_seed)

        if logging.getLogger().getEffectiveLevel() > logging.DEBUG:
            loader = tqdm(loader, ncols=100)

        loss_buffer, loss_stabil_buffer, pos_dist_buffer, neg_dist_buffer = [], [], [], []
        t0 = time.time()

        for bidx, batch in enumerate(loader):
            if 0 < args.max_train_samples < bidx * args.batch_size//args.batch_parts:
                break
            t_loader = 1000*(time.time()-t0)

            inputs = batch['inputs'].to(device)  # dimensions: batch_size x (4 or 2) x 24 x 24 x 24
            targets = batch['targets'].to(device)

            if bidx % args.batch_parts == 0:
                optimizer.zero_grad()
            t0 = time.time()

            outputs = model(inputs.view(-1, *inputs.shape[2:]))
            outputs = outputs.view(*inputs.shape[:2], -1)
            loss_joint, loss_match, loss_stabil, pos_dist, neg_dist = compute_loss(args, outputs, targets, True)
            loss_joint.backward()

            if bidx % args.batch_parts == args.batch_parts-1:
                if args.batch_parts > 1:
                    for p in model.parameters():
                        p.grad.data.div_(args.batch_parts)
                optimizer.step()

            t_trainer = 1000*(time.time()-t0)
            loss_buffer.append(loss_match.item())
            loss_stabil_buffer.append(loss_stabil.item() if isinstance(loss_stabil, torch.Tensor) else loss_stabil)
            pos_dist_buffer.extend(pos_dist.cpu().numpy().tolist())
            neg_dist_buffer.extend(neg_dist.cpu().numpy().tolist())
            logger.debug('Batch loss %f, Loader time %f ms, Trainer time %f ms.', loss_buffer[-1], t_loader, t_trainer)
            t0 = time.time()

        ret = {'loss': np.mean(loss_buffer), 'loss_stabil': np.mean(loss_stabil_buffer),
               'pos_dist': np.mean(pos_dist_buffer), 'neg_dist': np.mean(neg_dist_buffer)}
        return ret

    ############
    def test():
        model.eval()

        loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batch_size//args.batch_parts,
                                             num_workers=args.nworkers, worker_init_fn=set_worker_seed)

        if logging.getLogger().getEffectiveLevel() > logging.DEBUG:
            loader = tqdm(loader, ncols=100)

        loss_buffer, loss_stabil_buffer, pos_dist_buffer, neg_dist_buffer = [], [], [], []

        with torch.no_grad():
            for bidx, batch in enumerate(loader):
                if 0 < args.max_test_samples < bidx * args.batch_size//args.batch_parts:
                    break
                inputs = batch['inputs'].to(device)
                targets = batch['targets'].to(device)

                outputs = model(inputs.view(-1, *inputs.shape[2:]))
                outputs = outputs.view(*inputs.shape[:2], -1)
                loss_joint, loss_match, loss_stabil, pos_dist, neg_dist = compute_loss(args, outputs, targets, False)

                loss_buffer.append(loss_match.item())
                loss_stabil_buffer.append(loss_stabil.item() if isinstance(loss_stabil, torch.Tensor) else loss_stabil)
                pos_dist_buffer.extend(pos_dist.cpu().numpy().tolist())
                neg_dist_buffer.extend(neg_dist.cpu().numpy().tolist())

        return {'loss': np.mean(loss_buffer), 'loss_stabil': np.mean(loss_stabil_buffer),
                'pos_dist': np.mean(pos_dist_buffer), 'neg_dist': np.mean(neg_dist_buffer)}

    ############
    # Training loop
    for epoch in range(args.start_epoch, args.epochs):
        print(f'Epoch {epoch}/{args.epochs} ({args.output_dir}):')
        scheduler.step()

        train_stats = train()
        for k, v in train_stats.items():
            writer.add_scalar('train/' + k, v, epoch)
        print(f"-> Train distances: p {train_stats['pos_dist']}, n {train_stats['neg_dist']}, \tLoss: {train_stats['loss']}")

        if (epoch+1) % args.test_nth_epoch == 0 or epoch+1 == args.epochs:
            test_stats = test()
            for k, v in test_stats.items():
                writer.add_scalar('test/' + k, v, epoch)
            print(f"-> Test distances: p {test_stats['pos_dist']}, n {test_stats['neg_dist']}, \tLoss: {test_stats['loss']}")

        torch.save({'epoch': epoch + 1, 'args': args, 'state_dict': model.state_dict(),
                    'optimizer': optimizer.state_dict(), 'scheduler': scheduler.state_dict()},
                   os.path.join(args.output_dir, 'model.pth.tar'))

        if math.isnan(train_stats['loss']):
            break

def compute_loss(args, outputs, targets, training):
    """
    Computes both stability and contrastive loss
    """
    outputs = torch.squeeze(outputs)
    targets = torch.squeeze(targets)
    assert outputs.dim() == 3 and targets.dim() == 1

    if args.l2_normed_descriptors:
        outputs = nnf.normalize(outputs, p=2, dim=2)

    # Stability loss
    if training and args.stability_loss_weight > 0:
        # every odd entry in the batch is a perturbed version of the previous even entry
        a = outputs[:, 0::2].view(-1, outputs.shape[-1])
        b = outputs[:, 1::2].view(-1, outputs.shape[-1])
        if args.stability_loss_squared:
            loss_stabil = nnf.pairwise_distance(a, b).pow(2).mean()
        else:
            loss_stabil = nnf.pairwise_distance(a, b).mean()
        # continue with just the even ones
        outputs = outputs[:, 0::2]
    else:
        loss_stabil = 0

    # Contrastive loss
    assert outputs.shape[1] == 2
    dists = nnf.pairwise_distance(outputs[:, 0], outputs[:, 1]).view(-1)

    pos_loss = dists.pow(2)
    neg_loss = torch.clamp(args.loss_margin - dists, min=0).pow(2)
    loss_match = torch.sum(pos_loss * targets + neg_loss * (1 - targets)) / targets.numel()

    loss_joint = loss_match + args.stability_loss_weight * loss_stabil
    return loss_joint, loss_match, loss_stabil, dists[targets > 0.5].detach(), dists[targets < 0.5].detach()

def resume(args, dataset, device):
    """
    Loads model and optimizer state from a previous checkpoint
    """
    print(f"=> loading checkpoint '{args.resume}'")
    checkpoint = torch.load(args.resume, map_location=str(device))

    model = create_model(checkpoint['args'], dataset, device)
    model.load_state_dict(checkpoint['state_dict'])

    optimizer = create_optimizer(args, model)
    optimizer.load_state_dict(checkpoint['optimizer'])
    args.start_epoch = checkpoint['epoch']

    scheduler = MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_decay)
    scheduler.load_state_dict(checkpoint['scheduler'])

    return model, optimizer, scheduler

def create_optimizer(args, model):
    if args.optim == 'sgd':
        return optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.wd)
    elif args.optim == 'adam':
        return optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd)

def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)

def set_worker_seed(worker_id):
    np.random.seed(torch.initial_seed() % (2**32 - 1))

def estimate_scaler(args, train_dataset, nsamples):
    logger.info('Estimating dataset normalization')
    scaler = StandardScaler(copy=True, with_mean=True, with_std=True)
    bidx = 0
    with tqdm(total=nsamples) as pbar:
        while True:
            loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.nworkers,
                                                 shuffle=True, drop_last=True, worker_init_fn=set_worker_seed)
            for batch in loader:
                inputs = batch['inputs'].view(-1, *batch['inputs'].shape[2:])
                assert inputs.dim() == 5
                voxels = inputs.transpose(1, 4).contiguous().view(-1, inputs.shape[1]).numpy()
                scaler.partial_fit(voxels)

                bidx += inputs.shape[0]
                pbar.update(inputs.shape[0])
                if bidx >= nsamples:
                    return scaler

if __name__ == "__main__":
    main()

max_stars_repo_name: truatpasteurdotfr/DeeplyTough

max_stars_repo_licenses: FTL Xnet Net-SNMP

max_issues_repo_name: truatpasteurdotfr/DeeplyTough

max_issues_repo_licenses: FTL Xnet Net-SNMP

max_forks_repo_name: truatpasteurdotfr/DeeplyTough

max_forks_repo_licenses: FTL Xnet Net-SNMP


text:
'''VGG for CIFAR10. FC layers are removed.
(c) YANG, Wei
'''
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.model_zoo as model_zoo
import math
import numpy as np
from torch.autograd import Variable

class _gcn(nn.Module):

    def __init__(self, num_classes,
                t_kernel_size=1,
                t_stride=1,
                t_padding=0,
                t_dilation=1):

        super(_gcn, self).__init__()
        self.conv1da = nn.Conv1d(in_channels=4096, out_channels=2048*2, kernel_size=1, \
        padding=0, stride=1, dilation=1, bias=False)
        self.conv1db = nn.Conv1d(in_channels=2048, out_channels=1024*2, kernel_size=1, \
        padding=0, stride=1, dilation=1, bias=False)
        self.gclassifier = nn.Linear(2048, num_classes)
        self.pool = nn.AdaptiveAvgPool1d(1)
        self.lstm = nn.LSTM(1024*3, 2048, 1)
        #self.conv1dc = nn.Conv1d(in_channels=256, out_channels=128*2, kernel_size=1, \
        #padding=0, stride=1, dilation=1, bias=False)

        #self.convLinear = nn.Conv1d(in_channels=512, out_channels=num_classes, kernel_size=1, \
        #padding=0, stride=1, dilation=1, bias=False)
        self.gclassifier = nn.Linear(2048, num_classes)
        self.pool = nn.AdaptiveAvgPool1d(1)
        #self.avg_pool = nn.AdaptiveAvgPool1d(1)
        #self.linear = nn.Linear(256, num_classes)
        self.lstm = nn.LSTM(1024*3, 2048, 1)
        #self.lstm = nn.LSTM(128, 128, 1)

    def forward(self, x, dist):
        [N, T, M, C, H, W] = x.shape
        with torch.no_grad():
            base_out = self.base_model(x.view(-1, C, H, W)).view(N*T, M, -1)

        distb = dist[:,:,1,:,:,:]
        distc = dist[:,:,0,:,:,:]

        node1 = self.conv1da(base_out.permute(0,2,1))
        node1 = node1.view(N*T, 2, 2048, 12)
        node1b = torch.einsum('nkcv,nkvw->ncw', (node1, distb.view(N*T, 2, 12, 12).float()))
        node1c = torch.einsum('nkcv,nkvw->ncw', (node1, distc.view(N*T, 2, 12, 12).float()))
        node1 = node1b+node1c
        node1 = F.relu(node1)

        node2 = self.conv1db(node1)
        node2 = node2.view(N*T, 2, 1024, 12)
        node2b = torch.einsum('nkcv,nkvw->ncw', (node2, distb.view(N*T, 2, 12, 12).float()))
        node2c = torch.einsum('nkcv,nkvw->ncw', (node2, distc.view(N*T, 2, 12, 12).float()))
        node2b = F.relu(node2b)
        node2c = F.relu(node2c)

        pooled_feat_left = self.pool(node2b[:,:,:6]).squeeze(2)
        pooled_feat_right = self.pool(node2b[:,:,6:]).squeeze(2)
        pooled_feat_all = self.pool(node2c).squeeze(2)
        pooled_feat = torch.cat((pooled_feat_left, pooled_feat_right, pooled_feat_all),dim=1)

        video_feat, _ = self.lstm(pooled_feat.view(N, T, -1))
        group_cls = self.gclassifier(video_feat[:,-1,:])

        return group_cls, video_feat

    def _initialize_weights(self):

        for m in self.modules():
            if isinstance(m, nn.Linear):
                n = m.weight.size(1)
                m.weight.data.normal_(0, 0.01)
                m.bias.data.zero_()
            elif isinstance(m, nn.Conv1d):
                m.weight.data.normal_(0.0, 0.02)
                if m.bias is not None:
                    m.bias.data.fill_(0)

    def create_architecture(self):

        self._init_modules()
        self._initialize_weights()

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# encoding: utf-8
'''
@author: zst
@file: Locatation_curve_prediction.py
@date 2019-09-18 15:59
@desc:
'''

import numpy as np
from sklearn.preprocessing import PolynomialFeatures
import joblib

def prediction(LactationNumber, database):
    # 305预测

    X = np.arange(1, 306)[:, np.newaxis]
    # 保加载模型
    model_name = 'Lactation_curve_' + str(LactationNumber) + '_' + database + '.model'
    regr = joblib.load(model_name)

    # 预测值
    ploy = PolynomialFeatures(degree=6)
    X_6 = ploy.fit_transform(X)
    y_six_pre = regr.predict(X_6)
    return y_six_pre

def prediction_yield(LactationNumber, database):
    """

    :param LactationNumber: int 胎次
    :param database: str 数据库
    :return:
            y_six_pre: list 305天单产预测
    """
    y_six_pre = prediction(LactationNumber, database)
    pre_yield = y_six_pre.tolist()
    return pre_yield

def locatation_alpha(t, LactationNumber, database):
    """
    泌乳天数的校正系数
    :param t: int 泌乳天数
    :param LactationNumber: int 胎次
    :param database: str 数据库
    :return: alpha：int 泌乳系数
            dict_every：dict 泌乳天数之后的预测值
    """

    y_six_pre = prediction(LactationNumber, database)
    dict_every = {}
    for key, value in zip(map(str, range(t+1, 306)), y_six_pre[t:]):
        dict_every[str(key)] = value
    # print(dict_every)

    # 泌乳天数的校正系数
    f_305= np.sum(y_six_pre)
    f_t = np.sum(y_six_pre[0:t])
    alpha = f_305 / f_t

    return alpha, dict_every

if __name__ == "__main__":
    # 数据库
    database = "x6_H1170"
    # 胎次
    LactationNumber = 1
    # 泌乳天数
    t = 200
    # 校正系数，泌乳天数之后的预测值
    alpha, pre = locatation_alpha(t, LactationNumber, database)
    print(alpha)
    print(pre)
    pre_yield = prediction_yield(LactationNumber, database)
    print(pre_yield)


text:
import tensorflow as tf
import numpy as np
from styx_msgs.msg import TrafficLight

class TLClassifier(object):
    def __init__(self):
        #DONE load classifier
        tf.reset_default_graph()
        graph_file = 'frozen_model.pb'
        jit_level = 0
        config = tf.ConfigProto()
        use_xla = False
        if use_xla:
            jit_level = tf.OptimizerOptions.ON_1
            config.graph_options.optimizer_options.global_jit_level = jit_level

        sess = tf.Session(config=config)
        gd = tf.GraphDef()
        #---------------------remove---------------------------
        with open('classifier_saved_here.txt','w') as f:
            f.write("ok")
        #---------------------remove---------------------------
        with tf.gfile.Open(graph_file, 'rb') as f:
            data = f.read()
            gd.ParseFromString(data)
        tf.import_graph_def(gd, name='')
        self.image_input = sess.graph.get_tensor_by_name("raw_rgb_image:0")
        self.is_training = sess.graph.get_tensor_by_name("is_training:0")
        self.pred = sess.graph.get_tensor_by_name("logits:0")
        self.tf_sess = sess
        self.class_to_tl = {
                            0: TrafficLight.RED,
                            1: TrafficLight.YELLOW,
                            2: TrafficLight.GREEN,
                            3: TrafficLight.UNKNOWN,
                            }

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        #TODO implement light color prediction
        im_4d = np.expand_dims(image, axis=0)
        predict = self.tf_sess.run(self.pred, feed_dict = {self.image_input: im_4d,
                                                           self.is_training:False})

        predicted_class = np.argmax(predict[0])
        return self.class_to_tl[predicted_class]

max_stars_repo_name: IIIDimaIII/CarND-Capstone

max_issues_repo_name: IIIDimaIII/CarND-Capstone

max_forks_repo_name: IIIDimaIII/CarND-Capstone


text:
import pickle
import numpy as np
import tensorflow as tf

# Reads pickled lists containing cleaned data (stored as lists of tokens)
def read_data():
    with open('./input/data_pickles/train_list_sentence1.txt', "rb") as file:
        train_list_sentence1 = pickle.load(file)

    with open('./input/data_pickles/train_list_sentence2.txt', "rb") as file:
        train_list_sentence2 = pickle.load(file)

    with open('./input/data_pickles/train_list_gold_label.txt', "rb") as file:
        train_list_gold_label = pickle.load(file)

    with open('./input/data_pickles/test_list_sentence1.txt', "rb") as file:
        test_list_sentence1 = pickle.load(file)

    with open('./input/data_pickles/test_list_sentence2.txt', "rb") as file:
        test_list_sentence2 = pickle.load(file)

    with open('./input/data_pickles/test_list_gold_label.txt', "rb") as file:
        test_list_gold_label = pickle.load(file)

    data = [[train_list_sentence1], [train_list_sentence2], [train_list_gold_label],
            [test_list_sentence1], [test_list_sentence2], [test_list_gold_label]]

    return data

# Converts data to required format
def convert_data(data):
    # Get the sentences and labels from composite data
    list_sentence1 = data[0][0]
    list_sentence2 = data[1][0]
    list_gold_label = data[2][0]

    # Merge each sublist (tokens list of each sentence) to a string
    corpus_sentence1 = [' '.join(item) for item in list_sentence1]
    corpus_sentence2 = [' '.join(item) for item in list_sentence2]
    num_samples = len(list_gold_label)

    # Create a composite corpus over which to train the Keras tokenizer
    # Corresponding lines of sentence1 and sentence2 are merged together
    corpus = [corpus_sentence1[ind] + " " + corpus_sentence2[ind] for ind in range(num_samples)]

    # There are entries in dataset without any gold_label (with gold_label entry as "-")
    # I choose to delete those from my data as they do not provide any information
    del_list = []
    labels = [None] * num_samples
    for ind, item in enumerate(list_gold_label):
        if item == "contradiction":
            labels[ind] = 0
        elif item == "neutral":
            labels[ind] = 1
        elif item == "entailment":
            labels[ind] = 2
        else:
            labels[ind] = 99
            del_list.append(ind)

    # Delete entries with gold_label "-"
    del_list.sort(reverse=True)
    for ind in del_list:
        del corpus[ind]
        del corpus_sentence1[ind]
        del corpus_sentence2[ind]
        del labels[ind]

    labels = np.array(labels)
    data_converted = [corpus_sentence1, corpus_sentence2, labels]

    # """
    print(f"Non labelled: {len(del_list)}")
    print(f"Contradiction: {np.sum(labels == 0)}")
    print(f"Neutral: {np.sum(labels == 1)}")
    print(f"Entailment: {np.sum(labels == 2)}")
    # """

    return data_converted, corpus

def glove_dict(EMBEDDING_DIM):
    embedding_dict = {}

    # Open the GloVe embedding file
    glove_dir = './input/embeddings/glove.840B.'+ str(EMBEDDING_DIM)+'d.txt'
    file = open(glove_dir, encoding="utf8")

    for line in file:
        # Spilt the word and its embedding vector
        line_list = line.split(' ')
        word = line_list[0]
        embeddings = np.asarray(line_list[1:], dtype='float32')

        # Store the word and its embedding vector in a dictionary
        embedding_dict[word] = embeddings

    file.close()

    # Store the dictionary as a pickle file to reduce thw overhead of loading
    with open(f'./input/embeddings/glove_dict_840_{EMBEDDING_DIM}d.pickle', "wb") as file:
        pickle.dump(embedding_dict, file)

    return embedding_dict

def embedding_matrix(corpus, EMBEDDING_DIM, VOCAB_SIZE):
    # Try to load GloVe embedding dictionary if it exists. If not, create one
    try:
        with open(f'./input/embeddings/glove_dict_840_{EMBEDDING_DIM}d.pickle', "rb") as file:
            glove_embedding = pickle.load(file)
    except:
        glove_embedding = glove_dict(EMBEDDING_DIM)

    # Initialize and fit Keras tokenizer to convert words to integers
    tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=VOCAB_SIZE)
    tokenizer.fit_on_texts(corpus)

    # Save the tokenizer as a pickle file so that the same tokenizer (word-integer)
    # mapping can be used during testing time
    with open('./model/tokenizer.pickle', "wb") as file:
        pickle.dump(tokenizer, file)

    # Get an word-integer dictionary and use that to create an weight matrix
    # i-th column of weight matrix will have the vector of word with integer value i in dictionary
    word_index = tokenizer.word_index
    embed_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))

    for word, ind in word_index.items():
        # Get the embedding vector from GloVe dictionary, if available
        # Words not in the Glove would have the embedding matrix vector as random values uniformly generated between -0.05 and 0.05
        embedding_vector = glove_embedding.get(word)

        if embedding_vector is not None:
            embed_matrix[ind] = embedding_vector
        else:
            embed_matrix[ind] = np.array([np.random.uniform(-0.05, 0.05) for _ in range(EMBEDDING_DIM)])

    return embed_matrix, tokenizer

def preprocess_traindata(train_data, MAX_SEQ_LEN, EMBEDDING_DIM, VOCAB_SIZE):
    # print("Train Data information\n")

    # Convert data to required format
    data, corpus = convert_data(train_data)

    # Obtain the embedding weight matrix and the tokenizer
    embed_matrix, tokenizer = embedding_matrix(corpus, EMBEDDING_DIM, VOCAB_SIZE)

    # Process the data to integer sequences and labels to one-hot labels
    sequence = lambda sentence: tf.keras.preprocessing.sequence.pad_sequences(tokenizer.texts_to_sequences(sentence), maxlen=MAX_SEQ_LEN)
    process = lambda item: (sequence(item[0]), sequence(item[1]), tf.keras.utils.to_categorical(item[2]))

    training_data = process(data)

    return training_data, embed_matrix

# Function to preprocess test data
def preprocess_testdata(test_data, MAX_SEQ_LEN):
    # print("Test Data information\n")

    # Convert data to required format
    data, _ = convert_data(test_data)

    # Load the tokenizer from pickle file
    with open('./model/tokenizer.pickle', "rb") as file:
        tokenizer = pickle.load(file)

    # Process the data to integer sequences and labels to one-hot labels
    sequence = lambda sentence: tf.keras.preprocessing.sequence.pad_sequences(tokenizer.texts_to_sequences(sentence), maxlen=MAX_SEQ_LEN)
    process = lambda item: (sequence(item[0]), sequence(item[1]), tf.keras.utils.to_categorical(item[2]))

    test_data = process(data)

    return test_data


text:
# Mandelbrot Set by KuttaBoi

import numpy as np
import cv2

frameWidth = 700
frameHeight = 500

maxIterations = 100

img = np.zeros([frameHeight, frameWidth])

def mandelbrot(zx, zy):

    z = np.complex(0, 0)
    zn = np.complex(zx, zy)

    for b in range(maxIterations):
        z = np.square(z) + zn
        if np.abs(z) > 2:
            return b/maxIterations
    return 0

# Calculate Mandelbrot Set
origX = frameWidth/2

origY = frameHeight/2

scaleFactor = 200
offset = (-0.5, 0)

for i in range(frameWidth-1):
    for j in range(frameHeight-1):

        x = (i - origX)/scaleFactor + offset[0]
        y = (j - origY)/scaleFactor + offset[1]

        img[j, i] = mandelbrot(x, y)

while True:
    cv2.imshow("Mandelbrot", img)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

max_stars_repo_name: KuttaBoi/Mandelbrot

max_issues_repo_name: KuttaBoi/Mandelbrot

max_forks_repo_name: KuttaBoi/Mandelbrot


text:
"""
Data Pipeline Functions
"""

from pipeline_helper import __time_this, __make_folders, vbPrint, __getWindow, __listdir
from pipeline_config import *
from s3_helper import *
from split_merge_raster import Open_Raster,Split_Raster,Save_Tile,Save_Patches            #These are all called w/in genImgPatches
import numpy as np
from pycocotools import mask
from raster2coco import Raster2Coco
import json

@__time_this
def loadFiles():
    """
    * This function loads all required files from S3.
    * The files required at this stage of the pipeline would be the Tiles, Labels, and their World Files
    * This code may have to change depending on how the Input data is available.

    [Code by John]
    Note:
        loads region data from s3 into the directory:
            * ./data/<dataset>/region_<fileset>/

    Inputs:
            ds (str): local dataset folder (root folder)
            fs (str): local fileset folder (sub folder)
            s3td (str): bucket path to files
            fmts (str): list of file formats to search for in s3Dir, gets converted to list. Ex: "jpg, png" -> ['jpg','png'], "all" -> ["all"] (all returns all files)
    """
    ##----------------------------Configuration/Setup---------------------------##
    ds = config['ds']                                                           #dataset string. root folder of files
    fs = config['fs']                                                           #fileset string. folder to hold world files
    s3Dir = config['s3d']                                                       #AWS URI for folder located in S3 bucket

    file_formats = config['fmts'].lower().replace(' ','').split(',')            #convert string of file formats to list of formats
    loadDir = '%s/region_%s'%(ds,fs)                                             #root folder + subfolder

    vbPrint('Load File(s) Starting...')

    __make_folders(rootfolder=ds,                                               ## Making the dirs
                   subfolder=loadDir)                                           #method for making project rootfolder/subfolders

    ##----------------------------Load Files------------------------------------##

                                                                                ##Retrieve aws bucket object
    load_s3_to_local(s3_uri=s3Dir,                                              #S3 Bucket access. Uses boto3 + local aws credentials to access bucket
                     desired_formats=file_formats,
                     load_dir=loadDir,
                     verbose=config['verbose'])

    vbPrint('Loading Region File(s) Completed')

    """
    #Examples:
    python3 dataPipeline.py --ds=dataset7 --fs=inputs --fmts=all --s3d=s3://cv.datasets.aegean.ai/njtpa/njtpa-year-2/DOM2015/Selected_500/pipelineTest/ -loadFiles

    python3 dataPipeline.py --ds=dataset7 --fs=labels --fmts=all --s3d=s3://cv.datasets.aegean.ai/njtpa/njtpa-year-2/labels_ground_truth/year-2/output/ -loadFiles

    python3 dataPipeline.py --ds=dataset7 --fs=vectors --fmts=geojson --s3d=s3://cv.datasets.aegean.ai/njtpa/njtpa-year-2/labels_ground_truth/year-2/vector_files/ -loadFiles
    """

@__time_this
def genImgPatches():
    """
    [Code by John]
    Notes:
        Generates image patches by splitting the tiles
        Prerequisite: loadFiles() has been run or tiles are present in the data/<dataset>/tiles directory

    Inputs:
        ds (str): dataset folder. aka root folder for project
        fs (str): fileset folder. aka subfolder to hold patches
        mWH (int): maximum Width/Height of tiles
        fmts (str): what raster file types should be converted to patches
                        (this is important since image files are associated w/ other file type like an .xml,.cpg,.dbf this makes sure those are ignored)
        save_fmt (str): file format for patches. Exs: jpeg, png

    outputs:
        folder containing patch image-files and any support files.
    """

    ##----------------------------Configuration/Setup---------------------------##
    ds = config['ds']                                                           #dataset (root folder)
    fs = config['fs']                                                           #existing fileset (subfolder)
    ts = config['ts']                                                           #optional tileset (subfolder). Default is None, which means no tiles will be saved.

    window = __getWindow(window_config=config['mWH'])                           #dictonary w/ max window dimensions (default: width:5000,height:5000)

    file_formats = config['fmts'].replace(' ','').split(',')                    #convert string of file formats to list of formats
    save_fmt = config['sfmt']                                                   #file format for patches when they are saved. (png,jpeg,GTiff) https://gdal.org/drivers/raster/index.html

    worldDir = '%s/region_%s'%(ds,fs)                                           #directory with files to convert to patches
    worldFiles = __listdir(directory=worldDir,
                           extensions=file_formats)                             #list of files to convert to patches

    if ts is not None:                                                          #subfolder for saving tiles
        tilesDir = '%s/imageTiles_%s'%(ds,ts)
        __make_folders(rootfolder=ds,
                       subfolder=tilesDir)

    patchesDir = '%s/imagePatches_%s'%(ds,fs)                                   #subfolder for saving patches
    vbPrint('%i Region files found in %s'%(len(worldFiles),worldDir))           #display the number of files to be processed

    __make_folders(rootfolder=ds,                                               ##Make folder to hold patches
                   subfolder=patchesDir)                                        #method for making project rootfolder/subfolders

    ##----------------------------Create Patches------------------------------------##
    for count,imageName in enumerate(worldFiles):
        imgage_Path = fr'{worldDir}/{imageName}'                                #recreate image path
        vbPrint(f'Current Image Path: {imgage_Path}')

        #Open world file & apply window to generate Tiles
        for tile_array,tile_meta,tile_name in Open_Raster(path_to_region = imgage_Path,      #current world file
                                                          maxWidth = window['width'],       #this is maxWidth/maxHeight
                                                          maxHeight = window['height'],        #actual height/width may be less since no padding happens when opening
                                                          verbose = config['verbose']):

            if 0 in tile_array.shape:
                vbPrint(f'empty array generated: {tile_array.shape}. Skipping')
                continue

                                                                                ## Creation of Image patches
            split_array = Split_Raster(image=tile_array,                        #current tile to split into patches
                                       patchX = 256,                            #patch width
                                       patchY = 256,                            #patch height
                                       minScrapPercent = 0,                     #0 means every tile_array will be padded before splitting (otherwise patches along bot/right edges are discarded)
                                       verbose = config['verbose'])

            if ts is not None:
                Save_Tile(array=tile_array,                                     #tile array
                          save_directory=tilesDir,                              #folder location to save tile in
                          tile_name=tile_name,                                  #patches will be named as: region_tile_{tileXpos}_{tileYpos}
                          profile = tile_meta,                                  #rasterio requires a 'profile'(meta data) to create/write files
                          save_fmt = save_fmt,                                  #format to save tiles as (ex: png,jpeg,geoTiff)
                          verbose = config['verbose'])

                                                                                ##Save Image Patches:
            Save_Patches(array=split_array,                                     #split tile containing patches
                         save_directory=patchesDir,                             #folder location to save patches in
                         tile_name=tile_name,                                   #patches will be named as: tileName_tileXpos_tileYpos_patchXpos_patchYpos
                         profile = tile_meta,                                   #rasterio requires a 'profile'(meta data) to create/write files
                         save_fmt = save_fmt,                                   #format to save patches as (ex: png,jpeg,GTiff)
                         verbose = config['verbose'])

    ##----------------------------Quick Summary------------------------------------##
    if ts is not None:
        tileFiles = __listdir(directory=tilesDir,
                              extensions=['all'])
        vbPrint(f"Number of files created in {tilesDir}: {len(tileFiles)}\n{'-'*4}Image Tiles made successfully{'-'*4}")

    patchFiles = __listdir(directory=patchesDir,
                           extensions=['all'])

    vbPrint(f"Number of files created in {patchesDir}: {len(patchFiles)}\n{'-'*4}Image Patches made successfully{'-'*4}")

    """
    Example:
        * python3 dataPipeline.py --ds=dataset5 --fs=labels --fmts=tif --sfmt=png --mWH=5000,5000 -genImgPatches
        * python3 dataPipeline.py --ds=dataset5 --fs=inputs --fmts=jpg --sfmt=png --mWH=5000,5000 -genImgPatches
        * python3 dataPipeline.py --ds=dataset7 --fs=labels --ts=labels --fmts=tif --sfmt=gtiff --mWH=5000,5000 -genImgPatches
        * python3 dataPipeline.py --ds=dataset7 --fs=labels --ts=labels --fmts=tif --sfmt=tiff --mWH=5000,5000 -genImgPatches
        * python3 dataPipeline.py --ds=dataset8 --fs=labels --ts=labels --fmts=gtiff --sfmt=gtiff --mWH=5000,5000 -genImgPatches
        * python3 dataPipeline.py --ds=dataset9 --fs=labels --ts=labels9 --fmts=gtiff --sfmt=gtiff --mWH=5000,5000 -genImgPatches
    """

def genAnnotations():
    """
    Generates the train and test annotations files using the image and label patches

    Requirements:
        * label patches should be available in <ds>/labelPatches_<ts>/..
        * While not required by this function, corresponding image patches should be available in <ds>/imagePatches_<ts>/..

    Generates:
        * Two annotation files to be used by the model to train:
            1. <ds>/annotations_<ts>/annotations_train.json
            2. <ds>/annotations_<ts>/annotations_test.json

    """
    ds = config['ds']
    ts = config['ts']
    tvRatio = config['tvRatio']

    trainAnnFilename = 'annotations_train.json'
    testAnnFilename = 'annotations_test.json'

    labelPatchesDir = '%s/labelPatches_%s'%(ds,ts)
    annDir = '%s/annotations_%s'%(ds,ts)

    ## Making the dirs
    if(os.path.isdir(annDir)):
        vbPrint('Found dir: %s'%(annDir))
    else:
        vbPrint('Making dir: %s'%(annDir))
        os.mkdir(annDir)

    ## Getting the labels and splitting into training and validation images
    vbPrint('Reading `%s` for label patches'%(labelPatchesDir))
    labels = __listdir(labelPatchesDir, ['tif'])
    vbPrint('Dataset Size   : %i'%(len(labels)))
    labels = np.asarray(labels)
    np.random.shuffle(labels)
    splitIdx = int(labels.shape[0]*tvRatio)
    trainData,valData = np.split(labels,[splitIdx])

    vbPrint('Training Data  : %i'%(len(trainData)))
    vbPrint('Val Data       : %i'%(len(valData)))

    # Generate Annotations
    vbPrint('Generating annotations for the training data')
    trainR2C = Raster2Coco(trainData, labelPatchesDir)
    trainJSON = trainR2C.createJSON()

    with open('%s/annotations_train.json'%(annDir), 'w+') as outfile:
        json.dump(trainJSON, outfile)

    del trainJSON
    del trainR2C

    vbPrint('Generating annotations for the validation data')
    valR2C = Raster2Coco(valData, labelPatchesDir)
    valJSON = valR2C.createJSON()

    with open('%s/annotations_val.json'%(annDir), 'w+') as outfile:
        json.dump(valJSON, outfile)

    del valJSON
    del valR2C

    vbPrint('Annotations made and saved successfuly')

max_stars_repo_name: aegean-ai/yolact-original

max_issues_repo_name: aegean-ai/yolact-original

max_forks_repo_name: aegean-ai/yolact-original


text:
### Copyright (C) 2017 NVIDIA Corporation. All rights reserved.
### Licensed under the CC BY-NC-SA 4.0 license (https://creativecommons.org/licenses/by-nc-sa/4.0/legalcode).
import time
from collections import OrderedDict
from options.train_options import TrainOptions
from data.data_loader import CreateDataLoader
from models.models import create_model
import util.util as util
from util.visualizer import Visualizer
import torch
from torch.autograd import Variable
from data.base_dataset import get_params, get_transform
from PIL import Image

import os
import numpy as np
import shutil
import video_utils

opt = TrainOptions().parse()
iter_path = os.path.join(opt.checkpoints_dir, opt.name, 'iter.txt')
if opt.continue_train:
    try:
        start_epoch, epoch_iter = np.loadtxt(iter_path , delimiter=',', dtype=int)
    except:
        start_epoch, epoch_iter = 1, 0
    print('Resuming from epoch %d at iteration %d' % (start_epoch, epoch_iter))
else:
    start_epoch, epoch_iter = 1, 0

# additional enforced options for video
opt.video_mode = True
opt.label_nc = 0
opt.no_instance = True

# could be changed, but not tested
opt.resize_or_crop = "none"
opt.batchSize = 1
opt.flat = False

# this debug directory will contain input/output frame pairs
if opt.debug:
    debug_dir = os.path.join(opt.checkpoints_dir, opt.name, 'debug')
    if os.path.isdir(debug_dir):
        shutil.rmtree(debug_dir)
    os.mkdir(debug_dir)

if opt.scheduled_sampling:
    print('ATTN! scheduled_sampling True by default')
    if opt.batchSize > 1:
        raise Exception('(for now) in "scheduled sampling" mode, --batchSize has to be 1')
    if not opt.serial_batches:
        raise Exception('(for now) in "scheduled sampling" mode, the --serial_batches option is necessary')
    if not opt.no_flip:
        raise Exception('(for now) in "scheduled sampling" mode, the --no_flip option is necessary')
    latest_generated_frame = None
    recursion = 0

opt.video_mode = True
# load frames dataset at the specified location
data_loader = CreateDataLoader(opt)
dataset = data_loader.load_data()

print(f'Training data loaded from: {opt.dataroot}')
print(f'Saving model to: {opt.name}')

dataset_size = len(data_loader)
print('#training images = %d' % dataset_size)
total_steps = (start_epoch-1) * dataset_size + epoch_iter
# print(f'Cur mem allocated: {torch.cuda.memory_allocated() / 1e6} MB')
model = create_model(opt)
# print(f'After model init. Cur mem allocated: {torch.cuda.memory_allocated() / 1e6} MB')
if opt.gpu:
    # check if CUDA is available
    train_on_gpu = torch.cuda.is_available()

    if not train_on_gpu:
        print('CUDA is not available.  Training on CPU ...')
    else:
        print('CUDA is available!  Training on GPU ...')
        # print(f'Cur mem allocated: {torch.cuda.memory_allocated() / 1e6} MB')
        # move model to gpu
        model = model.to('cuda')
        # print(f'After model moved. Cur mem allocated: {torch.cuda.memory_allocated() / 1e6} MB')

visualizer = Visualizer(opt)
display_delta = total_steps % opt.display_freq
print_delta = total_steps % opt.print_freq
save_delta = total_steps % opt.save_latest_freq

for epoch in range(start_epoch, opt.niter + opt.niter_decay + 1):
    epoch_start_time = time.time()
    if epoch != start_epoch:
        epoch_iter = epoch_iter % dataset_size

    for i, data in enumerate(dataset, start=epoch_iter):
        iter_start_time = time.time()
        total_steps += opt.batchSize
        epoch_iter += opt.batchSize

        left_frame = Image.open(data['left_path'][0])
        right_frame = Image.open(data['right_path'][0])

        params = get_params(opt, left_frame.size)
        transform = get_transform(opt, params)

        left_frame = transform(left_frame.convert('RGB'))
        right_frame = transform(right_frame.convert('RGB'))

        if opt.gpu:
            left_frame = left_frame.unsqueeze(0).to('cuda')
            # print(f'After left frame moved. Cur mem allocated: {torch.cuda.memory_allocated() / 1e6} MB')
            right_frame = right_frame.unsqueeze(0).to('cuda')
            # print(f'After right frame moved. Cur mem allocated: {torch.cuda.memory_allocated() / 1e6} MB')

        if opt.debug:
            video_utils.save_tensor(left_frame, debug_dir + "/step-%d-left-r%d.jpg" % (total_steps, recursion))
            video_utils.save_tensor(right_frame, debug_dir + "/step-%d-right.jpg" % total_steps)

        # print(f"LEFT FRAME: {left_frame.size()}")
        # print(f"RIGHT FRAME: {right_frame.size()}")

        losses, latest_generated_frame = model(
            left_frame, None,
            right_frame, None,
            infer=opt.scheduled_sampling
        )

        # sum per device losses
        losses = [ torch.mean(x) if not isinstance(x, int) else x for x in losses ]
        loss_dict = dict(zip(model.module.loss_names, losses))

        # calculate final loss scalar
        loss_D = (loss_dict['D_fake'] + loss_dict['D_real']) * 0.5
        loss_G = loss_dict['G_GAN'] + loss_dict.get('G_GAN_Feat',0) + loss_dict.get('G_VGG',0)

        ############### Backward Pass - frame1->frame2 ####################
        # update generator weights
        model.module.optimizer_G.zero_grad()
        loss_G.backward()
        model.module.optimizer_G.step()

        # update discriminator weights
        model.module.optimizer_D.zero_grad()
        loss_D.backward()
        model.module.optimizer_D.step()

        #call(["nvidia-smi", "--format=csv", "--query-gpu=memory.used,memory.free"])

        ############## Display results and errors ##########
        ### print out errors
        if total_steps % opt.print_freq == print_delta:
            errors = {k: v.item() if not isinstance(v, int) else v for k, v in loss_dict.items()}
            t = (time.time() - iter_start_time) / opt.batchSize
            visualizer.print_current_errors(epoch, epoch_iter, errors, t)
            visualizer.plot_current_errors(errors, total_steps)

        ### save latest model
        if total_steps % opt.save_latest_freq == save_delta:
            print('saving the latest model (epoch %d, total_steps %d)' % (epoch, total_steps))
            model.module.save('latest')
            np.savetxt(iter_path, (epoch, epoch_iter), delimiter=',', fmt='%d')

        if epoch_iter >= dataset_size:
            break

    # end of epoch
    iter_end_time = time.time()
    print('End of epoch %d / %d \t Time Taken: %d sec' %
          (epoch, opt.niter + opt.niter_decay, time.time() - epoch_start_time))

    ### save model for this epoch
    if epoch % opt.save_epoch_freq == 0:
        print('saving the model at the end of epoch %d, iters %d' % (epoch, total_steps))
        model.module.save('latest')
        model.module.save(epoch)
        np.savetxt(iter_path, (epoch+1, 0), delimiter=',', fmt='%d')

    ### instead of only training the local enhancer, train the entire network after certain iterations
    if (opt.niter_fix_global != 0) and (epoch == opt.niter_fix_global):
        model.module.update_fixed_params()

    ### linearly decay learning rate after certain iterations
    if epoch > opt.niter:
        model.module.update_learning_rate()

max_stars_repo_name: Kebniss/AutoDetect

max_issues_repo_name: Kebniss/AutoDetect

max_forks_repo_name: Kebniss/AutoDetect


text:
#!/usr/bin/python

import numpy as np
from scipy import optimize
from sympy import *
import matplotlib.pyplot as plt
import pdb

def residualRK2(point,S,Sp):
    CFL = symbols('CFL')
    Res=0.
    if S.shape[0]==1:
        S1=[S[0,0]]
        S2=[S[0,1]]
        Sum1=np.sum(S1) ; Sum2=np.sum(S2)
        Nmp=1
    else:
        S1=np.asarray(S[0,:])[0]
        S2=np.asarray(S[1,:])[0]
        Sum1=np.sum(S1) ; Sum2=np.sum(S2)
        Nmp=len(S1)
    if Sp.shape[0]==1:
        Sp1=[Sp[0,0]]
        Sp2=[Sp[0,0]]
        Sump1=np.sum(Sp1) ; Sump2=np.sum(Sp2)
        Nmpp=1
    else:
        Sp1=np.asarray(Sp[0,:])[0]
        Sp2=np.asarray(Sp[1,:])[0]
        Sump1=np.sum(Sp1) ; Sump2=np.sum(Sp2)
        Nmpp=len(Sp1)
    # Sum over material points in curent cell
    for k in range(Nmp):
        ## First order contributions
        D_mu = S1[k]*S1[point]/Sum1 + S2[k]*S2[point]/Sum2 + CFL*( S2[point]/Sum2 - S1[point]/Sum1 -Nmp*S2[k]*S2[point]/(Sum2**2) )
        ## Second order contributions
        D_mu += 0.5*Nmp*(CFL**2)*((S2[k]/Sum2)*(S1[point]/Sum1-S2[point]/Sum2) + (S2[point]/Sum2)*(Nmp*S2[k]/Sum2-1.)/Sum2)
        Res = Res +np.abs(D_mu)
    # Sum over material points in previous cell
    for k in range(Nmpp):
        ## First order contributions
        D_mu = CFL*Nmp*Sp2[k]*S1[point]/(Sum1*Sump2)
        ## Second order contributions
        D_mu +=0.5*Nmp*(CFL**2)*( S1[point]/(Sum1*Sump2)*(1.-Nmp*Sp2[k]/Sump2) -(Sp2[k]/Sump2)*(S1[point]/Sum1-S2[point]/Sum2) )
        Res=Res + np.abs(D_mu)
    Residual = lambdify((CFL),Res-1.)
    return Residual

def residualEuler(point,S,Sp):
    CFL = symbols('CFL')
    Res=0.
    if S.shape[0]==1:
        S1=[S[0,0]]
        S2=[S[0,1]]
        Sum1=np.sum(S1) ; Sum2=np.sum(S2)
        Nmp=1
    else:
        S1=np.asarray(S[0,:])[0]
        S2=np.asarray(S[1,:])[0]
        Sum1=np.sum(S1) ; Sum2=np.sum(S2)
        Nmp=len(S1)
    if Sp.shape[0]==1:
        Sp1=[Sp[0,0]]
        Sp2=[Sp[0,0]]
        Sump1=np.sum(Sp1) ; Sump2=np.sum(Sp2)
        Nmpp=1
    else:
        Sp1=np.asarray(Sp[0,:])[0]
        Sp2=np.asarray(Sp[1,:])[0]
        Sump1=np.sum(Sp1) ; Sump2=np.sum(Sp2)
        Nmpp=len(Sp1)
    # Sum over material points in curent cell
    for k in range(Nmp):
        D_ma = S1[point]*S1[k]/Sum1 + S2[point]*S2[k]/Sum2 + CFL*( S2[point]/Sum2 - S1[point]/Sum1 -Nmp*S2[point]*S2[k]/(Sum2**2) )
        Res = Res +np.abs(D_ma)
    for k in range(Nmpp):
        D_ma = CFL*Nmp*S1[point]*Sp2[k]/(Sum1*Sump2)
        Res=Res + np.abs(D_ma)
    Residual = lambdify((CFL),Res-1.)
    return Residual

# def gridSearch(function,tol=1.e-7):
#     samples=100000
#     # Find the bigest root of the residual by grid search algorithm
#     CFL=np.linspace(0.,1.,samples)
#     for i in CFL:
#         if i==CFL[samples-1]: return i
#         a0=function(i)
#         if a0<tol:
#             continue
#         else:
#             return i
def gridSearch(function,tol=1.e-7):
    samples=100000
    # Find the bigest root of the residual by grid search algorithm
    CFL=np.linspace(0.,1.,samples)
    for i in range(samples):
        value=CFL[-1-i]
        a0=function(value)
        if a0<tol:
            return value
        else:
            continue
    return 0.

# Symbolic function to evaluate shape functions
shape_functions=lambda x: np.matrix([(1-x)/DX,x/DX])

xn = np.array([0.,1.])
DX = 1.

## required for plotting residual
CFL=np.linspace(0.,1.,100.)

shift=0.25

# # 1PPC
# print "**************************************************************"
# print "******************  1PPC discretization **********************"
# print "**************************************************************"
# print "   "
# shapes=shape_functions(0.00000000000001)
# eulerSolution=optimize.root(residualEuler(0,shapes,shapes),1.,method='hybr',options={'xtol':1.e-4}).x
# rk2Solution=optimize.root(residualRK2(0,shapes,shapes),1.,method='hybr',options={'xtol':1.e-4}).x
# print "Euler solution, CFL= ",eulerSolution
# print "RK2 solution, CFL= ",rk2Solution

# 2PPC
print "**************************************************************"
print "******************  2PPC discretization **********************"
print "**************************************************************"
print "   "
shapes=shape_functions(np.array([0.25,0.75]))
#shapes=shape_functions(np.array([(1.-1./np.sqrt(3.))/2.,(1.+1./np.sqrt(3.))/2.]))
eulerSolution=[]
rk2Solution=[]

for i in range(np.shape(shapes)[0]):
    # eulerSolution.append(optimize.root(residualEuler(i,shapes,shapes),1.,method='hybr',options={'xtol':1.e-12}).x[0])
    # rk2Solution.append(optimize.root(residualRK2(i,shapes,shapes),1.,method='hybr',options={'xtol':1.e-12}).x[0])
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted"
shift=DX/10.
shapes=shape_functions(np.array([0.25+shift,0.75+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted left on nodes"
shift=-0.25
shapes=shape_functions(np.array([0.25+shift,0.75+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted right on nodes"
shift=+0.25
shapes=shape_functions(np.array([0.25+shift,0.75+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted symmetrically"
shift=+DX/10.
shapes=shape_functions(np.array([0.25-shift,0.75+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted symmetrically to nodes"
shift=0.25
shapes=shape_functions(np.array([0.25-shift,0.75+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

# 3PPC
print "**************************************************************"
print "******************  3PPC discretization **********************"
print "**************************************************************"
print "   "
shift=0.
shapes=shape_functions(np.array([DX/6.,0.5,1-DX/6.]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted"
shift=0.1
shapes=shape_functions(np.array([0.5*DX/3.+shift,0.5+shift,1.-0.5*DX/3.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ",(eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted left on nodes"
shift=-DX/6.
shapes=shape_functions(np.array([0.5*DX/3.+shift,0.5+shift,1.-0.5*DX/3.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ",(eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted right on nodes"
shift=DX/6.
shapes=shape_functions(np.array([0.5*DX/3.+shift,0.5+shift,1.-0.5*DX/3.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ",(eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted symetrically"
shift=0.1
shapes=shape_functions(np.array([0.5*DX/3.-shift,0.5,1.-0.5*DX/3.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted symmetrically to nodes"
shift=DX/6.
shapes=shape_functions(np.array([0.5*DX/3.-shift,0.5,1-0.5*DX/3.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

# 4PPC
print "**************************************************************"
print "******************  4PPC discretization **********************"
print "**************************************************************"
print "   "

shapes=shape_functions(np.array([0.5*DX/4.,3*DX/8.,5*DX/8.,7*DX/8.]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted"
shift=0.1
shapes=shape_functions(np.array([0.5*DX/4.+shift,3*DX/8.+shift,5*DX/8.+shift,7*DX/8.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted left on nodes"
shift=-DX/8.
shapes=shape_functions(np.array([0.5*DX/4.+shift,3*DX/8.+shift,5*DX/8.+shift,7*DX/8.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted right on nodes"
shift=DX/8.
shapes=shape_functions(np.array([0.5*DX/4.+shift,3*DX/8.+shift,5*DX/8.+shift,7*DX/8.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
shift=0.1
print "Shifted symetrically"
shapes=shape_functions(np.array([0.5*DX/4.-shift,3*DX/8.-shift,5*DX/8.+shift,7*DX/8.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

print "   "
print "Shifted symetrically to nodes"
shift=DX/8.
shapes=shape_functions(np.array([0.5*DX/4.-shift,3*DX/8.-shift,5*DX/8.+shift,7*DX/8.+shift]))
eulerSolution=[]
rk2Solution=[]
for i in range(np.shape(shapes)[1]):
    eulerSolution.append(gridSearch(residualEuler(i,shapes,shapes)))
    rk2Solution.append(gridSearch(residualRK2(i,shapes,shapes)))
print "Euler solution, CFL= ", (eulerSolution)
print "RK2 solution, CFL= ", (rk2Solution)

max_stars_repo_name: adRenaud/research

max_issues_repo_name: adRenaud/research

max_forks_repo_name: adRenaud/research

reason: import numpy,from scipy,from sympy


text:
#동적 계획법: 등굣길
import numpy as np

def solution(m, n, puddles):
    num_list = np.zeros((m, n))
    num_list[0, 0] = 1

    for p in puddles:
        num_list[p[0] - 1][p[1] - 1] = -1

    for i in range(m):
        for j in range(n):
            if (num_list[i][j] == -1 or i == 0 and j == 0):
                continue
            ans = 0
            if (num_list[i - 1][j] != -1 and i - 1 >= 0):
                ans += num_list[i - 1][j]
            if (num_list[i][j - 1] != -1 and j - 1 >= 0):
                ans += num_list[i][j - 1]

            num_list[i][j] = int(ans % 1000000007)

    return int(num_list[m - 1][n - 1] % 1000000007)

if __name__ == '__main__':
    print(solution(4, 3, [[2, 2]]))


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Author: Zeyuan Shang
# @Date:   2016-01-13 15:36:10
# @Last Modified by:   Zeyuan Shang
# @Last Modified time: 2016-01-13 15:36:14
import numpy as np

def calc(gpas):
    scores = map(float, raw_input().split())
    coef = np.corrcoef(gpas, scores)[0, 1]
    return 0 if np.isnan(coef) else coef

def main():
    T = input()
    for _ in xrange(T):
        raw_input()
        gpas = map(float, raw_input().split())
        m = max([calc(gpas), i] for i in xrange(1, 6))[1]
        print m

if __name__ == "__main__":
    main()

max_stars_repo_name: zeyuanxy/HackerRank

max_issues_repo_name: zeyuanxy/HackerRank

max_forks_repo_name: zeyuanxy/HackerRank


text:
import os

import numpy as np
import matplotlib.pyplot as plt

prev_embeds = np.load('initial.npy')
prev_embeds_queries = np.load('initial_query_embeds.npy')
prev_topk_by_queries = np.load('initial_topk_by_queries.npy')

member_embeds_avg_change_history = []
member_embeds_min_change_history = []
member_embeds_max_change_history = []

nonmember_embeds_avg_change_history = []
nonmember_embeds_min_change_history = []
nonmember_embeds_max_change_history = []

member_query_num_neighbors_swapped_avg_history = []
member_query_num_neighbors_swapped_min_history = []
member_query_num_neighbors_swapped_max_history = []

nonmember_query_num_neighbors_swapped_avg_history = []
nonmember_query_num_neighbors_swapped_min_history = []
nonmember_query_num_neighbors_swapped_max_history = []

member_query_neighbors_zero_change_count_history = []
nonmember_query_neighbors_zero_change_count_history = []

member_query_nearest_distance_avg_history = []
member_query_nearest_distance_min_history = []
member_query_nearest_distance_max_history = []

member_query_embed_change_avg_history = []
member_query_embed_change_min_history = []
member_query_embed_change_max_history = []

nonmember_query_nearest_distance_avg_history = []
nonmember_query_nearest_distance_min_history = []
nonmember_query_nearest_distance_max_history = []

nonmember_query_embed_change_avg_history = []
nonmember_query_embed_change_min_history = []
nonmember_query_embed_change_max_history = []

for i in range(722):
    batch_members = set(np.load(str(i) + '_topk.npy').flatten())
    batch_embeds = np.load(str(i) + '.npy')
    batch_embeds_queries = np.load(str(i) + '_query_embeds.npy')
    batch_topk_by_queries = np.load(str(i) + '_topk_by_queries.npy')
    batch_query_idxs = np.load(str(i) + '_query_idx.npy')

    member_embeds_changes = []
    nonmember_embeds_changes = []

    changes = np.linalg.norm(batch_embeds - prev_embeds, ord=2, axis=1)

    for k in range(len(changes)):
        if k in batch_members:
            member_embeds_changes.append(changes[k])
        else:
            nonmember_embeds_changes.append(changes[k])

    member_embeds_changes_avg = np.mean(member_embeds_changes)
    member_embeds_changes_min = np.min(member_embeds_changes)
    member_embeds_changes_max = np.max(member_embeds_changes)

    member_embeds_avg_change_history.append(member_embeds_changes_avg)
    member_embeds_min_change_history.append(member_embeds_changes_min)
    member_embeds_max_change_history.append(member_embeds_changes_max)

    nonmember_embeds_changes_avg = np.mean(nonmember_embeds_changes)
    nonmember_embeds_changes_min = np.min(nonmember_embeds_changes)
    nonmember_embeds_changes_max = np.max(nonmember_embeds_changes)

    nonmember_embeds_avg_change_history.append(nonmember_embeds_changes_avg)
    nonmember_embeds_min_change_history.append(nonmember_embeds_changes_min)
    nonmember_embeds_max_change_history.append(nonmember_embeds_changes_max)

    member_query_num_neighbors_swapped = []
    nonmember_query_num_neighbors_swapped = []

    member_query_zero_neighbors_change_count = 0
    nonmember_query_zero_neighbors_change_count = 0

    member_query_nearest_distances = []
    member_query_embed_changes = []

    nonmember_query_nearest_distances = []
    nonmember_query_embed_changes = []

    for j in range(len(batch_topk_by_queries)):
        nn_prev = set(prev_topk_by_queries[j])
        nn_cur = set(batch_topk_by_queries[j])

        num_swapped = len(nn_prev - nn_cur)

        nearest_point_embed = batch_embeds[batch_topk_by_queries[j][0]]

        query_embed_cur = batch_embeds_queries[j]
        query_embed_prev = prev_embeds_queries[j]

        distance_to_nearest = np.linalg.norm(
            query_embed_cur - nearest_point_embed, ord=2)

        query_embed_changes = np.linalg.norm(
            query_embed_cur - query_embed_prev, ord=2)

        if j in batch_query_idxs:
            member_query_num_neighbors_swapped.append(num_swapped)

            member_query_nearest_distances.append(distance_to_nearest)
            member_query_embed_changes.append(query_embed_changes)
        else:
            nonmember_query_num_neighbors_swapped.append(num_swapped)

            nonmember_query_nearest_distances.append(distance_to_nearest)
            nonmember_query_embed_changes.append(query_embed_changes)

        if num_swapped == 0:
            if j in batch_query_idxs:
                member_query_zero_neighbors_change_count = member_query_zero_neighbors_change_count + 1
            else:
                nonmember_query_zero_neighbors_change_count = nonmember_query_zero_neighbors_change_count + 1

    member_query_num_neighbors_swapped_avg = np.mean(member_query_num_neighbors_swapped)
    member_query_num_neighbors_swapped_avg_history.append(member_query_num_neighbors_swapped_avg)

    member_query_num_neighbors_swapped_min = np.min(member_query_num_neighbors_swapped)
    member_query_num_neighbors_swapped_min_history.append(member_query_num_neighbors_swapped_min)

    member_query_num_neighbors_swapped_max = np.max(member_query_num_neighbors_swapped)
    member_query_num_neighbors_swapped_max_history.append(member_query_num_neighbors_swapped_max)

    nonmember_query_num_neighbors_swapped_avg = np.mean(nonmember_query_num_neighbors_swapped)
    nonmember_query_num_neighbors_swapped_avg_history.append(nonmember_query_num_neighbors_swapped_avg)

    nonmember_query_num_neighbors_swapped_min = np.min(nonmember_query_num_neighbors_swapped)
    nonmember_query_num_neighbors_swapped_min_history.append(nonmember_query_num_neighbors_swapped_min)

    nonmember_query_num_neighbors_swapped_max = np.max(nonmember_query_num_neighbors_swapped)
    nonmember_query_num_neighbors_swapped_max_history.append(nonmember_query_num_neighbors_swapped_max)

    member_query_neighbors_zero_change_count_history.append(member_query_zero_neighbors_change_count / len(batch_query_idxs))
    nonmember_query_neighbors_zero_change_count_history.append(nonmember_query_zero_neighbors_change_count / (len(batch_topk_by_queries) - len(batch_query_idxs)))

    member_query_nearest_distance_avg = np.mean(member_query_nearest_distances)
    member_query_nearest_distance_avg_history.append(member_query_nearest_distance_avg)

    member_query_nearest_distance_min = np.min(member_query_nearest_distances)
    member_query_nearest_distance_min_history.append(member_query_nearest_distance_min)

    member_query_nearest_distance_max = np.max(member_query_nearest_distances)
    member_query_nearest_distance_max_history.append(member_query_nearest_distance_max)

    member_query_embed_change_avg = np.mean(member_query_embed_changes)
    member_query_embed_change_avg_history.append(member_query_embed_change_avg)

    member_query_embed_change_min = np.min(member_query_embed_changes)
    member_query_embed_change_min_history.append(member_query_embed_change_min)

    member_query_embed_change_max = np.max(member_query_embed_changes)
    member_query_embed_change_max_history.append(member_query_embed_change_max)

    nonmember_query_nearest_distance_avg = np.mean(nonmember_query_nearest_distances)
    nonmember_query_nearest_distance_avg_history.append(nonmember_query_nearest_distance_avg)

    nonmember_query_nearest_distance_min = np.min(nonmember_query_nearest_distances)
    nonmember_query_nearest_distance_min_history.append(nonmember_query_nearest_distance_min)

    nonmember_query_nearest_distance_max = np.max(nonmember_query_nearest_distances)
    nonmember_query_nearest_distance_max_history.append(nonmember_query_nearest_distance_max)

    nonmember_query_embed_change_avg = np.mean(nonmember_query_embed_changes)
    nonmember_query_embed_change_avg_history.append(nonmember_query_embed_change_avg)

    nonmember_query_embed_change_min = np.min(nonmember_query_embed_changes)
    nonmember_query_embed_change_min_history.append(nonmember_query_embed_change_min)

    nonmember_query_embed_change_max = np.max(nonmember_query_embed_changes)
    nonmember_query_embed_change_max_history.append(nonmember_query_embed_change_max)

    print("Batch %d:" % i)
    print()

    print("Member embedding avg change =", member_embeds_changes_avg)
    print("Member embedding min change =", member_embeds_changes_min)
    print("Member embedding max change =", member_embeds_changes_max)
    print()

    print("Non-member embedding avg change =", nonmember_embeds_changes_avg)
    print("Non-member embedding min change =", nonmember_embeds_changes_min)
    print("Non-member embedding max change =", nonmember_embeds_changes_max)
    print()

    print("Avg. # of neighbors swapped per query in the batch =", member_query_num_neighbors_swapped_avg)
    print("Min # of neighbors swapped per query in the batch =", member_query_num_neighbors_swapped_min)
    print("Max # of neighbors swapped per query in the batch =", member_query_num_neighbors_swapped_max)
    print()

    print("Avg. # of neighbors swapped per query NOT in the batch =", nonmember_query_num_neighbors_swapped_avg)
    print("Min # of neighbors swapped per query NOT in the batch =", nonmember_query_num_neighbors_swapped_min)
    print("Max # of neighbors swapped per query NOT in the batch =", nonmember_query_num_neighbors_swapped_max)
    print()

    print("# of zero neighbor changes for query in the batch =", member_query_zero_neighbors_change_count, "out of", len(batch_query_idxs))
    print("# of zero neighbor changes for query NOT in the batch =", nonmember_query_zero_neighbors_change_count, "out of", len(batch_topk_by_queries) - len(batch_query_idxs))
    print()

    print("Avg. nearest distance per query in the batch =", member_query_nearest_distance_avg)
    print("Min nearest distance per query in the batch =", member_query_nearest_distance_min)
    print("Max nearest distance per query in the batch =", member_query_nearest_distance_max)
    print()

    print("Avg. nearest distance per query NOT in the batch =", nonmember_query_nearest_distance_avg)
    print("Min nearest distance per query NOT in the batch =", nonmember_query_nearest_distance_min)
    print("Max nearest distance per query NOT in the batch =", nonmember_query_nearest_distance_max)
    print()

    print("Avg. query embed changes in the batch =", member_query_embed_change_avg)
    print("Min query embed changes in the batch =", member_query_embed_change_min)
    print("Max query embed changes in the batch =", member_query_embed_change_max)
    print()

    print("Avg. query embed changes NOT in the batch =", nonmember_query_embed_change_avg)
    print("Min query embed changes NOT in the batch =", nonmember_query_embed_change_min)
    print("Max query embed changes NOT in the batch =", nonmember_query_embed_change_max)
    print()

    print()

    prev_embeds = batch_embeds
    prev_topk_by_queries = batch_topk_by_queries
    prev_embeds_queries = batch_embeds_queries

# Plot

# Create a directory for plots
plots_dir = os.path.join("__plots")

os.makedirs(plots_dir, exist_ok=True)

# Average embedding changes of dense representations in the dictionary
plt.figure()
plt.xlabel('steps')
plt.ylabel('Average embedding changes (L2 distance)')

plt.plot(
    list(range(len(member_embeds_avg_change_history))),
    member_embeds_avg_change_history,
    label='within batch')

plt.plot(
    list(range(len(nonmember_embeds_avg_change_history))),
    nonmember_embeds_avg_change_history,
    label='outside batch')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'embeds_avg_change_history.png'))
plt.close()

# Stats for embeddings changes of dense representations in the dictionary within batch
plt.figure()
plt.xlabel('steps')
plt.ylabel('embedding changes (L2 distance)')

plt.plot(
    list(range(len(member_embeds_avg_change_history))),
    member_embeds_avg_change_history,
    label='avg')

plt.plot(
    list(range(len(member_embeds_max_change_history))),
    member_embeds_max_change_history,
    label='max')

plt.plot(
    list(range(len(member_embeds_min_change_history))),
    member_embeds_min_change_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'member_embeds_change_history.png'))
plt.close()

# Stats for embeddings changes of dense representations in the dictionary OUTSIDE batch
plt.figure()
plt.xlabel('steps')
plt.ylabel('embedding changes (L2 distance)')

plt.plot(
    list(range(len(nonmember_embeds_avg_change_history))),
    nonmember_embeds_avg_change_history,
    label='avg')

plt.plot(
    list(range(len(nonmember_embeds_max_change_history))),
    nonmember_embeds_max_change_history,
    label='max')

plt.plot(
    list(range(len(nonmember_embeds_min_change_history))),
    nonmember_embeds_min_change_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'nonmember_embeds_change_history.png'))
plt.close()

# Number of neighbors swapped per query (within batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('# neighbors swapped (per query)')

plt.plot(
    list(range(len(member_query_num_neighbors_swapped_avg_history))),
    member_query_num_neighbors_swapped_avg_history,
    label='avg')

plt.plot(
    list(range(len(member_query_num_neighbors_swapped_max_history))),
    member_query_num_neighbors_swapped_max_history,
    label='max')

plt.plot(
    list(range(len(member_query_num_neighbors_swapped_min_history))),
    member_query_num_neighbors_swapped_min_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'member_query_num_neighbors_swapped_history.png'))
plt.close()

# Number of neighbors swapped per query (OUTSIDE batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('# neighbors swapped (per query)')

plt.plot(
    list(range(len(nonmember_query_num_neighbors_swapped_avg_history))),
    nonmember_query_num_neighbors_swapped_avg_history,
    label='avg')

plt.plot(
    list(range(len(nonmember_query_num_neighbors_swapped_max_history))),
    nonmember_query_num_neighbors_swapped_max_history,
    label='max')

plt.plot(
    list(range(len(nonmember_query_num_neighbors_swapped_min_history))),
    nonmember_query_num_neighbors_swapped_min_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'nonmember_query_num_neighbors_swapped_history.png'))
plt.close()

# Proportion of queries with ZERO neighbor changes (in/out batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('proportion of queries w/ ZERO neighbor changes')

plt.plot(
    list(range(len(member_query_neighbors_zero_change_count_history))),
    member_query_neighbors_zero_change_count_history,
    label='queries in batch')

plt.plot(
    list(range(len(nonmember_query_neighbors_zero_change_count_history))),
    nonmember_query_neighbors_zero_change_count_history,
    label='queries NOT in batch')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'zero_neighbors_change_count_history.png'))
plt.close()

# Nearest distance per query (within batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('distance to the nearest neighbor')

plt.plot(
    list(range(len(member_query_nearest_distance_avg_history))),
    member_query_nearest_distance_avg_history,
    label='avg')

plt.plot(
    list(range(len(member_query_nearest_distance_max_history))),
    member_query_nearest_distance_max_history,
    label='max')

plt.plot(
    list(range(len(member_query_nearest_distance_min_history))),
    member_query_nearest_distance_min_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'member_query_nearest_distance_history.png'))
plt.close()

# Nearest distance per query (outside batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('distance to the nearest neighbor')

plt.plot(
    list(range(len(nonmember_query_nearest_distance_avg_history))),
    nonmember_query_nearest_distance_avg_history,
    label='avg')

plt.plot(
    list(range(len(nonmember_query_nearest_distance_max_history))),
    nonmember_query_nearest_distance_max_history,
    label='max')

plt.plot(
    list(range(len(nonmember_query_nearest_distance_min_history))),
    nonmember_query_nearest_distance_min_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'nonmember_query_nearest_distance_history.png'))
plt.close()

# query embed changes per query (within batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('query embedding changes (L2 distance)')

plt.plot(
    list(range(len(member_query_embed_change_avg_history))),
    member_query_embed_change_avg_history,
    label='avg')

plt.plot(
    list(range(len(member_query_embed_change_max_history))),
    member_query_embed_change_max_history,
    label='max')

plt.plot(
    list(range(len(member_query_embed_change_min_history))),
    member_query_embed_change_min_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'member_query_embed_change_history.png'))
plt.close()

# query embed changes per query (outside batch)
plt.figure()
plt.xlabel('steps')
plt.ylabel('query embedding changes (L2 distance)')

plt.plot(
    list(range(len(nonmember_query_embed_change_avg_history))),
    nonmember_query_embed_change_avg_history,
    label='avg')

plt.plot(
    list(range(len(nonmember_query_embed_change_max_history))),
    nonmember_query_embed_change_max_history,
    label='max')

plt.plot(
    list(range(len(nonmember_query_embed_change_min_history))),
    nonmember_query_embed_change_min_history,
    label='min')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'nonmember_query_embed_change_history.png'))
plt.close()

# avg distance change + avg query embed change
plt.figure()
plt.xlabel('avg distance to the nearest neighbor')
plt.ylabel('avg query embed change')

plt.scatter(
    member_query_nearest_distance_avg_history,
    member_query_embed_change_avg_history,
    label='queries in batch')

plt.scatter(
    nonmember_query_nearest_distance_avg_history,
    nonmember_query_embed_change_avg_history,
    label='queries NOT in batch')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'nearest_distance_query_embed_change_history.png'))
plt.close()

# avg distance change + number of neighbors changed
plt.figure()
plt.xlabel('avg distance to the nearest neighbor')
plt.ylabel('avg number of neighbors changed')

plt.scatter(
    member_query_nearest_distance_avg_history,
    member_query_num_neighbors_swapped_avg_history,
    label='queries in batch')

plt.scatter(
    nonmember_query_nearest_distance_avg_history,
    nonmember_query_num_neighbors_swapped_avg_history,
    label='queries NOT in batch')

plt.legend()
plt.savefig(os.path.join(plots_dir, 'nearest_distance_num_neighbors_swapped_history.png'))
plt.close()

max_stars_repo_name: ronaldseoh/BioSyn

max_issues_repo_name: ronaldseoh/BioSyn

max_forks_repo_name: ronaldseoh/BioSyn


text:
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import numpy as np

from maro.rl import AbsAgent, ColumnBasedStore

class CIMAgent(AbsAgent):
    def __init__(
        self, name, algorithm, experience_pool: ColumnBasedStore, min_experiences_to_train, num_batches, batch_size
    ):
        super().__init__(name, algorithm, experience_pool)
        self._min_experiences_to_train = min_experiences_to_train
        self._num_batches = num_batches
        self._batch_size = batch_size

    def train(self):
        if len(self._experience_pool) < self._min_experiences_to_train:
            return

        for _ in range(self._num_batches):
            indexes, sample = self._experience_pool.sample_by_key("loss", self._batch_size)
            state = np.asarray(sample["state"])
            action = np.asarray(sample["action"])
            reward = np.asarray(sample["reward"])
            next_state = np.asarray(sample["next_state"])
            loss = self._algorithm.train(state, action, reward, next_state)
            self._experience_pool.update(indexes, {"loss": loss})

max_stars_repo_name: KangFengjian/maro

max_issues_repo_name: KangFengjian/maro

max_forks_repo_name: KangFengjian/maro


text:
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data

class DataGenerator:
    def __init__(self, config):

        self.config = config

        self.mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

    def next_batch(self, batch_size):
        x, y = self.mnist.train.next_batch(batch_size)
        x = np.reshape(x, (batch_size, 28, 28))
        #y = np.reshape(y, (batch_size, 10, 1))
        x = np.asarray(x)
        y = np.asarray(y)

        yield np.asarray(x), np.asarray(y)