text:
import os
import sys
import re
import numpy as np
import pandas as pd
from Bio import SeqIO

class align(object):
    def __init__(self, normalize=False):
        self.substitution_matrix = None
        self.scoring_matrix = None
        self.traceback_matrix = None
        self.seq_A = None
        self.seq_B = None

        # Gap opening and extension related
        self.gap_opening_penalty = -7
        self.gap_extension_penalty = -3
        # Keep track of gap openings across rows (iterate over rows to fill matrix so this is easy)
        self.extending_gap = False

        self.normalize = normalize
        self.max_alignment_score = []

    def initialize_scoring_matrix(self):
        """
        Initialize a [len(seq_A + 1) x len(seq_B + 1)] scoring matrix where columns and row indicies
        are 1-indexed (0-index reserved for 0 scores). Also initialize a traceback matrix to
        keep track of decisions (0: diag, 1: left, 2: up)

        Parameters
        ----------
        seq_A - represented across COLUMNS
        seq_B - represented down ROWS

        """

        # self.scoring_matrix = pd.DataFrame(columns= np.arange(len(self.seq_A) + 1), index= np.arange(len(self.seq_B) + 1))
        self.scoring_matrix = np.full((len(self.seq_B) + 1, len(self.seq_A) + 1), -1) # [ rows x columns ]
        self.traceback_matrix = np.full((len(self.seq_B) + 1, len(self.seq_A) + 1), -1) # [ rows x columns ]

    def fill_in_matrix(self, working_pair):
        """
        Fill in the scoring matrix based on the user selected substitution matrix

        """

        print('\n', working_pair)

        # Initialize 0-indexed row and column
        for column in np.arange(len(self.seq_A) + 1):
            self.scoring_matrix[0, column] = 0
        for row in np.arange(len(self.seq_B) + 1):
            self.scoring_matrix[row, 0] = 0

        # Make a list to keep track of column gap openings
        seq_A_gap_openings = [False] * (len(self.seq_A) + 1)

        # Iterate over rows and fill in the matrix based on substitution matrix
        for row_index, row in enumerate(self.scoring_matrix):

            # Reset row gap opening bool at start of each new row
            extending_gap = False

            for column_index in np.arange(len(row)):
                if row_index > 0 and column_index > 0:

                    # Determine similarity score
                    similarity_score = self.substitution_matrix.ix[str(self.seq_B.seq[row_index - 1]), str(self.seq_A.seq[column_index - 1])]

                    # Decide whether to use gap_extension or gap_opening penalty score across rows
                    if extending_gap == False:
                        gap_penalty_left = self.gap_opening_penalty
                    else:
                        gap_penalty_left = self.gap_extension_penalty

                    # Decide whether to use gap extension or gap opening penalty score down columns
                    if seq_A_gap_openings[column_index] == False:
                        gap_penalty_up = self.gap_opening_penalty
                    else:
                        gap_penalty_up = self.gap_extension_penalty

                    # Calculate possible scores for current cell in scoring matrix
                    score_up = self.scoring_matrix[row_index - 1, column_index] + gap_penalty_up
                    score_left = self.scoring_matrix[row_index, column_index - 1] + gap_penalty_left
                    score_diag = self.scoring_matrix[row_index - 1, column_index - 1] + similarity_score

                    # Take highest final score and assign to current in scoring matrix
                    final_score = max(score_up, score_left, score_diag)

                    # Toggle gap_opening or gap_extension across rows and down columns
                    # Priority diag > left > up
                    if max(score_up, score_left, score_diag) == score_diag:
                        extending_gap = False
                        seq_A_gap_openings[column_index] = False
                        self.traceback_matrix[row_index, column_index] = 0
                    elif max(score_up, score_left, score_diag) == score_left:
                        extending_gap = True
                        seq_A_gap_openings[column_index] = False
                        self.traceback_matrix[row_index, column_index] = 1
                    else: # max(score_up, score_left, score_diag) == score_up
                        seq_A_gap_openings[column_index] = True
                        extending_gap = False
                        self.traceback_matrix[row_index, column_index] = 2

                    self.scoring_matrix[row_index, column_index] = final_score if final_score > 0 else 0

        assert len(np.argwhere(self.scoring_matrix == -1)) == 0, "The scoring matrix was not completely filled in!"

        np.savetxt("Traceback_matrix.csv", self.traceback_matrix, delimiter=",")

        names = re.split('/|\.', working_pair.strip())
        file_name = '_'.join([names[1], names[3]])
        np.savetxt("{}.csv".format(file_name), self.scoring_matrix, delimiter=",")

    def traceback_II(self):
        # Find maximum score in matrix

        row_max_indicies = np.amax(self.scoring_matrix, axis=0)
        seq_B_high_score = np.argmax(row_max_indicies, axis=0)
        seq_A_high_score = np.argmax(self.scoring_matrix, axis=0)

        column_index_high = seq_B_high_score
        row_index_high = seq_A_high_score[column_index_high]

        print("Starting Column index: {}".format(column_index_high))
        print("Starting Row index: {}".format(row_index_high))

        # Initialize variables needed for traceback
        current_score = self.scoring_matrix[row_index_high, column_index_high]

        # Record maximum scores for each alignment
        if self.normalize == True:
            shorter_length = min(len(self.seq_A.seq), len(self.seq_B.seq))
            self.max_alignment_score.append(current_score / shorter_length)

            print("SHORTER LENGTH: {}".format(shorter_length))
            print("NORMALIZED SCORE: {}".format(current_score / shorter_length))

        else:

            self.max_alignment_score.append(current_score)

        print("Current Score: {}".format(current_score))

        seq_A_alignment = []
        seq_B_alignment = []

        current_row_index = row_index_high
        current_column_index = column_index_high

        # Begin traceback, stop when (above | left | diagonal) == 0
        while self.scoring_matrix[current_row_index, current_column_index] > 0:
            direction = self.traceback_matrix[current_row_index, current_column_index]

            if direction == 0: # Diag
                seq_A_alignment.append(str(self.seq_A.seq[current_column_index - 1]))
                seq_B_alignment.append(str(self.seq_B.seq[current_row_index - 1]))
                current_row_index -= 1
                current_column_index -= 1

            elif direction == 1: # Left
                seq_A_alignment.append(str(self.seq_A.seq[current_column_index - 1]))
                seq_B_alignment.append('-')
                current_column_index -= 1

            elif direction == 2: # Up
                seq_A_alignment.append('-')
                seq_B_alignment.append(str(self.seq_B.seq[current_row_index - 1]))
                current_row_index -= 1

            else:
                print('How did you get here')

        # construct_bar_things
        bar_thing_list = []
        for seq_A_seq, seq_B_seq in zip(seq_A_alignment, seq_B_alignment):
            if seq_A_seq == seq_B_seq:
                bar_thing_list.append('|')
            elif seq_A_seq == '-' or seq_B_seq == '-':
                bar_thing_list.append(' ')
            else:
                bar_thing_list.append(':')

        print(''.join(reversed(seq_A_alignment)))
        print(''.join(reversed(bar_thing_list)))
        print(''.join(reversed(seq_B_alignment)))

        seq_A_string = ''.join(reversed(seq_A_alignment))
        bar_things = ''.join(reversed(bar_thing_list))
        seq_B_string = ''.join(reversed(seq_B_alignment))

        return seq_A_string, bar_things, seq_B_string


text:
from .__activation import ActivationFunction
import numpy as np

class HardTanh(ActivationFunction):
    """
    Class represents the hardtanh activation function

    Examples:
    ---------
    >>> x = np.array([1, 2, 3])           # input vector
    >>> f = HardTanh()                    # creating HardTanh object
    >>> print(f(x))                       # calculating HardTanh of the input
    >>> print(f.local_grad)               # get local_grad of the HardTanh at the input

    """

    ID = 0

    def __init__(self):
        super(HardTanh, self).__init__()

    def _forward(self, x):
        """
        - Forward pass of the HardTanh function
        - hardtanh(x) ={
                        1   x>1
                       -1   x<-1
        }

        :param x: input that is wanted to calculate the HardTanh at
        :return: HardTanh value at input x
        :rtype: np.ndarray
        """
        X = np.copy(x)
        X[x > 1] = 1
        X[x < -1] = -1
        return X  # or np.maximum(-1, np.minimum(1, x))

    def _calc_local_grad(self, x):
        """
        - Backward pass of the tanh function
        - ∇ HardTanh = {
                            0        1 < x < -1
                            1       -1 <= x < 1
        }

        to get more info about HardTanh
        :param x: input that is wanted to calculate the HardTanh at
        :return: HardTanh gradient at input x
        :rtype: np.ndarray
        """
        X = np.copy(x)
        X[-1 > x > 1] = 0
        X[-1 <= x < 1] = 1
        return X

ext: py

reason: import numpy


text:
import nibabel as nib
import numpy as np
import os
from os.path import join as opj
import scipy.io as sio
from glob import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
from matplotlib import rc
from matplotlib.colors import ListedColormap
from matplotlib.colors import Normalize
import mayavi as my
import surfer
from scipy import ma
from scipy import interp

import sys
sys.path.append('code/visualize/ejcbrain')
import ejcbrain as EB

hemi = 'lh'
views = ['lat','med']
thrsh = -10
subject_id = 'fsaverage5'
surf = 'pial'
cbar = False
cmap_array = np.load('data/colors/ejc_custom2.npy')
cmap = ListedColormap(cmap_array)

font = {'family' : 'arial',
        'size'   : 6}
rc('font', **font)

# create a class to make an asymmetric colorbar
class MidpointNormalize(Normalize):
    def __init__(self, vmin, vmax, midpoint=0, clip=False):
        self.midpoint = midpoint
        Normalize.__init__(self, vmin, vmax, clip)

    def __call__(self, value, clip=None):
        normalized_min = max(0, 1 / 2 * (1 - abs((self.midpoint - self.vmin) / (self.midpoint - self.vmax))))
        normalized_max = min(1, 1 / 2 * (1 + abs((self.vmax - self.midpoint) / (self.midpoint - self.vmin))))
        normalized_mid = 0.5
        x, y = [self.vmin, self.midpoint, self.vmax], [normalized_min, normalized_mid, normalized_max]
        return ma.masked_array(interp(value, x, y))

for m in ['CT','SA']:
	for view in views:
		b = nib.freesurfer.mghformat.load('data/Sunetal2018_Fig1Maps/left-'+m+'-beta.mgh')
		p = nib.freesurfer.mghformat.load('data/Sunetal2018_Fig1Maps/left-'+m+'-sig.mgh')
		# p = nib.freesurfer.mghformat.load('data/Sunetal2018_Fig1Maps/left-CT-sig.mgh')
		b = np.squeeze(np.asarray(b.get_data()[:,:,:,1])) # select 22q-control predictor betas
		p = np.squeeze(np.asarray(p.get_data()))

		data_plot = -b
		clim = np.abs(b).max()
		cmax = b.max()
		cmin = b.min()
		fig = my.mlab.figure(size=(340,340))
		fig = my.mlab.gcf()
		brain = surfer.Brain(subject_id, hemi, surf,figure=fig,views=view,background='white', alpha = 1)
		brain.add_data(data_plot,min=cmin,max=cmax,mid=0,thresh = thrsh, colormap= cmap,alpha=1,colorbar=cbar)
		my.mlab.savefig(figure=fig,filename=opj('data','Sunetal2018_Fig1Maps',view+m+subject_id+'Betas.png'),magnification=1)

	x = np.vstack([np.linspace(cmin,cmax,100) for y in np.arange(3)])
	fig, ax = plt.subplots()
	cax=ax.imshow(x,cmap=cmap,norm=MidpointNormalize(vmin=cmin,vmax=cmax,midpoint=0))
	fig.colorbar(cax,ticks=[cmin,0,cmax],orientation='horizontal')
	fig.set_size_inches(1.2,2)
	fig.savefig(opj('data','Sunetal2018_Fig1Maps',view+m+subject_id+'BetaColormap.pdf'))

reason: import numpy,import scipy,from scipy


text:
from __future__ import print_function

import sys
import math
import threading
import argparse
import time

import numpy as np
from mpi4py import MPI

import torch
from torch.autograd import Variable
from torch import nn
import torch.nn.functional as F

from nn_ops import NN_Trainer, accuracy
from data_loader_ops.my_data_loader import DataLoader

from distributed_worker import *
from sync_replicas_master_nn import *

def add_fit_args(parser):
    """
    parser : argparse.ArgumentParser
    return a parser added with args required by fit
    """
    # Training settings
    parser.add_argument('--batch-size', type=int, default=128, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test-batch-size', type=int, default=500, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=100, metavar='N',
                        help='number of epochs to train (default: 10)')
    parser.add_argument('--max-steps', type=int, default=10000, metavar='N',
                        help='the maximum number of iterations')
    parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                        help='learning rate (default: 0.01)')
    parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                        help='SGD momentum (default: 0.5)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')
    parser.add_argument('--network', type=str, default='LeNet', metavar='N',
                        help='which kind of network we are going to use, support LeNet and ResNet currently')
    parser.add_argument('--mode', type=str, default='normal', metavar='N',
                        help='determine if we kill the stragglers or just implement normal training')
    parser.add_argument('--kill-threshold', type=float, default=7.0, metavar='KT',
                        help='timeout threshold which triggers the killing process (default: 7s)')
    parser.add_argument('--dataset', type=str, default='MNIST', metavar='N',
                        help='which dataset used in training, MNIST and Cifar10 supported currently')
    parser.add_argument('--comm-type', type=str, default='Bcast', metavar='N',
                        help='which kind of method we use during the mode fetching stage')
    parser.add_argument('--num-aggregate', type=int, default=5, metavar='N',
                        help='how many number of gradients we wish to gather at each iteration')
    parser.add_argument('--eval-freq', type=int, default=50, metavar='N',
                        help='it determines per how many step the model should be evaluated')
    parser.add_argument('--train-dir', type=str, default='output/models/', metavar='N',
                        help='directory to save the temp model during the training process for evaluation')
    parser.add_argument('--compress-grad', type=str, default='compress', metavar='N',
                        help='compress/none indicate if we compress the gradient matrix before communication')
    parser.add_argument('--enable-gpu', type=bool, default=False, help='whether to use gradient approx method')
    args = parser.parse_args()
    return args

if __name__ == "__main__":
    # this is only a simple test case
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
    world_size = comm.Get_size()

    args = add_fit_args(argparse.ArgumentParser(description='PyTorch MNIST Single Machine Test'))

    train_loader, test_loader = prepare_data(args)

    device = torch.device("cuda" if args.enable_gpu else "cpu")

    kwargs_master = {'batch_size':args.batch_size,
                'learning_rate':args.lr,
                'max_epochs':args.epochs,
                'momentum':args.momentum,
                'network':args.network,
                'comm_method':args.comm_type,
                'kill_threshold': args.num_aggregate,
                'timeout_threshold':args.kill_threshold,
                'eval_freq':args.eval_freq,
                'train_dir':args.train_dir,
                'max_steps':args.max_steps,
                'compress_grad':args.compress_grad,
                'device':device}

    kwargs_worker = {'batch_size':args.batch_size,
                'learning_rate':args.lr,
                'max_epochs':args.epochs,
                'momentum':args.momentum,
                'network':args.network,
                'comm_method':args.comm_type,
                'kill_threshold':args.kill_threshold,
                'eval_freq':args.eval_freq,
                'train_dir':args.train_dir,
                'max_steps':args.max_steps,
                'compress_grad':args.compress_grad,
                'device':device}

    if rank == 0:
        create_data_csv('ps_node_data.csv', 'ps')
        master_fc_nn = SyncReplicasMaster_NN(comm=comm, **kwargs_master)
        if args.dataset == 'Cifar100':
            master_fc_nn.build_model(num_classes=100)
        else:
            master_fc_nn.build_model(num_classes=10)
        print("I am the master: the world size is {}, cur step: {}".format(master_fc_nn.world_size, master_fc_nn.cur_step))
        master_fc_nn.start()
        print("Done sending messages to workers!")
    else:
        create_data_csv('worker_node_data.csv', 'worker')
        worker_fc_nn = DistributedWorker(comm=comm, **kwargs_worker)
        if args.dataset == 'Cifar100':
            worker_fc_nn.build_model(num_classes=100)
        else:
            worker_fc_nn.build_model(num_classes=10)
        print("I am worker: {} in all {} workers, next step: {}".format(worker_fc_nn.rank, worker_fc_nn.world_size-1, worker_fc_nn.next_step))
        worker_fc_nn.train(train_loader=train_loader, test_loader=test_loader)
        print("Now the next step is: {}".format(worker_fc_nn.next_step))


text:
import pandas as pd
import re
from os import system, listdir
from os.path import isfile, join
from random import shuffle
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from joblib import dump, load
from scipy.sparse import save_npz, load_npz
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import train_test_split
import numpy as np

'''
Make directories to store data and learning model for subsequent testing.
'''
# system("mkdir data_preprocessors")
# system("mkdir vectorized_data")
# system("mkdir csv")

def create_data_frame(folder):
    pos_folder = f'{folder}/pos'  # positive reviews
    neg_folder = f'{folder}/neg'  # negative reviews
    def get_files(folder):
        '''
        Get all files in the input folder
        '''
        return [join(folder, f) for f in listdir(folder) if isfile(join(folder, f))]

    def append_files_data(data_list, files, label):
        '''
        Appends to 'data_list' tuples of form (file content, label)
        for each file in 'files' input list
        '''
        for file in files:
            with open(file, 'r', encoding='utf-8') as f:
                text = f.read()
                data_list.append((text, label))

    pos_files = get_files(pos_folder)
    neg_files = get_files(neg_folder)

    data_list = []
    append_files_data(data_list, pos_files, 1)
    append_files_data(data_list, neg_files, 0)
    shuffle(data_list)

    text, label = tuple(zip(*data_list))
    # replacing line breaks with spaces
    text = list(map(lambda txt: re.sub('(<br\s*/?>)+', ' ', txt), text))

    return pd.DataFrame({'text': text, 'label': label})

# imdb_train = create_data_frame('aclImdb/train')
# imdb_test = create_data_frame('aclImdb/test')

# imdb_train.to_csv('csv/imdb_train.csv', index=False)
# imdb_test.to_csv('csv/imdb_test.csv', index=False)

imdb_train = pd.read_csv('csv/imdb_train.csv')
imdb_test = pd.read_csv('csv/imdb_test.csv')

# bigram_vectorizer = CountVectorizer(ngram_range=(1, 2))
# bigram_vectorizer.fit(imdb_train['text'].values)
# dump(bigram_vectorizer, 'data_preprocessors/bigram_vectorizer.joblib') # dump model so that we don't have to fit it again if we run the code again
bigram_vectorizer = load('data_preprocessors/bigram_vectorizer.joblib')

# X_train_bigram = bigram_vectorizer.transform(imdb_train['text'].values)
# save_npz('vectorized_data/X_train_bigram.npz', X_train_bigram) # dump matrix for future re-use
X_train_bigram = load_npz('vectorized_data/X_train_bigram.npz')

# bigram_tf_idf_transformer = TfidfTransformer()
# bigram_tf_idf_transformer.fit(X_train_bigram)
# dump(bigram_tf_idf_transformer, 'data_preprocessors/bigram_tf_idf_transformer.joblib')

bigram_tf_idf_transformer = load('data_preprocessors/bigram_tf_idf_transformer.joblib')

# X_train_bigram_tf_idf = bigram_tf_idf_transformer.transform(X_train_bigram)
# save_npz('vectorized_data/X_train_bigram_tf_idf.npz', X_train_bigram_tf_idf)
X_train_bigram_tf_idf = load_npz('vectorized_data/X_train_bigram_tf_idf.npz')

y_train = imdb_train['label'].values

# clf = SGDClassifier()
# clf.fit(X_train_bigram_tf_idf, y_train)
# system("mkdir classifiers")
# dump(clf, 'classifiers/sgd_classifier.joblib')
sgd_classifier = load('classifiers/sgd_classifier.joblib')

test_text = "Test classifier"
test = bigram_vectorizer.transform([test_text])
test = bigram_tf_idf_transformer.transform(test)

print(sgd_classifier.predict(test)[0])

reason: import numpy,from scipy


text:
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved

"""
Training sentiment prediction model on IMDB movie reviews dataset.
Architecture and reference results from https://arxiv.org/pdf/1911.11607.pdf
"""
import argparse

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from datasets import load_dataset
from opacus import PrivacyEngine
from torch.functional import F
from torch.nn.utils.rnn import pad_sequence
from torch.utils.data import DataLoader
from tqdm import tqdm
from transformers import BertTokenizerFast

class SampleNet(nn.Module):
    def __init__(self, vocab_size: int):
        super().__init__()
        self.emb = nn.Embedding(vocab_size, 16)
        self.pool = nn.AdaptiveAvgPool1d(1)
        self.fc1 = nn.Linear(16, 16)
        self.fc2 = nn.Linear(16, 2)

    def forward(self, x):
        x = self.emb(x)
        x = x.transpose(1, 2)
        x = self.pool(x).squeeze()
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        return x

    def name(self):
        return "SampleNet"

def binary_accuracy(preds, y):
    correct = (y.long() == torch.argmax(preds, dim=1)).float()
    acc = correct.sum() / len(correct)
    return acc

def padded_collate(batch, padding_idx=0):
    x = pad_sequence(
        [elem["input_ids"] for elem in batch],
        batch_first=True,
        padding_value=padding_idx,
    )
    y = torch.stack([elem["label"] for elem in batch]).long()
    return x, y

def train(args, model, train_loader, optimizer, epoch):
    criterion = nn.CrossEntropyLoss()
    losses = []
    accuracies = []
    device = torch.device(args.device)
    model = model.train().to(device)

    for data, label in tqdm(train_loader):
        data = data.to(device)
        label = label.to(device)

        optimizer.zero_grad()
        predictions = model(data).squeeze(1)
        loss = criterion(predictions, label)
        acc = binary_accuracy(predictions, label)

        loss.backward()
        optimizer.step()

        losses.append(loss.item())
        accuracies.append(acc.item())

    if not args.disable_dp:
        epsilon, best_alpha = optimizer.privacy_engine.get_privacy_spent(args.delta)
        print(
            f"Train Epoch: {epoch} \t"
            f"Train Loss: {np.mean(losses):.6f} "
            f"Train Accuracy: {np.mean(accuracies):.6f} "
            f"(ε = {epsilon:.2f}, δ = {args.delta}) for α = {best_alpha}"
        )
    else:
        print(
            f"Train Epoch: {epoch} \t Loss: {np.mean(losses):.6f} ] \t Accuracy: {np.mean(accuracies):.6f}"
        )

def evaluate(args, model, test_loader):
    criterion = nn.CrossEntropyLoss()
    losses = []
    accuracies = []
    device = torch.device(args.device)
    model = model.eval().to(device)

    with torch.no_grad():
        for data, label in tqdm(test_loader):
            data = data.to(device)
            label = label.to(device)

            predictions = model(data).squeeze(1)

            loss = criterion(predictions, label)
            acc = binary_accuracy(predictions, label)

            losses.append(loss.item())
            accuracies.append(acc.item())

    print(
        "\nTest set: Average loss: {:.4f}, Accuracy: {:.2f}%\n".format(
            np.mean(losses), np.mean(accuracies) * 100
        )
    )

def main():
    parser = argparse.ArgumentParser(description="PyTorch IMDB Example")
    parser.add_argument(
        "-b",
        "--batch-size",
        type=int,
        default=64,
        metavar="B",
        help="input batch size for training (default: 64)",
    )
    parser.add_argument(
        "-n",
        "--epochs",
        type=int,
        default=10,
        metavar="N",
        help="number of epochs to train (default: 10)",
    )
    parser.add_argument(
        "--lr",
        type=float,
        default=0.02,
        metavar="LR",
        help="learning rate (default: .02)",
    )
    parser.add_argument(
        "--sigma",
        type=float,
        default=0.56,
        metavar="S",
        help="Noise multiplier (default 0.56)",
    )
    parser.add_argument(
        "-c",
        "--max-per-sample-grad_norm",
        type=float,
        default=1.0,
        metavar="C",
        help="Clip per-sample gradients to this norm (default 1.0)",
    )
    parser.add_argument(
        "--delta",
        type=float,
        default=1e-5,
        metavar="D",
        help="Target delta (default: 1e-5)",
    )
    parser.add_argument(
        "--max-sequence-length",
        type=int,
        default=256,
        metavar="SL",
        help="Longer sequences will be cut to this length (default: 256)",
    )
    parser.add_argument(
        "--device",
        type=str,
        default="cuda",
        help="GPU ID for this process (default: 'cuda')",
    )
    parser.add_argument(
        "--save-model",
        action="store_true",
        default=False,
        help="Save the trained model (default: false)",
    )
    parser.add_argument(
        "--disable-dp",
        action="store_true",
        default=False,
        help="Disable privacy training and just train with vanilla optimizer",
    )
    parser.add_argument(
        "--secure-rng",
        action="store_true",
        default=False,
        help="Enable Secure RNG to have trustworthy privacy guarantees. Comes at a performance cost",
    )
    parser.add_argument(
        "--data-root", type=str, default="../imdb", help="Where IMDB is/will be stored"
    )
    parser.add_argument(
        "-j",
        "--workers",
        default=2,
        type=int,
        metavar="N",
        help="number of data loading workers (default: 2)",
    )

    args = parser.parse_args()
    device = torch.device(args.device)

    raw_dataset = load_dataset("imdb", cache_dir="imdb")
    tokenizer = BertTokenizerFast.from_pretrained("bert-base-cased")
    dataset = raw_dataset.map(
        lambda x: tokenizer(
            x["text"], truncation=True, max_length=args.max_sequence_length
        ),
        batched=True,
    )
    dataset.set_format(type="torch", columns=["input_ids", "label"])

    train_dataset = dataset["train"]
    test_dataset = dataset["test"]

    if args.secure_rng:
        try:
            import torchcsprng as prng
        except ImportError as e:
            msg = (
                "To use secure RNG, you must install the torchcsprng package! "
                "Check out the instructions here: https://github.com/pytorch/csprng#installation"
            )
            raise ImportError(msg) from e

        generator = prng.create_random_device_generator("/dev/urandom")

    else:
        generator = None

    train_loader = DataLoader(
        train_dataset,
        batch_size=args.batch_size,
        shuffle=True,
        num_workers=args.workers,
        drop_last=True,
        generator=generator,
        collate_fn=padded_collate,
        pin_memory=True,
    )

    test_loader = torch.utils.data.DataLoader(
        test_dataset,
        batch_size=args.batch_size,
        shuffle=False,
        num_workers=args.workers,
        collate_fn=padded_collate,
        pin_memory=True,
    )

    model = SampleNet(vocab_size=len(tokenizer)).to(device)
    optimizer = optim.Adam(model.parameters(), lr=args.lr)

    if not args.disable_dp:
        privacy_engine = PrivacyEngine(
            model,
            batch_size=args.batch_size,
            sample_size=len(train_dataset),
            alphas=[1 + x / 10.0 for x in range(1, 100)] + list(range(12, 64)),
            noise_multiplier=args.sigma,
            max_grad_norm=args.max_per_sample_grad_norm,
            secure_rng=args.secure_rng,
        )
        privacy_engine.attach(optimizer)

    for epoch in range(1, args.epochs + 1):
        train(args, model, train_loader, optimizer, epoch)
        evaluate(args, model, test_loader)

if __name__ == "__main__":
    main()

max_forks_repo_name: PenelopeCorsica/opacus


text:
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.13.1
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5" _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19"
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

import matplotlib.pyplot as plt
from datetime import datetime
import seaborn as sns

# %% [markdown]
# # **1. Read the Data**

# %%
Train_data = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2022/train.csv')
Test_data = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2022/test.csv')
train_data = Train_data.copy()
test_data = Test_data.copy()

# %% [markdown]
# # **2. Data Proprecessing**
#
# * I Split the 'date' column to 'Month' column and 'Day' column, I didn't consider the Year.
#
# * I define the 'weekday' and 'weekend' columns from 'date' columns.
#
# * I dummies the 'country', 'store', 'product' columns.

# %%
#Month
def split_month(date):
  return date.split('-')[1]

#Day
def split_day(date):
  return date.split('-')[2]

#Weekend
def weekend(date):
  import datetime
  weekend = []
  a = pd.to_datetime(date)
  for i in range(len(a)):
    if a.iloc[i].weekday() >= 5 :
      weekend.append(1)
    else:
      weekend.append(0)
  return weekend

#Weekday
def weekday(date):
    import datetime
    weekday = []
    a = pd.to_datetime(date)
    for i in range(len(a)):
        weekday.append(a.iloc[i].weekday())
    return weekday

# %%
train_data['Month'] = train_data['date'].apply(split_month)
train_data['Day'] = train_data['date'].apply(split_day)
train_data['Weekend'] = weekend(train_data['date'])
train_data['Weekday'] = weekday(train_data['date'])
train_data = train_data.drop(columns = ['row_id', 'date'])

test_data['Month'] = test_data['date'].apply(split_month)
test_data['Day'] = test_data['date'].apply(split_day)
test_data['Weekend'] = weekend(test_data['date'])
test_data['Weekday'] = weekday(test_data['date'])
test_data = test_data.drop(columns = ['row_id', 'date'])

# %%
#Dummies the 'country', 'store', 'product'
train_data_dum = pd.get_dummies(train_data[['country', 'store', 'product']])
test_data_dum = pd.get_dummies(test_data[['country', 'store', 'product']])

train_data = pd.concat([train_data, train_data_dum],axis = 1)
test_data = pd.concat([test_data, test_data_dum],axis = 1)

train_data = train_data.drop(columns = ['country', 'store', 'product'])
test_data = test_data.drop(columns = ['country', 'store', 'product'])

# %% [markdown]
# ### Define the training data and training target

# %%
data = train_data.drop(columns = 'num_sold')
target = train_data['num_sold']

from sklearn.preprocessing import StandardScaler
Normalize = StandardScaler()
target = np.log(target)

# %%
from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(data, target, train_size = 0.8, random_state = 5)
x_train = Normalize.fit_transform(x_train)
x_test = Normalize.transform(x_test)

# %% [markdown]
# # **3. Seven Model Performance**
#

# %% [markdown]
# ## 3-1. DecisionTreeRegressor

# %%
from sklearn.tree import DecisionTreeRegressor
sns.set()
DTR = DecisionTreeRegressor(max_depth = 12, min_samples_leaf = 2).fit(x_train, y_train)
y_pred_DTR = DTR.predict(x_test)
plt.scatter(y_test, y_pred_DTR)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.title('DecisionTreeRegressor')
plt.show()
plt.clf()

# %% [markdown]
# ## 3-2. RandomForestRegressor

# %%
from sklearn.ensemble import RandomForestRegressor
sns.set()
RFR = RandomForestRegressor(max_depth = 15, random_state = 2).fit(x_train, y_train)
y_pred_RFR = RFR.predict(x_test)
plt.scatter(y_test, y_pred_RFR)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.title('RandomFroestRegressor')
plt.show()
plt.clf()

# %% [markdown]
# ## 3-3. GradientBoostingRegressor

# %%
from sklearn.ensemble import GradientBoostingRegressor
sns.set()
GBR = GradientBoostingRegressor(learning_rate=0.10, max_depth= 6,
                                min_samples_leaf = 5,n_estimators = 500, random_state = 40,subsample = 0.3).fit(x_train, y_train)
y_pred_GBR = GBR.predict(x_test)
plt.scatter(y_test, y_pred_GBR)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.title('GradientBoostingRegressor')
plt.show()
plt.clf()
print(GBR.score(x_test, y_test))

# %% [markdown]
# ## 3-4. SVR rbf

# %%
from sklearn.svm import SVR
sns.set()
svr_rbf = SVR(kernel = 'rbf', gamma = 0.2 , C = 0.15, degree = 2, epsilon=0.1).fit(x_train, y_train)
y_pred_svr = svr_rbf.predict(x_test)
plt.scatter(y_test, y_pred_svr)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.title('SVM_RBF')
plt.show()
plt.clf()

# %% [markdown]
# ## 3-5. SVR Linear

# %%
from sklearn.svm import SVR
sns.set()
svr_linear = SVR(kernel = 'linear').fit(x_train, y_train)
y_pred_svr = svr_linear.predict(x_test)
plt.scatter(y_test, y_pred_svr)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.title('SVM_Linear')
plt.show()
plt.clf()

# %% [markdown]
# ## 3-6. Deep Learning

# %%
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout
data = Normalize.fit_transform(data)
model = Sequential()
model.add(Dense(512, input_shape = (data.shape[1], ), activation = 'sigmoid'))
model.add(Dense(64,activation = 'sigmoid'))
model.add(Dense(8))
model.add(Dense(1))
model.compile(loss = 'mse', optimizer = 'adam', metrics= 'mse')
history = model.fit(data, target, batch_size = 128, epochs = 100 , validation_split= 0.2, verbose = 1)

# %%
import seaborn as sns
sns.set()
df_history = pd.DataFrame(history.history)
sns.lineplot(x = df_history.index, y = df_history.loss)

# %%
y_pred = model.predict(data)
plt.scatter(target, y_pred)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.show()
plt.clf()

# %% [markdown]
# ## 3-7. KNeighborsRegressor

# %%
from sklearn.neighbors import KNeighborsRegressor
sns.set()
KNN = KNeighborsRegressor(n_neighbors = 3, weights = 'distance').fit(x_train, y_train)
y_pred_KNN = KNN.predict(x_test)
plt.scatter(y_test, y_pred_KNN)
plt.plot([x for x in range(4, 10)], [x for x in range(4, 10)], color = 'r')
plt.xlabel("Reality")
plt.ylabel("Predicted")
plt.title('KNeighborsRegressor')
plt.show()
plt.clf()

# %% [markdown]
# # **4. Six Model Comparsion(R2_score and Mean squared error)**

# %%
from sklearn.metrics import mean_squared_log_error
from sklearn.metrics import mean_squared_error
def model_fit(x_train, x_test, y_train, y_test):

  DTR = DecisionTreeRegressor(max_depth = 10,
                              min_samples_leaf = 8).fit(x_train, y_train)

  RFR = RandomForestRegressor(max_depth = 30).fit(x_train, y_train)

  GBR = GradientBoostingRegressor(learning_rate=0.10, max_depth= 6,
                                min_samples_leaf = 5,n_estimators = 500, random_state = 40,subsample = 0.3).fit(x_train, y_train)

  svr_rbf = SVR(kernel = 'rbf',
                gamma = 0.2 ,
                C = 0.15,
                degree = 2,
                epsilon=0.1).fit(x_train, y_train)

  svr_linear = SVR(kernel = 'linear').fit(x_train, y_train)

  KNN = KNeighborsRegressor(n_neighbors = 10).fit(x_train, y_train)

  return DTR, RFR, GBR, svr_rbf, svr_linear, KNN

# %% [markdown]
# ## 4-1. R square score Comparsion

# %%
Model = model_fit(x_train, x_test, y_train, y_test)
ML_model = ['DecisionTreeRegressor', 'RandomForestRegressor', 'GradientBoostingRegressor', 'svr_rbf', 'svr_linear', 'KNeighborsRegressor', 'DeepLearning']
sns.set()
from sklearn.metrics import r2_score
R_square_num = []
for i in range(6):
  R_square = r2_score(y_test, Model[i].predict(x_test))
  R_square_num.append(R_square)
R_square_num.append(r2_score(y_test, model.predict(x_test)))
plt.figure(figsize = (10, 10))
plt.xlabel('R Square Score')
plt.ylabel('Model Type')
plt.title('The R Square Score Comparsion')
sns.barplot(x = R_square_num, y = ML_model)

# %% [markdown]
# ## 4-2. Mean Square Root Comparsion

# %%
Model = model_fit(x_train, x_test, y_train, y_test)
ML_model = ['DecisionTreeRegressor', 'RandomForestRegressor', 'GradientBoostingRegressor', 'svr_rbf', 'svr_linear', 'KNeighborsRegressor', 'DeepLearning']
sns.set()
from sklearn.metrics import mean_squared_error
mse_num = []
for i in range(6):
  mse = mean_squared_error(y_test, Model[i].predict(x_test))
  mse_num.append(mse)
mse_num.append(mean_squared_error(y_test, model.predict(x_test)))
plt.figure(figsize = (10, 10))
plt.xlabel('mean_square_error')
plt.ylabel('Model Type')
plt.title('The mean_square_error Comparsion')
sns.barplot(x = mse_num, y = ML_model)

# %% [markdown]
# # **5. Predict the Test_data**

# %%
test_data = Normalize.transform(test_data)
submission_target = np.exp(Model[2].predict(test_data))

# %% [markdown]
# # **6. Submission**

# %%
submission = pd.read_csv('/kaggle/input/tabular-playground-series-jan-2022/sample_submission.csv')
submission['num_sold'] = submission_target
submission.to_csv('submission.csv', index=False)

# %%
submission

# %% [markdown]
# # **7. Thanks for your view!**


text:
import numpy as np

def green(x, y, x0, y0, w=1.0):
    """
    Greens function to Laplaces equation in semi-infinite strip, with reflective
    walls and absorbing floor.

    Parameters
    ----------
    x : float
        x coordinate of sample point
    y : float
        y coordinate of sample point
    x0 : float
        x coordinate of point source
    y0 : float
        y coordinate of point source
    w : float, default 1.0
        width of the strip

    Returns
    -------
    float
        value of Greens function at loction (x,y) with source located at (x0,y0).

    """

    return 0.5 * np.log(
        (
            (np.cos(np.pi * (x - x0) / w) - np.cosh(np.pi * (y - y0) / w))
            * (np.cos(np.pi * (x + x0) / w) - np.cosh(np.pi * (y - y0) / w))
        )
        / (
            (np.cos(np.pi * (x - x0) / w) - np.cosh(np.pi * (y + y0) / w))
            * (np.cos(np.pi * (x + x0) / w) - np.cosh(np.pi * (y + y0) / w))
        )
    )

def nonsingular_part_green(x, y, w=1.0):
    """
    Nonsingular part of the Greens function evaluated at the location of the
    source. :math:`\lim_{(x,y)\to(x_0,y_0)}G(x,y,x_0,y_0) - 1/2 \log((x-x_0)^2+(y-y_0)^2)`.
    In other words contributions from reflected sources only evaluated at the
    original source.

    Parameters
    ----------
    x : float
        x coordinate of sample point
    y : float
        y coordinate of sample point
    w : float, default 1.0
        width of the strip

    Returns
    -------
    float
        value of the nonsingular component of the Greens function at source.

    """

    return 0.5 * np.log(
        (np.pi ** 2 * np.sin(np.pi * x / w) ** 2)
        / (
            2
            * w ** 2
            * np.sinh(np.pi * y / w) ** 2
            * (np.cosh(2 * np.pi * y / w) - np.cos(2 * np.pi * x / w))
        )
    )

def singular_part_green(mesh_size):
    """
    Singular asymptotic of Greens function evaluated at the location of the
    source. If the source was smeared on line element of length `mesh_size` then
    this would be the value of the field in the middle of the line segment.

    Parameters
    ----------
    mesh_size : float
        size of the smear line segment

    Returns
    -------
    float
        value of the regularized Greens function evaluated at the source.

    """

    return 0.5 * (-2 + np.log(mesh_size ** 2 / 4))

max_stars_repo_name: RadostW/patyczki

max_issues_repo_name: RadostW/patyczki

max_forks_repo_name: RadostW/patyczki


text:
# -*- coding: utf-8 -*-

# Copyright 2019 Resonai Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
utilities for tests
~~~~~~~~~~~~~~~~~~~~~~~

:author: Dana Shamir
"""
import re
import random
import string
import networkx as nx
from datetime import datetime

def generate_random_dag(nodes, min_rank=0, max_rank=10, edge_prob=0.3):
    """Return a random DAG with nodes from `nodes`.

    and edges can only go from nodes[i] -> nodes[j] if i < j
    (guaranteeing DAGness).
    """
    g = nx.DiGraph()
    g.add_nodes_from(nodes)
    for j in range(1, len(nodes)):
        rank = random.randint(min_rank, min(j, max_rank))
        g.add_edges_from((nodes[i], nodes[j])
                         for i in random.sample(range(j), k=rank)
                         if random.random() > edge_prob)
    return g

class CBuildTrgtTest:
    """Class for creating a random package
        with projects and their dependencies.

        The package contains several projects. All projects consist of files.
        Single-file projects are also allowed. Some projects may be
        independent of others, some should depend on others, such as
        an installation script. Not all dependencies are valid, for example,
        the library cannot depend on the executable file. All applicable rules
        and dependencies were originally read from a json file. Then I
        hard-coded the contents of the most successful file into sort_test_cfg.
            The CBuildTrgtType class contains the name and properties of the
        project type. In the constructor of the CBuildTrgtTest class, all
        project types are written to the trgs variable and configuration data
        to the min_name_len, max_name_len, steps, max_step_nodes
        variables. File names will be randomly generated from
        min_name_len to max_name_len in length.
            A random package is created in the create_rand_graph function.
        In the loop, we call the __add_nodes function steps times.
        The __add_nodes function adds a maximum of MaxStepNodes
        files. Each file is added along with its random dependencies in
        accordance with the rules from the trgs table. Since files without
        parents are added at each step, the cycle cannot be formed
        on a common graph. Thus, the create_rand_graph function
        always returns a Directed Acyclic Graph (DAG)
    """
    sort_test_cfg = [
        {
            "BuildTargets": [
                {
                    "Name": "AptPackage", "Childs": ["AptPackage"],
                    "MinChilds": 0
                },
                {
                    "Name": "CppGTest", "Childs": ["CppLib", "CppProg"],
                    "MinChilds": 1
                },
                {
                    "Name": "CppLib", "Childs": ["Proto", "AptPackage"],
                    "MinChilds": 1
                },
                {
                    "Name": "CppProg",
                    "Childs": ["CppLib", "Proto", "AptPackage"],
                    "MinChilds": 1
                },
                {
                    "Name": "CustomInstaller", "Childs":
                    [
                        "CppLib", "Proto", "CppProg", "CppGTest",
                        "Python", "PythonPackage", "PythonTest"
                    ], "MinChilds": 3
                },
                {
                    "Name": "Proto", "Childs": ["Proto"], "MinChilds": 0
                },
                {
                    "Name": "Python", "Childs": ["Python", "PythonPackage"],
                    "MinChilds": 0
                },
                {
                    "Name": "PythonPackage", "Childs": ["PythonPackage"],
                    "MinChilds": 0
                },
                {
                    "Name": "PythonTest",
                    "Childs": ["PythonPackage", "Python"],
                    "MinChilds": 2
                }
            ]
        },
        {
            "Config": {
                "MinNameLen": 2, "MaxNameLen": 5,
                "Steps": 100, "MaxStepNodes": 8
            }
        }
    ]

    class CBuildTrgtType:
        """ Type of project"""
        def __init__(self, trg):
            self.name = trg['Name']
            self.childs = trg['Childs']  # Types of available dependencies
            self.min_childs = trg['MinChilds']  # Minimum of dependencies

        def is_child(self, trg):
            return trg.name in self.childs

    def __init__(self):
        bd_trgs = CBuildTrgtTest.sort_test_cfg[0]['BuildTargets']
        self.trgs = list()
        for trg in bd_trgs:
            self.trgs.append(CBuildTrgtTest.CBuildTrgtType(trg))
        cfg = CBuildTrgtTest.sort_test_cfg[1]['Config']
        self.min_name_len = cfg['MinNameLen']
        self.max_name_len = cfg['MaxNameLen']
        self.steps = cfg['Steps']
        self.max_step_nodes = cfg['MaxStepNodes']
        self.letters = string.ascii_lowercase

    def __new_build_trgt(self):
        """Generate a random string """
        str_len = random.randint(self.min_name_len, self.max_name_len)
        name = ''.join(random.choice(self.letters) for i in range(str_len))
        i = random.randint(0, len(self.trgs) - 1)
        return [name, self.trgs[i]]

    def __add_nodes(self, graph):
        """ Added random nodes with its random dependencies"""
        edges = dict()
        nod_num = random.randint(1, self.max_step_nodes)
        for _ in range(nod_num):
            trg = self.__new_build_trgt()
            edges.update({trg[0]: trg[1]})
        nodes = list(graph)
        trg_attrs = nx.get_node_attributes(graph, 'trg')
        nodes_len = len(nodes)
        for name in edges:
            if name not in graph:
                trg = edges[name]
                childs = 0
                if nodes_len > 0:
                    child_num = random.randint(0, nodes_len - 1)
                    for i in range(0, child_num):
                        child_name = nodes[i]
                        child_trg = trg_attrs[child_name]
                        if trg.is_child(child_trg):
                            graph.add_edge(name, child_name)
                            childs = childs + 1
                if trg.min_childs <= childs:
                    graph.add_node(name, trg=trg)
                elif childs > 0:
                    graph.remove_node(name)

    def create_rand_graph(self):
        """ Created string random DAG """
        graph = nx.DiGraph()
        for _ in range(self.steps):
            self.__add_nodes(graph)
        return graph

def write_test_dot(graph, out_fname=''):
    """Write build graph in dot format to `out_f` file-like object."""
    if not out_fname:
        now = datetime.now()
        out_fname = now.strftime('ybt_%2d_%2m_%Y_%2H_%2M_%2S_%f.dot')
    fout = open(out_fname, 'w')
    fout.write('strict digraph    {\n')
    clrs = nx.get_node_attributes(graph, 'color')
    stls = nx.get_node_attributes(graph, 'style')
    fclrs = nx.get_node_attributes(graph, 'fillcolor')
    for node in graph.nodes():
        s = '['
        if node in clrs:
            s = s + 'color=' + str(clrs[node]) + ','
        if node in stls:
            s = s + 'color=' + str(stls[node]) + ','
        if node in fclrs:
            s = s + 'color=' + str(fclrs[node]) + ','
        fout.write('  {} {}];\n'.format(node, s))
    fout.writelines('    {} -> {};\n'.format(u, v)
                    for u, v in graph.edges())
    fout.write('}\n\n')
    fout.close()
    print('Output file is "{}"'.format(out_fname))
    return out_fname

def dot_file_line_proc(graph, line):
    """ Parsing dot file line """
    edge = re.split(r'\s*->\s*', line)
    if len(edge) == 2:
        name = re.sub(r'\s+|;', '', edge[0]).strip('\"')
        child_name = re.sub(r'\s+|;', '', edge[1]).strip('\"')
        graph.add_edge(name, child_name)
    else:
        temp = re.split(r'\s+', line, 2)
        if (not temp[0]) & bool(temp[1]):
            name = temp[1].strip('\"')
            attributes = dict()
            for attr_str in re.split(r'\[|,|\]|;|\n', temp[2]):
                attr = re.split('=', attr_str)
                if len(attr) == 2:
                    attributes.update({
                        attr[0].strip('\"'): attr[1].strip('\"')})
            graph.add_node(name, **attributes)

def load_dot(in_fname):
    """ Read graph from dot format """
    with open(in_fname, 'r') as in_f:
        graph = nx.DiGraph()
        for line in in_f:
            dot_file_line_proc(graph, line)
    return graph

max_stars_repo_name: resonai/ybt

max_issues_repo_name: resonai/ybt

max_forks_repo_name: resonai/ybt

reason: import networkx


text:
from itertools import product
import numpy as np
import nose.tools as nt

from ..convenience import lasso, step, threshold
from ..query import optimization_sampler
from ...tests.instance import (gaussian_instance,
                               logistic_instance,
                               poisson_instance)
from ...tests.flags import SMALL_SAMPLES
from ...tests.decorators import set_sampling_params_iftrue, set_seed_iftrue

from scipy.stats import t as tdist
from ..glm import target as glm_target, glm_nonparametric_bootstrap, pairs_bootstrap_glm
from ..M_estimator import restricted_Mest

@set_seed_iftrue(True, 200)
@set_sampling_params_iftrue(SMALL_SAMPLES, ndraw=1000, burnin=100)
def test_opt_weighted_intervals(ndraw=20000, burnin=2000):

    cls = lasso
    for const_info, rand in product(zip([gaussian_instance], [cls.gaussian]), ['laplace']):

        inst, const = const_info

        X, Y = inst(n=100, p=10, s=0)[:2]
        n, p = X.shape

        W = np.ones(X.shape[1]) * 1
        conv = const(X, Y, W, randomizer=rand)
        signs = conv.fit()
        print("signs", signs)

        #marginalizing_groups = np.zeros(p, np.bool)
        #marginalizing_groups[:int(p/2)] = True
        #conditioning_groups = ~marginalizing_groups
        #conditioning_groups[-int(p/4):] = False

        selected_features = conv._view.selection_variable['variables']

        #conv.summary(selected_features,
        #             ndraw=ndraw,
        #             burnin=burnin,
        #             compute_intervals=True)

        #conv.decompose_subgradient(marginalizing_groups=marginalizing_groups,
        #                           conditioning_groups=conditioning_groups)

        conv._queries.setup_sampler(form_covariances=None)
        conv._queries.setup_opt_state()
        opt_sampler = optimization_sampler(conv._queries)

        S = opt_sampler.sample(ndraw,
                               burnin,
                               stepsize=1.e-3)
        #print(S.shape)
        #print([np.mean(S[:,i]) for i in range(p)])

        unpenalized_mle = restricted_Mest(conv.loglike, selected_features)
        form_covariances = glm_nonparametric_bootstrap(n, n)
        conv._queries.setup_sampler(form_covariances)
        boot_target, boot_target_observed = pairs_bootstrap_glm(conv.loglike, selected_features, inactive=None)
        opt_sampler.setup_target(boot_target,
                                 form_covariances)

        selective_CI = opt_sampler.confidence_intervals(unpenalized_mle, sample=S)
        print(selective_CI)

        return selective_CI

max_stars_repo_name: nanbi/Python-software

max_issues_repo_name: nanbi/Python-software

max_forks_repo_name: nanbi/Python-software


text:
import copy
import functools
import json
import logging
import numpy as np
import os
import random
from sacred import Experiment
import subprocess
import sys
import time
import torch
import torch.nn as nn
import torch.optim as optim
import torch.optim.lr_scheduler as lr_scheduler
import torchvision.transforms as transforms
import torchvision.utils as tvutils
from torchsummary import summary
from tqdm import tqdm as tqdm

from tlkit.get_reprs import need_to_save
from evkit.models.taskonomy_network import TaskonomyDecoder, TaskonomyNetwork
from tlkit.models.student_models import AtariStudentNet, AtariStudentSmallNet
from tlkit.utils import update, var_to_numpy, save_checkpoint, log, reset_log
import tlkit.utils
from tlkit.data.taskonomy_dataset import get_dataloaders
import tlkit.data.splits as splits
from tlkit.utils import LIST_OF_TASKS, TASKS_TO_CHANNELS

from evkit.saving.observers import FileStorageObserverWithExUuid
import evkit.saving.checkpoints as checkpoints
from evkit.utils.profiler import Profiler
from evkit.utils.random import set_seed

from evkit.models.taskonomy_network import TaskonomyEncoder

# This has been giving me a lot of headaches... just try both
try:
    import tnt.torchnet as tnt
    from tnt.torchnet.logger import FileLogger
except ModuleNotFoundError:
    import torchnet as tnt
    from torchnet.logger import FileLogger

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

logging.basicConfig(level=logging.INFO, format='%(message)s')
logger = logging.getLogger()

ex = Experiment(name="Student distillation")
LOG_DIR = sys.argv[1].strip()
sys.argv.pop(1)

def get_logger(cfg, uuid):
    if cfg['saving']['logging_type'] == 'visdom':
        mlog = tnt.logger.VisdomMeterLogger(
            title=uuid, env=uuid, server=cfg['saving']['visdom_server'],
            port=cfg['saving']['visdom_port'],
            log_to_filename=cfg['saving']['visdom_log_file']
        )
    elif cfg['saving']['logging_type'] == 'tensorboard':
        mlog = tnt.logger.TensorboardMeterLogger(
            env=uuid,
            log_dir=cfg['saving']['log_dir'],
            plotstylecombined=True,
            train_only=cfg['training']['train_only']
        )
    else:
        assert False, 'no proper logger!'
    return mlog

# This is a cool bit of code--but seems like a recipe for bugs
# def load_experiment_config(experiment_path):
#     experiment_metadata_path = [os.path.join(experiment_path, f) for f in os.listdir(experiment_path) if f.endswith('metadata')][0]
#     experiment_config_path = os.path.join(experiment_metadata_path, 'config.json')
#     with open(experiment_config_path) as f:
#         config = json.load(f)  # keys are ['cfg', 'uuid', 'seed']
#     cfg = update(cfg, config['cfg'])
#     uuid = config['uuid']
#     return cfg, uuid

def maybe_bake_decodings(cfg,logger):
    task = cfg['training']['taskonomy_encoder']
    need_encodings = (cfg['training']['baked_encoding'] \
                   and not os.path.isdir(os.path.join(cfg['training']['data_dir'], f'{task}_encoding')))
    need_decodings = (cfg['training']['baked_decoding'] \
                   and not os.path.isdir(os.path.join(cfg['training']['data_dir'], f'{task}_decoding')))

    split_to_use = eval(cfg['training']['split_to_use'])
    folders_to_convert=set(split_to_use['train'] + split_to_use['val'] + split_to_use['test'])
    if not (need_decodings or need_encodings or need_to_save(
        task=task,
        folders_to_convert=folders_to_convert,
        data_dir=cfg['training']['data_dir'],
        save_dir=cfg['training']['data_dir'],
        store_representation=cfg['training']['baked_encoding'],
        store_prediction=cfg['training']['baked_decoding'],
    )):
        return

    logger.info(f"Requiring at least one of baked encodings ({need_encodings}) or decodings ({need_decodings}). Baking...")
    from tlkit.get_reprs import save_reprs as base_decodings
    base_decodings(cfg['training']['taskonomy_encoder'],
            model_base_path=cfg['training']['encoding_base_path'],
            folders_to_convert=folders_to_convert,
            split_to_convert=None,
            data_dir=cfg['training']['data_dir'],
            save_dir=cfg['training']['data_dir'],
            store_representation=cfg['training']['baked_encoding'],
            store_prediction=cfg['training']['baked_decoding'],
            n_dataloader_workers=cfg['training']['num_workers'],
            batch_size=cfg['training']['batch_size_val'],
            skip_done_folders=True)

@ex.main
def train(cfg, uuid):

    logger.setLevel(logging.INFO)
    logger.info(cfg)
    logger.debug(f'Loaded Torch version: {torch.__version__}')
    logger.debug(f'Using device: {device}')

    task = cfg['training']['taskonomy_encoder']
    start_epoch = 0

    logger.debug(f'Starting data loaders')
    maybe_bake_decodings(cfg, logger)
    set_seed(cfg['training']['seed'])
    data_subfolders = ['rgb']
    if cfg['training']['baked_encoding']:
        data_subfolders.append(f'{task}_encoding')
    if cfg['training']['baked_decoding']:
        data_subfolders.append(f'{task}_decoding')
    dataloaders = get_dataloaders(cfg['training']['data_dir'],
                    data_subfolders,
                    batch_size=cfg['training']['batch_size'],
                    batch_size_val=cfg['training']['batch_size_val'],
                    zip_file_name=False,
                    train_folders=eval(cfg['training']['split_to_use'])['train'],
                    val_folders=eval(cfg['training']['split_to_use'])['val'],
                    test_folders=eval(cfg['training']['split_to_use'])['test'],
                    num_workers=cfg['training']['num_workers'],
                    load_to_mem=cfg['training']['load_to_mem'],
                    pin_memory=cfg['training']['pin_memory'])

    logger.debug(f'Setting up student model')
    set_seed(cfg['training']['seed'])
    student = eval(cfg['learner']['model'])(**cfg['learner']['model_kwargs'])
    if cfg['training']['resume_from_checkpoint_path'] is not None:
        ckpt_fpath = cfg['training']['resume_from_checkpoint_path']
        checkpoint = torch.load(ckpt_fpath)
        student.load_state_dict(checkpoint['state_dict'])
        start_epoch = checkpoint['epoch'] if 'epoch' in checkpoint else 0
        logger.info(f"Loaded student (epoch {start_epoch if 'epoch' in checkpoint else 'unknown'}) from {ckpt_fpath}")
    student.to(device)

    logger.debug(f'Setting up teacher')
    set_seed(cfg['training']['seed'])
    out_channels = TASKS_TO_CHANNELS[task] if task in TASKS_TO_CHANNELS else None
    teacher = TaskonomyNetwork(out_channels=out_channels)
    if cfg['training']['baked_encoding']:
        teacher.encoder = None
    else:
        teacher.load_encoder(os.path.join(cfg['training']['encoding_base_path'], f'{task}_encoder.dat'))
    if cfg['training']['baked_decoding'] and not (cfg['training']['loss_type'].upper() == 'PERCEPTION'):
        teacher.decoder = None  # No need for this
    else:
        assert out_channels is not None, f"Decoder needed for config, but unknown decoder format for task {task}."
        teacher.load_decoder(os.path.join(cfg['training']['encoding_base_path'], f'{task}_decoder.dat'))
    teacher.eval()
    teacher.to(device)

    if torch.cuda.device_count() > 1:
        logger.info(f"Using {torch.cuda.device_count()} GPUs!")
        teacher.encoder = torch.nn.DataParallel(teacher.encoder)
        teacher.decoder = torch.nn.DataParallel(teacher.decoder)
        student = torch.nn.DataParallel(student)

    if cfg['training']['loss_fn'] == 'L2':
        loss_fn = nn.MSELoss()
    elif cfg['training']['loss_fn'] == 'L1':
        loss_fn = nn.L1Loss()
    else:
        logger.warning('Using default L2/MSE loss')
        loss_fn = nn.MSELoss()

    flog = tnt.logger.FileLogger(cfg['saving']['results_log_file'], overwrite=True)
    mlog = get_logger(cfg, uuid)
    mlog.add_meter('decoded_image', tnt.meter.ValueSummaryMeter(), ptype='image')

    if cfg['training']['train']:
        mlog.add_meter('teacher_histogram', tnt.meter.ValueSummaryMeter(), ptype='histogram')
        mlog.add_meter('student_histogram', tnt.meter.ValueSummaryMeter(), ptype='histogram')
        mlog.add_meter('loss', tnt.meter.ValueSummaryMeter())
        mlog.add_meter('image', tnt.meter.ValueSummaryMeter(), ptype='image')
        optimizer = optim.Adam(student.parameters(), lr=cfg['learner']['lr'])
        scheduler = None
        if cfg['learner']['lr_scheduler_method'] is not None:
            scheduler = eval(cfg['learner']['lr_scheduler_method'])(optimizer, **cfg['learner']['lr_scheduler_method_kwargs'])
        logger.info("Starting training...")
        train_model(cfg, student, teacher, dataloaders, loss_fn, optimizer, start_epoch=start_epoch,
                                      num_epochs=cfg['training']['num_epochs'], save_epochs=cfg['saving']['save_interval'],
                                      scheduler=scheduler, mlog=mlog, flog=flog)

    if cfg['training']['test']:
        NotImplementedError()

def train_model(cfg, student, teacher, dataloaders, loss_fn, optimizer, start_epoch=0, num_epochs=250, save_epochs=25, scheduler=None, mlog=None, flog=None):
    checkpoint_dir = os.path.join(cfg['saving']['log_dir'], cfg['saving']['save_dir'])
    run_kwargs = {
        'baked_encoding': cfg['training']['baked_encoding'],
        'baked_decoding': cfg['training']['baked_decoding'],
        'mlog': mlog,
        'flog': flog,
        'optimizer': optimizer,
        'loss_type': cfg['training']['loss_type'],
        'loss_fn': loss_fn,
        'student':student,
        'teacher': teacher,
        'decoder': teacher.decoder,
        'cfg': cfg
    }

    loss = 99999
    for epoch in range(start_epoch, start_epoch + num_epochs):
        if epoch % save_epochs == save_epochs - 1 or epoch == 0:
            checkpoints.save_checkpoint({
                'state_dict': student.state_dict(),
                'epoch': epoch
            }, directory=checkpoint_dir, step_num=epoch)

        _ = run_one_epoch(dataloader=dataloaders['train'], epoch=epoch, train=True, **run_kwargs)

        if scheduler is not None:
            try:
                scheduler.step(loss)
            except:
                scheduler.step()

        loss = run_one_epoch(dataloader=dataloaders['val'], epoch=epoch, train=False, **run_kwargs)
    checkpoints.save_checkpoint({
        'state_dict': student.state_dict(),
        'epoch': epoch
        }, directory=checkpoint_dir, step_num=epoch)
    return student

def run_one_epoch(student, teacher, decoder, dataloader, loss_fn, loss_type, optimizer, epoch, baked_encoding, baked_decoding, mlog, flog, train, cfg):
    student.train(train)
    phase = 'train' if train else 'val'
    with torch.set_grad_enabled(train), Profiler(f"Epoch {epoch} ({'train' if train else 'val'})", logger) as prof:
        running_loss = 0.0
        for batch_tuple in tqdm(dataloader):

            # Decompose batch
            batch_tuple = [x.to(device, non_blocking=True) for x in batch_tuple]
            x = batch_tuple[0]
            if baked_encoding:
                encoding_label = batch_tuple[1]
            if baked_decoding:
                decoding_label = batch_tuple[-1]

            if train:
                optimizer.zero_grad()

            student_encoding = student(x)
            if loss_type.upper() == 'PERCEPTION':
                if not baked_decoding:
                    encoding_label = teacher.encoder(x)
                if not baked_decoding:
                    # logger.warning("Decoder is being used for forward pass. This is valid, though you may see speedups from using pre-baked representations")
                    with torch.no_grad():
                        decoding_label = decoder(encoding_label)
                prediction, label = decoder(student_encoding), decoding_label
            else:
                prediction, label = student_encoding, encoding_label

            loss = loss_fn(prediction, label)

            # backward + optimize only if in training phase
            if train:
                loss.backward()
                if cfg['learner']['max_grad_norm'] is not None:
                    torch.nn.utils.clip_grad_norm_(student.parameters(), cfg['learner']['max_grad_norm'])
                optimizer.step()
            mlog.update_meter(loss.detach().item(), meters={'loss'}, phase=phase)

    if decoder is not None: # outputs images
        if len(label.shape) == 4 and label.shape[1] == 2:
            zeros = torch.zeros(label.shape[0], 1, label.shape[2], label.shape[3]).to(label.device)
            label = torch.cat([label, zeros], dim=1)
            prediction = torch.cat([prediction, zeros], dim=1)
        im_samples = torch.cat([x, prediction.expand_as(x), label.expand_as(x)], dim=3)
        im_samples = tvutils.make_grid(im_samples.detach().cpu(), nrow=1, padding=2)
        log(mlog, f'decoded_image', im_samples, phase=phase)
    logs = mlog.peek_meter(phase=phase)
    logger.info(phase + ' loss: {0:.6f}'.format(logs['loss'].item()))
    tlkit.utils.log(mlog, f'image', var_to_numpy(x[0]), phase=phase)
    tlkit.utils.reset_log(mlog, flog=flog, epoch=epoch, phase=phase)
    return logs['loss'].item()

@ex.config
def cfg_base():
    uuid = 'basic'
    cfg = {}
    cfg['learner'] = {
        'model': 'atari_residual',
        'model_kwargs' : {},
        'eps': 1e-5,                 # Small epsilon to prevent divide-by-zero
        'lr': 1e-3,                  # Learning rate for algorithm
        'lr_scheduler_method': None,
        'lr_scheduler_method_kwargs': {},
        'max_grad_norm': 1,        # Clip grads
        'test':False,
        'scheduler': 'plateau',     # Automatically reduce LR once loss plateaus (plateau, step)
    }
    cfg['training'] = {
        'baked_encoding': False,
        'baked_decoding': True,
        'batch_size': 64,
        'batch_size_val': 64,
        'cuda': True,
        'data_dir': '/mnt/hdd2/taskonomy_reps', # for docker
        'epochs': 100,
        'encoding_base_path': '/root/tlkit/taskonomy/taskbank/pytorch',
        'loss_fn': 'L1',
        'loss_type': 'perception',  # If distance in encoding space, use ''. If distance in decoding space, use 'PERCEPTION'
        'load_to_mem': False,  # if dataset small enough, can load activations to memory
        'num_workers': 8,
        'num_epochs': 1,
        'pin_memory': True,
        'resume_from_checkpoint_path': None,
        'seed': random.randint(0,1000),
        'split_to_use': 'splits.taskonomy_no_midlevel["debug"]',
        'taskonomy_encoder': 'autoencoding',
        'train': True,
        'train_only': False,
        'test': False,
    }
    cfg['saving'] = {
        'log_dir': LOG_DIR,
        'log_interval': 1,
        'logging_type': 'tensorboard',
        'results_log_file': os.path.join(LOG_DIR, 'result_log.pkl'),
        'reward_log_file': os.path.join(LOG_DIR, 'rewards.pkl'),
        'save_interval': 1,
        'save_dir': 'checkpoints',
        'visdom_log_file': os.path.join(LOG_DIR, 'visdom_logs.json'),
        'visdom_server': 'r2d2.eecs.berkeley.edu',
        'visdom_port': '8097',
    }

##################
# Students
##################
@ex.named_config
def model_fcn5():
    cfg = {'learner': {
        'model': 'FCN5Residual',
        'model_kwargs': {
            'num_groups': 2,
            'use_residual': False,
            'normalize_output': False,
        } } }

@ex.named_config
def model_fcn5_residual():
    cfg = {'learner': {
        'model': 'FCN5Residual',
        'model_kwargs': {
            'num_groups': 2,
            'use_residual': True,
            'normalize_output': False,
        } } }

@ex.named_config
def model_fcn3():
    cfg = { 'learner': {
        'model': 'FCN3',
        'model_kwargs': {
            'num_groups': 2,
            'normalize_output': False,
        } } }

@ex.named_config
def student_taskonomy_encoder_penultimate():
    cfg = { 'learner': {
            'model': 'TaskonomyEncoder',
            'model_kwargs': {
                'train': True,
                'eval_only': False,
          } } }

@ex.named_config
def student_taskonomy_encoder():
    cfg = { 'learner': {
            'model': 'TaskonomyEncoder',
            'model_kwargs': {
                'train_penultimate': True,
                'eval_only': False,
          } } }

##################
# Learning Rates
##################
@ex.named_config
def scheduler_reduce_on_plateau():
    cfg = { 'learner': {
            'lr_scheduler_method': 'lr_scheduler.ReduceLROnPlateau',
            'lr_scheduler_method_kwargs': {
                'factor': 0.1,
                'patience': 5
          } } }

@ex.named_config
def scheduler_step_lr():
    cfg = { 'learner': {
            'lr_scheduler_method': 'lr_scheduler.StepLR',
            'lr_scheduler_method_kwargs': {
                'lr_decay_epochs': 30,       # number of epochs before the LR drops by factor of 10
                'gamma': 0.1
          } } }

@ex.named_config
def cfg_eval():
    uuid = 'eval'
    cfg = {}
    cfg['learner'] = {
        'model': 'FCN5',
        'test': True,
    }
    cfg['training'] = {
        'train': False,
    }

if __name__ == '__main__':
    assert LOG_DIR, 'log dir cannot be empty'
    os.makedirs(LOG_DIR, exist_ok=True)
    # subprocess.call("rm -rf {}/*".format(LOG_DIR), shell=True)  # need this to clean
    ex.observers.append(FileStorageObserverWithExUuid.create(LOG_DIR))
    ex.run_commandline()
else:
    print(__name__)


text:
import numpy as np
import tensorflow as tf
import collections
from ..model import average_grad
from ..misc import delete_and_make_dir, create_op_graph
from ..tftrain import load_ckpt, load_ckpt_path, create_init_op
from ..metrics import get_metric_name
from tqdm import trange

__all__ = ['ENASModelTensors', 'ENASModel']

ENASModelTensors = collections.namedtuple('ENASModelTensors',
                                          ('inputs',
                                           'labels',
                                           'is_training',
                                           'is_fixed_arc',
                                           'fixed_arc',
                                           'child',
                                           'controller',
                                           'train_input_func',
                                           'reward_input_func',
                                           'reward_metric_func',
                                           'reward_batches',
                                           'child_loss_func',
                                           'child_optimizer',
                                           'controller_optimizer'))

class ENASModel:
    def __init__(self, enas_tensors: ENASModelTensors, n_gpu, ckpt_dir=None, training=True, one_shot_reward=False):
        self.inputs = enas_tensors.inputs
        self.labels = enas_tensors.labels
        self.is_training = enas_tensors.is_training
        self.is_fixed_arc = enas_tensors.is_fixed_arc
        self.fixed_arc = enas_tensors.fixed_arc
        self.child = enas_tensors.child
        self.controller = enas_tensors.controller
        self.train_input_func = enas_tensors.train_input_func
        self.reward_input_func = enas_tensors.reward_input_func
        self.reward_metric_func = enas_tensors.reward_metric_func
        self.reward_batches = enas_tensors.reward_batches
        self.one_shot_reward = one_shot_reward
        self.child_loss_func = enas_tensors.child_loss_func
        self.child_optimizer = enas_tensors.child_optimizer
        self.controller_optimizer = enas_tensors.controller_optimizer

        self.n_gpu = n_gpu
        self.ckpt_dir = ckpt_dir
        self.sess = None
        self.model_loaded = False
        self.graph_def = None

        self.child.connect_controller(self.controller)
        with tf.name_scope('main'):
            self.controller()
            self.child_logits = self.child(self.inputs, self.is_training, self.is_fixed_arc, self.fixed_arc)
        self.progress_range = trange
        if training:
            with tf.name_scope('training'):
                self._build_training_graph()

    def load_weights(self, sess, ckpt_path=None):
        self.sess = sess
        if ckpt_path is None:
            load_ckpt(sess, self.ckpt_dir)
        else:
            load_ckpt_path(sess, ckpt_path)
        self.model_loaded = True
        self.graph_def = sess.graph_def

    def _build_training_graph(self):
        with tf.device('/cpu:0'):
            tower_grads = []
            child_losses = []
            for i in range(self.n_gpu):
                with tf.device(f'/gpu:{i}'):
                    with tf.name_scope(f'GPU_{i}'):
                        (x, y), _ = self.train_input_func()
                        child_logits = self.child(x, tf.constant(True), tf.constant(False), tf.constant([0]))
                        child_loss = self.child_loss_func(y, child_logits)
                        grads = self.child_optimizer.compute_gradients(child_loss)
                        grads = list(filter(lambda g: g[0] is not None, grads))
                        tower_grads.append(grads)
                        child_losses.append(child_loss)
            self.child_loss = tf.reduce_mean(child_losses)
            grads = average_grad(tower_grads)
            with tf.control_dependencies(tf.get_collection(tf.GraphKeys.UPDATE_OPS)):
                self.train_op = self.child_optimizer.apply_gradients(grads, global_step=self.child.train_step)
        self.controller_loss = self.controller.get_loss(self.fixed_arc, self.reward_input_func, self.reward_metric_func,
                                                        self.reward_batches, self.one_shot_reward)
        self.controller_train_op = self.controller_optimizer.minimize(self.controller_loss,
                                                                      global_step=self.controller.train_step)
        self.graph_def = tf.get_default_graph().as_graph_def()

    def train(self,
              num_steps,
              summ_steps,
              ckpt_steps,
              metric_opdefs,
              num_sampled_childs,
              num_ckpts_training_controller,
              num_ckpts_before_sampling_childs,
              controller_steps=None,
              controller_summ_steps=None,
              train_controller=True,
              extra_summ_ops=None,
              listeners=None,
              max_ckpt_to_keep=10,
              sess_config_proto=None,
              from_scratch=True):
        metric_ops, update_ops, reset_ops, _, _ = list(zip(*metric_opdefs))
        metric_summ_names = ['train/child_{0}'.format(m.name.split('/')[-2]) for m in metric_ops]
        metric_summ_ops = [tf.summary.scalar(*tup) for tup in list(zip(metric_summ_names, metric_ops))]
        summ_ops = metric_summ_ops + list(extra_summ_ops) if extra_summ_ops else metric_summ_ops
        summ_op = tf.summary.merge(summ_ops)
        summ_loss_op = tf.summary.scalar('train/child_loss', self.child_loss)
        if from_scratch:
            delete_and_make_dir(self.ckpt_dir)
        child_global_step = self.child.train_step
        data_gntr, _ = self.train_input_func()
        num_ckpts = num_steps // ckpt_steps
        saver = tf.train.Saver(max_to_keep=max_ckpt_to_keep)
        summ_writer = tf.summary.FileWriter(f'{self.ckpt_dir}/train')
        if listeners:
            for l in listeners:
                l.begin(self.ckpt_dir, self.inputs, self.labels, self.is_training, self.is_fixed_arc, self.fixed_arc,
                        self.child_logits, ckpt_steps)
        summ_writer.add_graph(create_op_graph(self.graph_def))
        ctrl_global_step = self.controller.train_step
        ctrl_summ_writer = tf.summary.FileWriter(f'{self.ckpt_dir}/controller')
        ctrl_summ_sample_entropy = tf.summary.scalar('controller/sample_entropy', self.controller.sample_entropy)
        ctrl_summ_baseline = tf.summary.scalar('controller/baseline', self.controller.baseline)
        ctrl_summ_skip_rate = tf.summary.scalar('controller/skip_rate', self.controller.skip_rate)
        ctrl_summ_nofd_op = tf.summary.merge([ctrl_summ_sample_entropy, ctrl_summ_baseline, ctrl_summ_skip_rate])

        ctrl_summ_loss = tf.summary.scalar('controller/loss', self.controller_loss)
        ctrl_summ_reward = tf.summary.scalar('controller/reward', self.controller.raw_reward)
        ctrl_summ_op = tf.summary.merge([ctrl_summ_loss, ctrl_summ_reward])

        if sess_config_proto is None:
            sess_config_proto = tf.ConfigProto(allow_soft_placement=True)
            sess_config_proto.gpu_options.allow_growth = True
        else:
            sess_config_proto = sess_config_proto
        with tf.Session(config=sess_config_proto) as sess:
            sess.run(create_init_op())
            if not from_scratch:
                load_ckpt(sess, model_dir=self.ckpt_dir)
            child_global_step_val = sess.run(child_global_step)
            id_ckpt = child_global_step_val // ckpt_steps
            for _ in range(num_ckpts):
                progress_desc = f'Checkpoint {id_ckpt+1}'
                for _ in self.progress_range(ckpt_steps, desc=progress_desc):
                    xb, yb = sess.run(data_gntr)
                    fd = {self.inputs: xb, self.labels: yb, self.is_training: True,
                          self.is_fixed_arc: False, self.fixed_arc: [0]}
                    if child_global_step_val == 0:
                        sess.run(update_ops, feed_dict=fd)
                        summ = sess.run(summ_op, feed_dict=fd)
                        summ_writer.add_summary(summ, global_step=child_global_step_val)
                        sess.run(reset_ops)
                    summ_loss, _ = sess.run([summ_loss_op, self.train_op])
                    child_global_step_val = sess.run(child_global_step)
                    if child_global_step_val % summ_steps == 0:
                        sess.run(update_ops, feed_dict=fd)
                        summ = sess.run(summ_op, feed_dict=fd)
                        summ_writer.add_summary(summ, global_step=child_global_step_val)
                        summ_writer.add_summary(summ_loss, global_step=child_global_step_val)
                        sess.run(reset_ops)
                id_ckpt = child_global_step_val // ckpt_steps
                summ_writer.flush()

                if train_controller:
                    if id_ckpt > num_ckpts_training_controller:
                        ctr_progress_desc = f'Checkpoint {id_ckpt} Controller'
                        ctrl_global_step_val = sess.run(ctrl_global_step)
                        for _ in self.progress_range(controller_steps, desc=ctr_progress_desc):
                            if ctrl_global_step_val == 0:
                                ctrl_summ_nofd, sampled_arc = sess.run([ctrl_summ_nofd_op, self.controller.sample_arc])
                                ctrl_summ_writer.add_summary(ctrl_summ_nofd, global_step=ctrl_global_step_val)
                                ctrl_summ = sess.run(ctrl_summ_op, feed_dict={self.fixed_arc: sampled_arc})
                                ctrl_summ_writer.add_summary(ctrl_summ, global_step=ctrl_global_step_val)

                            sampled_arc = sess.run(self.controller.sample_arc)
                            sess.run(self.controller_train_op, feed_dict={self.fixed_arc: sampled_arc})
                            ctrl_global_step_val = sess.run(ctrl_global_step)
                            if ctrl_global_step_val % controller_summ_steps == 0:
                                ctrl_summ_nofd, sampled_arc = sess.run([ctrl_summ_nofd_op, self.controller.sample_arc])
                                ctrl_summ_writer.add_summary(ctrl_summ_nofd, global_step=ctrl_global_step_val)
                                ctrl_summ = sess.run(ctrl_summ_op, feed_dict={self.fixed_arc: sampled_arc})
                                ctrl_summ_writer.add_summary(ctrl_summ, global_step=ctrl_global_step_val)
                        ctrl_summ_writer.flush()
                if id_ckpt > num_ckpts_before_sampling_childs:
                    if listeners:
                        sampled_arcs = []
                        for _ in range(num_sampled_childs):
                            sampled_arc = sess.run(self.controller.sample_arc)
                            sampled_arcs.append(sampled_arc)
                        print(f'{num_sampled_childs} architectures presented:')
                        for i, sampled_arc in enumerate(sampled_arcs):
                            print(f'Arch #{i+1}/{num_sampled_childs}:')
                            start = 0
                            for layer_id in range(self.child.num_layers):
                                layer_skip_count = layer_id
                                end = start + 1 + layer_skip_count
                                print(np.reshape(sampled_arc[start:end], [-1]))
                                start = end
                        for l in listeners:
                            l.run(sampled_arcs, sess, child_global_step_val)
                saver.save(sess, f'{self.ckpt_dir}/model.ckpt', global_step=child_global_step_val,
                           write_meta_graph=False)
            if listeners:
                for l in listeners:
                    l.end()
            summ_writer.close()
            ctrl_summ_writer.close()

    def eval(self, metric_opdefs, gnte, provided_arc=None):
        data_gn, steps_per_epoch = gnte
        metric_ops, update_ops, reset_ops, _, _ = list(zip(*metric_opdefs))
        metric_names = [get_metric_name(m) for m in metric_ops]
        logits_lst = []
        if not self.model_loaded:
            raise RuntimeError('Load model first!')
        sess = self.sess
        if provided_arc is None:
            sampled_arc = sess.run(self.controller.sample_arc)
        else:
            sampled_arc = provided_arc
        sess.run(reset_ops)
        progress_desc = 'Test evaluation'
        for _ in self.progress_range(steps_per_epoch, desc=progress_desc):
            xb, yb = sess.run(data_gn)
            fd = {self.inputs: xb, self.labels: yb, self.is_training: False, self.is_fixed_arc: True,
                  self.fixed_arc: sampled_arc}
            logits_val, _ = sess.run([self.child_logits, update_ops], feed_dict=fd)
            logits_lst.append(logits_val)
        logits = np.concatenate(logits_lst, axis=0)
        metrics = sess.run(metric_ops)
        num_child_layers = self.child.num_layers
        print('Sampled architecture:')
        start = 0
        for layer_id in range(num_child_layers):
            layer_skip_count = layer_id
            end = start + 1 + layer_skip_count
            print(np.reshape(sampled_arc[start:end], [-1]))
            start = end
        for name, metric in zip(metric_names, metrics):
            print(f'{name}: {metric}')
        return logits, metrics

max_stars_repo_name: MichaelChuai/modelzoo

max_issues_repo_name: MichaelChuai/modelzoo

max_forks_repo_name: MichaelChuai/modelzoo


text:
import os
import unittest
from tempfile import TemporaryDirectory

import numpy as np

from l5kit.visualization import write_video

class TestVideoVisualizationHelpers(unittest.TestCase):
    def test_write_video(self) -> None:
        # Just a smoke test
        images = (np.random.rand(5, 512, 512, 3) * 255).astype(np.uint8)

        with TemporaryDirectory() as d:
            video_filepath = os.path.join(d, "test_video.mp4")
            write_video(video_filepath, images, (512, 512))

            self.assertTrue(os.path.isfile(video_filepath))

    def test_write_video_with_resize(self) -> None:
        # Just a smoke test
        images = (np.random.rand(5, 256, 256, 3) * 255).astype(np.uint8)

        with TemporaryDirectory() as d:
            video_filepath = os.path.join(d, "test_video.mp4")
            write_video(video_filepath, images, (512, 512))

            self.assertTrue(os.path.isfile(video_filepath))

    def test_write_video_bw(self) -> None:
        # Just a smoke test
        images = (np.random.rand(5, 512, 512) * 255).astype(np.uint8)

        with TemporaryDirectory() as d:
            video_filepath = os.path.join(d, "test_video.mp4")
            write_video(video_filepath, images, (512, 512))

            self.assertTrue(os.path.isfile(video_filepath))


text:
import os

import matplotlib.pyplot as plt
import numpy as np
from spars_wrapper import getSparsRoadmap2d

from ctrm.environment import ObstacleBox, ObstacleSphere

def test_getSparsRoadmap2d():
    obs = [
        ObstacleSphere(pos=np.array([0.3, 0.3]), rad=0.1).get_dict_for_spars(),
        ObstacleBox(
            pos=np.array([0.7, 0.7]), size=np.array([0.01, 0.2])
        ).get_dict_for_spars(),
    ]

    # agent
    rad = 0.1
    speed = 0.2
    sparse_delta_fraction = 0.1
    dense_delta_fraction = 0.1
    stretch_factor = 1.3
    max_sample_num = 100000
    time_limit = 0.1
    lower_bound = 0
    upper_bound = 1

    # get roadmap
    res = getSparsRoadmap2d(
        rad,
        speed,
        sparse_delta_fraction,
        dense_delta_fraction,
        stretch_factor,
        max_sample_num,
        time_limit,
        obs,
        lower_bound,
        upper_bound,
    )
    rmp = res[0]

    for s in rmp:
        pos1 = np.array([s[0], s[1]])
        for i in s[-1]:
            _s = rmp[i]
            pos2 = np.array([_s[0], _s[1]])
            assert np.linalg.norm(pos1 - pos2) <= speed

    # illustrate roadmap
    plt.figure(figsize=(4, 4))
    for v in rmp:
        plt.scatter([v[0]], [v[1]], marker="x", color="blue")
        for j in v[-1]:
            plt.plot(
                [v[0], rmp[j][0]],
                [v[1], rmp[j][1]],
                color="black",
                linewidth=0.2,
            )

    # save result
    dirname = "local/fig"
    if not os.path.exists(dirname):
        os.makedirs(dirname)
    filename = os.path.join(dirname, "test_getSparsRoadmap2d.png")
    plt.savefig(
        filename, pad_inches=0.05, transparent=False, bbox_inches="tight"
    )
    print(f"file is saved into {dirname}")

max_stars_repo_name: omron-sinicx/ctrm

max_issues_repo_name: omron-sinicx/ctrm

max_forks_repo_name: omron-sinicx/ctrm