text:
import argparse
import model
import data
import torch
import time
from pathlib import Path
import tqdm
import json
import utils
import sklearn.preprocessing
import numpy as np
import random
from git import Repo
import os
from torch.utils.data.sampler import SubsetRandomSampler
import torchaudio

tqdm.monitor_interval = 0
# Firstly we define additional Losses:
def SISDR(s, s_hat):
    """Computes the Scale-Invariant SDR as in [1]_.
    References
    ----------
    .. [1] Le Roux, Jonathan, et al. "SDR–half-baked or well done?." ICASSP 2019-2019 IEEE International Conference on
    Acoustics, Speech and Signal Processing (ICASSP). IEEE, 2019.

    Parameters:
        s: list of targets of any shape
        s_hat: list of corresponding estimates of any shape
    """
    s = torch.stack(s).view(-1)
    EPS = torch.finfo(s.dtype).eps
    s_hat = torch.stack(s_hat).view(-1)
    a = (torch.dot(s_hat, s) * s) / ((s ** 2).sum() + EPS)
    b = a - s_hat
    return -10*torch.log10(((a*a).sum()) / ((b*b).sum()+EPS))

def minSNRsdsdr(s,s_hat):
    """Computes the minimum between SNR and Scale-Dependant SDR as proposed in [1]_, obtaining a loss sensitive to both
    upscalings and downscalings.

    References
    ----------
    .. [1] Le Roux, Jonathan, et al. "SDR–half-baked or well done?." ICASSP 2019-2019 IEEE International Conference on
    Acoustics, Speech and Signal Processing (ICASSP). IEEE, 2019.

    Parameters:
        s: list of targets of any shape
        s_hat: list of corresponding estimates of any shape
    """
    s = torch.stack(s).view(-1)
    EPS = torch.finfo(s.dtype).eps
    s_hat = torch.stack(s_hat).view(-1)
    snr = 10*torch.log10((s**2).sum() / ((s-s_hat)**2).sum() + EPS)
    sdsdr = snr + 20*torch.log10(torch.dot(s_hat,s)/(s**2).sum() + EPS)
    return -torch.min(snr, sdsdr)

def SNRPSA(s,s_hat):
    """Computes the SNR_PSA as proposed in [1], with no compression and a saturation value of 20

    References
    ----------
    .. [1] Erdogan, Hakan, and Takuya Yoshioka. "Investigations on Data Augmentation and Loss Functions
    for Deep Learning Based Speech-Background Separation." Interspeech. 2018.

    Parameters:
        s: list of targets of any shape, with len(x) = #sources
        s_hat: list of corresponding estimates
    """
    EPS = torch.finfo(s[0].dtype).eps
    den = [x_hat - x for x, x_hat in zip(s, s_hat)]
    prima = [-10 * torch.log10((x ** 2).sum()/(xa ** 2).sum() + EPS) for x, xa in zip(s, den)]
    clipping = [20*torch.tanh(x/20) for x in prima]
    return torch.stack(clipping).sum()/len(clipping)

def create_criteria(loss, targets):   # Instantiates the pytorch losses
    if loss in ['L2time', 'L2mask', 'L2freq', 'LogL2time', 'LogL2freq', 'PSA']:
        criteria = [torch.nn.MSELoss() for t in targets]
    elif loss == 'Dissimilarity':
        criteria = torch.nn.MSELoss()
    elif loss in ['L1time', 'L1mask', 'L1freq', 'LogL1time', 'LogL1freq']:
        criteria = [torch.nn.L1Loss() for t in targets]
    elif loss == 'BinaryCrossEntropy':
        criteria = [torch.nn.BCEWithLogitsLoss() for t in targets]
    elif loss == 'CrossEntropy':
        criteria = torch.nn.CrossEntropyLoss()
    else:
        criteria = []
    return criteria

# Next, train and validation loops
def train(args, unmix, device, train_sampler, optimizer):
    losses = utils.AverageMeter()
    criteria = create_criteria(args.loss, args.targets)
    # Set model mode as train.
    unmix.train()
    print('Chosen loss: ')
    print(args.loss)
    # Initialize progress bar
    pbar = tqdm.tqdm(train_sampler, disable=args.quiet)
    # Start training loop
    for x, y in pbar:
        pbar.set_description("Training batch")
        # Send inputs and targets to GPU on time domain
        x = x.to(device)
        y = [i.to(device) for i in y]
        # Clear gradients
        optimizer.zero_grad()
        # Obtain tha estimated magnitude mask and reshape it
        Y_hats = unmix(x)    # outputs list of masks: frames, batch, channels, bins; len=sources
        X = unmix.stft(x).permute(3,0,1,2,4)
        # Compute input or mixture magnitude from mixture spectrogram X
        mag = (torchaudio.functional.complex_norm(X))
        loss = 0.0
        EPS = torch.finfo().eps
        # IF FREQUENCY MASKING LOSS:
        if args.loss in ['L1mask', 'L2mask', 'CrossEntropy', 'BinaryCrossEntropy']:
            # Targets are built from the STFT(y)
            Ys = [torchaudio.functional.complex_norm(unmix.stft(target).permute(3, 0, 1, 2, 4)) for target in y]
            # Energy normalization for convergence, obtaining IRM Y:
            energy = torch.sum(torch.stack(Ys), dim=0)
            Y = [Y / (energy + 1e-18) for Y in Ys]
            # For the BCE case, compute IBM setting argmax(pixels) among all sources to 1 and the rest to 0
            if args.loss in ['BinaryCrossEntropy', 'CrossEntropy']:
                Y = torch.stack(Y)
                _, Y = Y.max(0)
                if args.loss == 'BinaryCrossEntropy':   # We one-hot encode the targets for aggregating all BCEs
                    Y = torch.nn.functional.one_hot(Y,4).float().unbind(4)
            # Compute Cross-Entropy mask loss:
            if args.loss == 'CrossEntropy':
                Y_hats = torch.stack(Y_hats).permute(1,0,2,3,4)    # Reshape so it matches expected CE input
                loss = criteria(Y_hats, Y)
            # Or Compute the aggregate losses (L1, L2 or BinaryCrossEntropy)
            else:
                for Y_hat, target, criterion in zip(Y_hats, Y, criteria):
                    loss = loss + criterion(Y_hat, target)
        # IF MAPPING LOSS:
        else:
            # Apply the masks
            Y_hats = [Y_hat * mag for Y_hat in Y_hats]    # obtaining magnitude estimates
            # IF TIME DOMAIN LOSS
            if args.loss in ['L2time', 'L1time', 'SISDRtime', 'MinSNRsdsdr', 'LogL2time', 'LogL1time']:
                phase = torchaudio.functional.angle(X) # phase from mixture STFT X
                # Incorporate mixture phase into our estimates
                specs = [torch.stack([magnitude * torch.cos(phase),
                                      magnitude * torch.sin(phase)],
                                     dim=len(magnitude.shape)) for magnitude in Y_hats]
                # GPU ISTFT
                y_hats = [unmix.istft(spec, x.shape[-1]) for spec in specs]
                # Compute the time-domain loss:
                if args.loss == 'SISDRtime':
                    loss = SISDR(y, y_hats)
                elif args.loss == 'MinSNRsdsdr':
                    loss = minSNRsdsdr(y, y_hats)
                else:
                    for Y_hat, target, criterion in zip(y_hats, y, criteria):
                        if args.loss in ['LogL1time', 'LogL2time']:
                            loss = loss + 10 * torch.log10(criterion(Y_hat, target) + EPS)
                        else:
                            loss = loss + criterion(Y_hat, target)
            # IF FREQUENCY MAPPING:
            else:
                # If using PSA, discount phase error to targets
                if args.loss in ['PSA', 'SNRPSA']:
                    cY = [unmix.stft(target).permute(3, 0, 1, 2, 4) for target in y]    # complex STFT(y)
                    Ymag = [torchaudio.functional.complex_norm(target) for target in cY]   # magnitude of Y
                    phase = torchaudio.functional.angle(X)   # phase of X
                    Yphase = [torchaudio.functional.angle(target) for target in cY]   # phase of Y
                    if args.loss == 'SNRPSA':
                        Y = [(tarmag ** 0.5) * torch.cos(phase - tarphase) for tarmag, tarphase in zip(Ymag, Yphase)]
                    else:
                        Y = [tarmag  * torch.cos(phase - tarphase) for tarmag, tarphase in zip(Ymag, Yphase)]
                # Else, targets are abs(stft(y))
                else:
                    Y = [torchaudio.functional.complex_norm(unmix.stft(target).permute(3, 0, 1, 2, 4)) for target in y]
                # Compute the loss
                if args.loss == 'SISDRfreq':
                    loss = SISDR(Y, Y_hats)
                if args.loss == 'SNRPSA':
                    loss = SNRPSA(Y, Y_hats)
                if args.loss == 'Dissimilarity':
                    for s in range(len(Y_hats)):
                        loss += criteria(Y_hats[s], Y[s])
                        for t in range(len(Y)):   # Dissimilarity term
                            loss -= 0.1 * criteria(Y_hats[s], Y[t])
                else:
                    for Y_hat, target, criterion in zip(Y_hats, Y, criteria):
                        if args.loss in ['LogL1freq', 'LogL2freq']:
                            loss = loss + 10 * torch.log10(criterion(Y_hat, target) + EPS)
                        else:
                            loss = loss + criterion(Y_hat, target)

        loss.backward()
        optimizer.step()
        losses.update(loss.item())
    return losses.avg

def valid(args, unmix, device, valid_sampler):
    #Sames as train() --above--, but with unmix.eval(), no backward and no_grad() mode
    losses = utils.AverageMeter()
    criteria = create_criteria(args.loss, args.targets)
    unmix.eval()
    with torch.no_grad():
        for x, y in valid_sampler:
            x = x.to(device)
            y = [i.to(device) for i in y]
            Y_hats = unmix(x)  # outputs list of masks: frames, batch, channels, bins; len=sources
            X = unmix.stft(x).permute(3, 0, 1, 2, 4)
            mag = torchaudio.functional.complex_norm(X)
            loss = 0.0
            EPS = torch.finfo().eps
            #IF FREQUENCY MASKING:
            if args.loss in ['L1mask', 'L2mask', 'CrossEntropy', 'BinaryCrossEntropy']:
                Ys = [torchaudio.functional.complex_norm(unmix.stft(target).permute(3, 0, 1, 2, 4)) for target in y]
                energy = torch.sum(torch.stack(Ys), dim=0)
                Y = [Y / (energy + 1e-18) for Y in Ys]
                # For the Cross-Entropy cases, compute IBM setting argmax(pixels) among all sources to 1 in a zeros
                if args.loss in ['BinaryCrossEntropy', 'CrossEntropy']:
                    Y = torch.stack(Y)
                    _, Y = Y.max(0)   # Returns which source has the maximum amplitude at each TF pixel
                    if args.loss == 'BinaryCrossEntropy':    # We one-hot encode the targets for aggregating all BCEs
                        Y = torch.nn.functional.one_hot(Y, 4).float().unbind(4)
                if args.loss == 'CrossEntropy':
                    Y_hats = torch.stack(Y_hats).permute(1, 0, 2, 3, 4)    # Reshape so it matches expected CE input
                    loss = criteria(Y_hats, Y)
                # Compute the L1, L2 or Binary Cross-Entropy mask loss:
                else:
                    for Y_hat, target, criterion in zip(Y_hats, Y, criteria):
                        loss = loss + criterion(Y_hat, target)
            #IF NOT MASKING
            else: #Apply the masks
                Y_hats = [Y_hat * mag for Y_hat in Y_hats]
                #if time domain:
                if args.loss in ['L2time', 'L1time', 'SISDRtime', 'MinSNRsdsdr', 'LogL2time', 'LogL1time']:
                    phase = torchaudio.functional.angle(X)
                    specs = [torch.stack([magnitude * torch.cos(phase),
                                          magnitude * torch.sin(phase)],
                                         dim=len(magnitude.shape)) for magnitude in Y_hats]
                    y_hats = [unmix.istft(spec, x.shape[-1]) for spec in specs]
                    if args.loss == 'SISDRtime':
                        loss = SISDR(y, y_hats)
                    elif args.loss == 'MinSNRsdsdr':
                        loss = minSNRsdsdr(y, y_hats)
                    else:
                        for Y_hat, target, criterion in zip(y_hats, y, criteria):
                            if args.loss in ['LogL1time', 'LogL2time']:
                                loss = loss + 10*torch.log10(criterion(Y_hat, target) + EPS)
                            else:
                                loss = loss + criterion(Y_hat, target)

                # IF FREQUENCY MAPPING:
                else:
                    # If using PSA, discount phase error to targets
                    if args.loss in ['PSA', 'SNRPSA']:
                        cY = [unmix.stft(target).permute(3, 0, 1, 2, 4) for target in y]  # complex STFT(y)
                        Ymag = [torchaudio.functional.complex_norm(target) for target in cY]  # magnitude of Y
                        phase = torchaudio.functional.angle(X)  # phase of X
                        Yphase = [torchaudio.functional.angle(target) for target in cY]  # phase of Y
                        if args.loss == 'SNRPSA':
                            # When using SNRPSA, we additionally power-law compress the targets
                            Y = [(tarmag ** 0.5) * torch.cos(phase - tarphase) for tarmag, tarphase in
                                 zip(Ymag, Yphase)]
                        else:
                            Y = [tarmag * torch.cos(phase - tarphase) for tarmag, tarphase in zip(Ymag, Yphase)]
                    # Else, targets are abs(stft(y))
                    else:
                        Y = [torchaudio.functional.complex_norm(unmix.stft(target).permute(3, 0, 1, 2, 4)) for target in
                             y]
                    # Compute the loss
                    if args.loss == 'SISDRfreq':
                        loss = SISDR(Y, Y_hats)
                    if args.loss == 'SNRPSA':
                        loss = SNRPSA(Y, Y_hats)
                    if args.loss == 'Dissimilarity':
                        for s in range(len(Y_hats)):
                            loss += criteria(Y_hats[s], Y[s])
                            for t in range(len(Y)):  # Dissimilarity term
                                loss -= 0.1 * criteria(Y_hats[s], Y[t])
                    else:
                        for Y_hat, target, criterion in zip(Y_hats, Y, criteria):
                            if args.loss in ['LogL1freq', 'LogL2freq']:
                                loss = loss + 10 * torch.log10(criterion(Y_hat, target) + EPS)
                            else:
                                loss = loss + criterion(Y_hat, target)
            losses.update(loss.item())
        return losses.avg

def get_statistics(args, dataloader):
    # If using a different dataset than MUSDB18HQ, uncomment this and import sklearn.preprocessing
    '''
    # What follows computes the dataset statistics with sklearn
    scaler = sklearn.preprocessing.StandardScaler()

    spec = torch.nn.Sequential(
        model.STFT(n_fft=args.nfft, n_hop=args.nhop),
        model.Spectrogram(mono=True)
    )

    pbar = tqdm.tqdm(dataloader, disable=args.quiet)
    for x, y in pbar:
        pbar.set_description("Compute dataset statistics")
        X = spec(x)
        if args.model == 'SNRPSA':
            X = X ** 0.5
        scaler.partial_fit(np.squeeze(X))

    std = np.maximum(
        scaler.scale_,
        1e-4*np.max(scaler.scale_)
    )
    np.save('scalermean.npy',scaler.mean_)
    np.save('std.npy', std)

    return scaler.mean_, std

    '''
    # Otherwise, we directly load the MUSDB18-HQ statistics from file
    return np.load('scalermean.npy'), np.load('std.npy')

def main():
    parser = argparse.ArgumentParser(description='Open Unmix Trainer')
    # Loss parameters
    parser.add_argument('--loss', type=str, default="L2freq",
                        choices=[
                            'L2freq', 'L1freq', 'L2time', 'L1time',
                            'L2mask', 'L1mask', 'SISDRtime', 'SISDRfreq',
                            'MinSNRsdsdr', 'CrossEntropy', 'BinaryCrossEntropy',
                            'LogL2time', 'LogL1time', 'LogL2freq', 'LogL1freq',
                            'PSA', 'SNRPSA', 'Dissimilarity'
                        ],
                        help='kind of loss used during training')

    # Dataset paramaters
    parser.add_argument('--dataset', type=str, default="musdb",
                        choices=[
                            'musdb', 'aligned', 'sourcefolder',
                            'trackfolder_var', 'trackfolder_fix'
                        ],
                        help='Name of the dataset.')

    parser.add_argument('--root', type=str, help='root path of dataset')
    parser.add_argument('--output', type=str, default="open-unmix",
                        help='provide output path base folder name')
    parser.add_argument('--model', type=str, help='Path to checkpoint folder')

    # Trainig Parameters
    parser.add_argument('--epochs', type=int, default=1000)
    parser.add_argument(
        '--reduce-samples',
        type=int,
        default=1,
        help="reduce training samples by factor n"
    )

    parser.add_argument('--batch-size', type=int, default=16)
    parser.add_argument('--lr', type=float, default=0.001,
                        help='learning rate, defaults to 1e-3')
    parser.add_argument('--patience', type=int, default=140,
                        help='maximum number of epochs to train (default: 140)')
    parser.add_argument('--lr-decay-patience', type=int, default=80,
                        help='lr decay patience for plateau scheduler')
    parser.add_argument('--lr-decay-gamma', type=float, default=0.3,
                        help='gamma of learning rate scheduler decay')
    parser.add_argument('--weight-decay', type=float, default=0.00001,
                        help='weight decay')
    parser.add_argument('--seed', type=int, default=42, metavar='S',
                        help='random seed (default: 42)')

    # Model Parameters
    parser.add_argument('--seq-dur', type=float, default=6.0,
                        help='Sequence duration in seconds'
                        'value of <=0.0 will use full/variable length')
    parser.add_argument('--unidirectional', action='store_true', default=False,
                        help='Use unidirectional LSTM instead of bidirectional')
    parser.add_argument('--nfft', type=int, default=4096,
                        help='STFT fft size and window size')
    parser.add_argument('--nhop', type=int, default=1024,
                        help='STFT hop size')
    parser.add_argument('--hidden-size', type=int, default=512,
                        help='hidden size parameter of dense bottleneck layers')
    parser.add_argument('--bandwidth', type=int, default=16000,
                        help='maximum model bandwidth in herz')
    parser.add_argument('--nb-channels', type=int, default=2,
                        help='set number of channels for model (1, 2)')
    parser.add_argument('--nb-workers', type=int, default=0,
                        help='Number of workers for dataloader.')

    # Misc Parameters
    parser.add_argument('--quiet', action='store_true', default=False,
                        help='less verbose during training')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')

    args, _ = parser.parse_known_args()

    use_cuda = not args.no_cuda and torch.cuda.is_available()
    print("Using GPU:", use_cuda)
    dataloader_kwargs = {'num_workers': args.nb_workers, 'pin_memory': True} if use_cuda else {}

    repo_dir = os.path.abspath(os.path.dirname(__file__))
    repo = Repo(repo_dir)
    commit = repo.head.commit.hexsha[:7]

    # use jpg or npy
    torch.manual_seed(args.seed)
    random.seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")
    train_dataset, valid_dataset, args = data.load_datasets(parser, args)

    num_train = len(train_dataset)
    indices = list(range(num_train))

    # shuffle train indices once and for all
    np.random.seed(args.seed)
    np.random.shuffle(indices)

    if args.reduce_samples > 1:
        split = int(np.floor(num_train / args.reduce_samples))
        train_idx = indices[:split]
    else:
        train_idx = indices
    sampler = SubsetRandomSampler(train_idx)
    # create output dir if not exist
    target_path = Path(args.output)
    target_path.mkdir(parents=True, exist_ok=True)

    train_sampler = torch.utils.data.DataLoader(
        train_dataset, batch_size=args.batch_size,
        sampler=sampler, **dataloader_kwargs
    )

    stats_sampler = torch.utils.data.DataLoader(
        train_dataset, batch_size=1,
        sampler=sampler, **dataloader_kwargs
    )

    valid_sampler = torch.utils.data.DataLoader(
        valid_dataset, batch_size=1,
        **dataloader_kwargs
    )

    if args.model:
        scaler_mean = None
        scaler_std = None
    else:
        scaler_mean, scaler_std = get_statistics(args, stats_sampler)

    max_bin = utils.bandwidth_to_max_bin(
        train_dataset.sample_rate, args.nfft, args.bandwidth
    )
    # SNRPSA: de-compress the scaler in order to avoid an exploding gradient from the  uncompressed initial statistics
    if args.loss == 'SNRPSA':
        power = 2
    else:
        power = 1

    unmix = model.OpenUnmixSingle(
        n_fft=4096,
        n_hop=1024,
        input_is_spectrogram=False,
        hidden_size=args.hidden_size,
        nb_channels=args.nb_channels,
        sample_rate=train_dataset.sample_rate,
        nb_layers=3,
        input_mean=scaler_mean,
        input_scale=scaler_std,
        max_bin=max_bin,
        unidirectional=args.unidirectional,
        power=power,
    ).to(device)
    print('learning rate:')
    print(args.lr)
    optimizer = torch.optim.Adam(
        unmix.parameters(),
        lr=args.lr,
        weight_decay=args.weight_decay
    )

    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
        optimizer,
        factor=args.lr_decay_gamma,
        patience=args.lr_decay_patience,
        cooldown=10
    )

    es = utils.EarlyStopping(patience=args.patience)

    # if a model is specified: resume training
    if args.model:
        print('LOADING MODEL')
        model_path = Path(args.model).expanduser()
        with open(Path(model_path, str(len(args.targets)) + '.json'), 'r') as stream:
            results = json.load(stream)

        target_model_path = Path(model_path, "model.chkpnt")
        checkpoint = torch.load(target_model_path, map_location=device)
        unmix.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        scheduler.load_state_dict(checkpoint['scheduler'])
        # train for another epochs_trained
        t = tqdm.trange(
            results['epochs_trained'],
            results['epochs_trained'] + args.epochs + 1,
            disable=args.quiet
        )
        train_losses = results['train_loss_history']
        valid_losses = results['valid_loss_history']
        train_times = results['train_time_history']
        best_epoch = results['best_epoch']
        es.best = results['best_loss']
        es.num_bad_epochs = results['num_bad_epochs']
        print('Model loaded')
    # else start from 0
    else:
        t = tqdm.trange(1, args.epochs + 1, disable=args.quiet)
        train_losses = []
        valid_losses = []
        train_times = []
        best_epoch = 0

    for epoch in t:
        t.set_description("Training Epoch")
        end = time.time()
        train_loss = train(args, unmix, device, train_sampler, optimizer)
        valid_loss = valid(args, unmix, device, valid_sampler)
        scheduler.step(valid_loss)
        train_losses.append(train_loss)
        valid_losses.append(valid_loss)

        t.set_postfix(
            train_loss=train_loss, val_loss=valid_loss
        )

        stop = es.step(valid_loss)

        if valid_loss == es.best:
            best_epoch = epoch

        utils.save_checkpoint({
                'epoch': epoch + 1,
                'state_dict': unmix.state_dict(),
                'best_loss': es.best,
                'optimizer': optimizer.state_dict(),
                'scheduler': scheduler.state_dict()
            },
            is_best=valid_loss == es.best,
            path=target_path,
        )

        # save params
        params = {
            'epochs_trained': epoch,
            'args': vars(args),
            'best_loss': es.best,
            'best_epoch': best_epoch,
            'train_loss_history': train_losses,
            'valid_loss_history': valid_losses,
            'train_time_history': train_times,
            'num_bad_epochs': es.num_bad_epochs,
            'commit': commit
        }

        with open(Path(target_path,  str(len(args.targets)) + '.json'), 'w') as outfile:
            outfile.write(json.dumps(params, indent=4, sort_keys=True))

        train_times.append(time.time() - end)

        if stop:
            print("Apply Early Stopping")
            break

if __name__ == "__main__":
    main()

max_stars_repo_name: enricguso/open-unmix-pytorch

max_issues_repo_name: enricguso/open-unmix-pytorch

max_forks_repo_name: enricguso/open-unmix-pytorch


text:
import numpy as np

from .. import RandomImageCropper

def create_random_img_array(img_height, img_width):
    return np.random.randint(0, 256, (img_height, img_width, 3))

def test_random_crop():
    img_size = 217
    img_array = create_random_img_array(img_size, img_size)
    output_dim = 20
    num_patches = 20
    segmenter = RandomImageCropper(output_dim, num_patches)
    docs_chunks = segmenter.segment(np.stack([img_array, img_array]))
    assert len(docs_chunks) == 2
    for chunks in docs_chunks:
        assert len(chunks) == num_patches
        for chunk in chunks:
            assert chunk['location'][0] <= (img_size - output_dim)
            assert chunk['location'][1] <= (img_size - output_dim)

ext: py

max_stars_repo_name: strawberrypie/jina-hub

max_issues_repo_name: strawberrypie/jina-hub

max_forks_repo_name: strawberrypie/jina-hub

reason: import numpy


text:
import sys

import numpy as np
import cv2

def main():
    w, h = map(int, (sys.argv[1] if len(sys.argv) > 1 else '2048x1944').split('x'))
    imgfile = sys.argv[2] if len(sys.argv) > 2 else r'D:\Downloads\example-navcam-imgs\navcamTests0619\rubbleML-def-14062019-25.raw'
    imgout = sys.argv[3] if len(sys.argv) > 3 else r'D:\Downloads\example-navcam-imgs\navcamTests0619\rubbleML-def-14062019-25.png'

    with open(imgfile, 'rb') as fh:
        raw_img = np.fromfile(fh, dtype=np.uint16, count=w * h)

    raw_img = raw_img.reshape((h, w))

    # flip rows pairwise
    final_img = raw_img[:, np.array([(i*2+1, i*2) for i in range(0, w//2)]).flatten()]

    cv2.imshow('img', final_img)
    cv2.waitKey()

    cv2.imwrite(imgout, final_img)

if __name__ == '__main__':
    main()

max_stars_repo_name: oknuutti/visnav-py

max_issues_repo_name: oknuutti/visnav-py

max_forks_repo_name: oknuutti/visnav-py


text:
from typing import MutableMapping

import numpy as np
import pytest
import xarray as xr
from numpy.testing import assert_array_equal

from sgkit_vcf import partition_into_regions, vcf_to_zarr
from sgkit_vcf.tests.utils import path_for_test

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__small_vcf(shared_datadir, is_path):
    path = path_for_test(shared_datadir, "sample.vcf.gz", is_path)
    output = "vcf.zarr"

    vcf_to_zarr(path, output, chunk_length=5, chunk_width=2)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds.attrs["contigs"] == ["19", "20", "X"]
    assert_array_equal(ds["variant_contig"], [0, 0, 1, 1, 1, 1, 1, 1, 2])
    assert_array_equal(
        ds["variant_position"],
        [111, 112, 14370, 17330, 1110696, 1230237, 1234567, 1235237, 10],
    )
    assert_array_equal(
        ds["variant_allele"],
        [
            ["A", "C", "", ""],
            ["A", "G", "", ""],
            ["G", "A", "", ""],
            ["T", "A", "", ""],
            ["A", "G", "T", ""],
            ["T", "", "", ""],
            ["G", "GA", "GAC", ""],
            ["T", "", "", ""],
            ["AC", "A", "ATG", "C"],
        ],
    )
    assert ds["variant_allele"].dtype == "O"
    assert_array_equal(
        ds["variant_id"],
        [".", ".", "rs6054257", ".", "rs6040355", ".", "microsat1", ".", "rsTest"],
    )
    assert ds["variant_id"].dtype == "O"
    assert_array_equal(
        ds["variant_id_mask"],
        [True, True, False, True, False, True, False, True, False],
    )

    assert_array_equal(ds["sample_id"], ["NA00001", "NA00002", "NA00003"])

    call_genotype = np.array(
        [
            [[0, 0], [0, 0], [0, 1]],
            [[0, 0], [0, 0], [0, 1]],
            [[0, 0], [1, 0], [1, 1]],
            [[0, 0], [0, 1], [0, 0]],
            [[1, 2], [2, 1], [2, 2]],
            [[0, 0], [0, 0], [0, 0]],
            [[0, 1], [0, 2], [-1, -1]],
            [[0, 0], [0, 0], [-1, -1]],
            [[0, -1], [0, 1], [0, 2]],
        ],
        dtype="i1",
    )
    call_genotype_phased = np.array(
        [
            [True, True, False],
            [True, True, False],
            [True, True, False],
            [True, True, False],
            [True, True, False],
            [True, True, False],
            [False, False, False],
            [False, True, False],
            [True, False, True],
        ],
        dtype=bool,
    )
    assert_array_equal(ds["call_genotype"], call_genotype)
    assert_array_equal(ds["call_genotype_mask"], call_genotype < 0)
    assert_array_equal(ds["call_genotype_phased"], call_genotype_phased)

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__large_vcf(shared_datadir, is_path):
    path = path_for_test(shared_datadir, "CEUTrio.20.21.gatk3.4.g.vcf.bgz", is_path)
    output = "vcf.zarr"

    vcf_to_zarr(path, output, chunk_length=5_000)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (1,)
    assert ds["call_genotype"].shape == (19910, 1, 2)
    assert ds["call_genotype_mask"].shape == (19910, 1, 2)
    assert ds["call_genotype_phased"].shape == (19910, 1)
    assert ds["variant_allele"].shape == (19910, 4)
    assert ds["variant_contig"].shape == (19910,)
    assert ds["variant_id"].shape == (19910,)
    assert ds["variant_id_mask"].shape == (19910,)
    assert ds["variant_position"].shape == (19910,)

    assert ds["variant_allele"].dtype == "O"
    assert ds["variant_id"].dtype == "O"

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__mutable_mapping(shared_datadir, is_path):
    path = path_for_test(shared_datadir, "CEUTrio.20.21.gatk3.4.g.vcf.bgz", is_path)
    output: MutableMapping[str, bytes] = {}

    vcf_to_zarr(path, output, chunk_length=5_000)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (1,)
    assert ds["call_genotype"].shape == (19910, 1, 2)
    assert ds["call_genotype_mask"].shape == (19910, 1, 2)
    assert ds["call_genotype_phased"].shape == (19910, 1)
    assert ds["variant_allele"].shape == (19910, 4)
    assert ds["variant_contig"].shape == (19910,)
    assert ds["variant_id"].shape == (19910,)
    assert ds["variant_id_mask"].shape == (19910,)
    assert ds["variant_position"].shape == (19910,)

    assert ds["variant_allele"].dtype == "O"
    assert ds["variant_id"].dtype == "O"

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__parallel(shared_datadir, is_path):
    path = path_for_test(shared_datadir, "CEUTrio.20.21.gatk3.4.g.vcf.bgz", is_path)
    output = "vcf_concat.zarr"
    regions = ["20", "21"]

    vcf_to_zarr(path, output, regions=regions, chunk_length=5_000)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (1,)
    assert ds["call_genotype"].shape == (19910, 1, 2)
    assert ds["call_genotype_mask"].shape == (19910, 1, 2)
    assert ds["call_genotype_phased"].shape == (19910, 1)
    assert ds["variant_allele"].shape == (19910, 4)
    assert ds["variant_contig"].shape == (19910,)
    assert ds["variant_id"].shape == (19910,)
    assert ds["variant_id_mask"].shape == (19910,)
    assert ds["variant_position"].shape == (19910,)

    assert ds["variant_allele"].dtype == "S48"
    assert ds["variant_id"].dtype == "S1"

@pytest.mark.parametrize(
    "is_path", [False],
)
def test_vcf_to_zarr__parallel_temp_chunk_length(shared_datadir, is_path):
    path = path_for_test(shared_datadir, "CEUTrio.20.21.gatk3.4.g.vcf.bgz", is_path)
    output = "vcf_concat.zarr"
    regions = ["20", "21"]

    # Use a temp_chunk_length that is smaller than chunk_length
    vcf_to_zarr(
        path, output, regions=regions, chunk_length=5_000, temp_chunk_length=2_500
    )
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (1,)
    assert ds["call_genotype"].shape == (19910, 1, 2)
    assert ds["call_genotype_mask"].shape == (19910, 1, 2)
    assert ds["call_genotype_phased"].shape == (19910, 1)
    assert ds["variant_allele"].shape == (19910, 4)
    assert ds["variant_contig"].shape == (19910,)
    assert ds["variant_id"].shape == (19910,)
    assert ds["variant_id_mask"].shape == (19910,)
    assert ds["variant_position"].shape == (19910,)

    assert ds["variant_allele"].dtype == "S48"
    assert ds["variant_id"].dtype == "S1"

def test_vcf_to_zarr__parallel_temp_chunk_length_not_divisible(shared_datadir):
    path = path_for_test(shared_datadir, "CEUTrio.20.21.gatk3.4.g.vcf.bgz", False)
    output = "vcf_concat.zarr"
    regions = ["20", "21"]

    with pytest.raises(
        ValueError,
        match=r"Temporary chunk length in variant dimension \(4000\) must evenly divide target chunk length 5000",
    ):
        # Use a temp_chunk_length that does not divide into chunk_length
        vcf_to_zarr(
            path, output, regions=regions, chunk_length=5_000, temp_chunk_length=4_000
        )

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__parallel_partitioned(shared_datadir, is_path):
    path = path_for_test(
        shared_datadir,
        "1000G.phase3.broad.withGenotypes.chr20.10100000.vcf.gz",
        is_path,
    )
    output = "vcf_concat.zarr"

    regions = partition_into_regions(path, num_parts=4)

    vcf_to_zarr(path, output, regions=regions, chunk_length=1_000, chunk_width=1_000)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (2535,)
    assert ds["variant_id"].shape == (1406,)

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__multiple(shared_datadir, is_path):
    paths = [
        path_for_test(shared_datadir, "CEUTrio.20.gatk3.4.g.vcf.bgz", is_path),
        path_for_test(shared_datadir, "CEUTrio.21.gatk3.4.g.vcf.bgz", is_path),
    ]
    output = "vcf_concat.zarr"

    vcf_to_zarr(paths, output, chunk_length=5_000)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (1,)
    assert ds["call_genotype"].shape == (19910, 1, 2)
    assert ds["call_genotype_mask"].shape == (19910, 1, 2)
    assert ds["call_genotype_phased"].shape == (19910, 1)
    assert ds["variant_allele"].shape == (19910, 4)
    assert ds["variant_contig"].shape == (19910,)
    assert ds["variant_id"].shape == (19910,)
    assert ds["variant_id_mask"].shape == (19910,)
    assert ds["variant_position"].shape == (19910,)

    assert ds.chunks["variants"] == (5000, 5000, 5000, 4910)

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__multiple_partitioned(shared_datadir, is_path):
    paths = [
        path_for_test(shared_datadir, "CEUTrio.20.gatk3.4.g.vcf.bgz", is_path),
        path_for_test(shared_datadir, "CEUTrio.21.gatk3.4.g.vcf.bgz", is_path),
    ]
    output = "vcf_concat.zarr"

    regions = [partition_into_regions(path, num_parts=2) for path in paths]

    vcf_to_zarr(paths, output, regions=regions, chunk_length=5_000)
    ds = xr.open_zarr(output)  # type: ignore[no-untyped-call]

    assert ds["sample_id"].shape == (1,)
    assert ds["call_genotype"].shape == (19910, 1, 2)
    assert ds["call_genotype_mask"].shape == (19910, 1, 2)
    assert ds["call_genotype_phased"].shape == (19910, 1)
    assert ds["variant_allele"].shape == (19910, 4)
    assert ds["variant_contig"].shape == (19910,)
    assert ds["variant_id"].shape == (19910,)
    assert ds["variant_id_mask"].shape == (19910,)
    assert ds["variant_position"].shape == (19910,)

    assert ds.chunks["variants"] == (5000, 5000, 5000, 4910)

@pytest.mark.parametrize(
    "is_path", [True, False],
)
def test_vcf_to_zarr__mutiple_partitioned_invalid_regions(shared_datadir, is_path):
    paths = [
        path_for_test(shared_datadir, "CEUTrio.20.gatk3.4.g.vcf.bgz", is_path),
        path_for_test(shared_datadir, "CEUTrio.21.gatk3.4.g.vcf.bgz", is_path),
    ]
    output = "vcf_concat.zarr"

    # invalid regions, should be a sequence of sequences
    regions = partition_into_regions(paths[0], num_parts=2)

    with pytest.raises(
        ValueError,
        match=r"multiple input regions must be a sequence of sequence of strings",
    ):
        vcf_to_zarr(paths, output, regions=regions, chunk_length=5_000)

max_stars_repo_name: ravwojdyla/sgkit-vcf

max_issues_repo_name: ravwojdyla/sgkit-vcf

max_forks_repo_name: ravwojdyla/sgkit-vcf

reason: import numpy,from numpy


text:
from multiprocessing import Pool  #parallel processing
import multiprocessing as mp
import structure
from structure.global_constants import *
from structure.cell import Tissue, BasicSpringForceNoGrowth
import structure.initialisation as init
import sys,os
import numpy as np
import libs.contact_inhibition_lib as lib
import libs.data as data
from functools import partial
import pandas as pd

def mean_proportion_coop_neighbours_ready(idxlist,all_neighbours,cooperators):
    neighbours = [all_neighbours[i] for i in idxlist]
    data = [float(sum(cooperators[cell_neighbours]))/len(cell_neighbours) for cell_neighbours in neighbours]
    return np.mean(data),np.std(data)

def proliferating_cells_data(tissue,alpha,mutant_index,n):
    cooperators = tissue.properties['ancestor']==mutant_index
    proliferating = tissue.mesh.areas > alpha*A0
    proliferating_cooperators = np.where(proliferating&cooperators)[0]
    proliferating_defectors = np.where(proliferating&~cooperators)[0]
    prop_neighbours_coop,prop_neighbours_coop_std = mean_proportion_coop_neighbours(np.where(cooperators)[0],tissue.mesh.neighbours,cooperators)
    prop_neighbours_defect,prop_neighbours_defect_std = mean_proportion_coop_neighbours(np.where(~cooperators)[0],tissue.mesh.neighbours,cooperators)
    prop_neighbours_coop_ready,prop_neighbours_coop_ready_std = mean_proportion_coop_neighbours(proliferating_cooperators,tissue.mesh.neighbours,cooperators)
    prop_neighbours_defect_ready,prop_neighbours_defect_ready_std = mean_proportion_coop_neighbours(proliferating_defectors,tissue.mesh.neighbours,cooperators)
    return (n,len(tissue)-n,len(proliferating_cooperators),len(proliferating_defectors),
                    prop_neighbours_coop,prop_neighbours_coop_std,prop_neighbours_defect,prop_neighbours_defect_std,
                        prop_neighbours_coop_ready,prop_neighbours_coop_ready_std,prop_neighbours_defect_ready,prop_neighbours_defect_ready_std)

def run_sim(alpha,db,m,i):
    """run a single simulation and save interaction data for each clone"""
    rates = (DEATH_RATE,DEATH_RATE/db)
    rand = np.random.RandomState()
    data = [(tissue.time,proliferating_cells_data(tissue,alpha,mutant_index,n))
                for tissue in lib.run_simulation(simulation,L,TIMESTEP,TIMEND,rand,progress_on=False,ancestors=True,
                                    init_time=INIT_TIME,til_fix=True,save_areas=True,return_events=False,save_cell_histories=False,
                                    N_limit=MAX_POP_SIZE,game=None, mutant_num=None,domain_size_multiplier=m,rates=rates,threshold_area_fraction=alpha,generator=True)
                                for mutant_index,n in enumerate(np.bincount(tissue.properties['ancestor'])) if n>=n_min]
    if i%100 == 0:
        print('%d complete'%i)
    return data

def sort_data(data):
    data = [[i,round(time),nc,nd,proliferating_coop,proliferating_defect,
                prop_neighbours_coop,prop_neighbours_coop_std,prop_neighbours_defect,prop_neighbours_defect_std,
                prop_neighbours_coop_ready,prop_neighbours_coop_ready_std,prop_neighbours_defect_ready,prop_neighbours_defect_ready_std]
                for i,run_data in enumerate(data)
                    for time,(nc,nd,proliferating_coop,proliferating_defect,
                    prop_neighbours_coop,prop_neighbours_coop_std,prop_neighbours_defect,prop_neighbours_defect_std,
                    prop_neighbours_coop_ready,prop_neighbours_coop_ready_std,prop_neighbours_defect_ready,prop_neighbours_defect_ready_std
                    ) in run_data]
    df = pd.DataFrame(data,columns = ['run','time','nc','nd','ncready','ndready',
                                        'propcc','propccstd','propcd','propcdstd',
                                        'propccready','propccreadystd','propcdready','propcdreadystd'])
    return df

L = 10 # population size N = l*l
INIT_TIME = 96. # initial simulation time to equilibrate
TIMEND = 80000. # length of simulation (hours)
TIMESTEP = 96. # time intervals to save simulation history
MAX_POP_SIZE = 1000
DEATH_RATE = 0.25/24.
SIM_RUNS = int(sys.argv[1]) # number of sims to run
n_min = 1
simulation = lib.simulation_contact_inhibition_area_dependent

params = [[0.800000, 0.100000, 0.859628],[1.000000, 0.100000, 0.948836],[1.200000, 0.100000, 1.030535]]
alpha,db,m = params[int(sys.argv[2])]

outdir = 'CD_data/cip/'

if not os.path.exists(outdir): # if the outdir doesn't exist create it
     os.makedirs(outdir)

# run simulations in parallel
cpunum=mp.cpu_count()
pool = Pool(processes=cpunum-1,maxtasksperchild=1000)
df = sort_data(pool.map(partial(run_sim,alpha,db,m),range(SIM_RUNS)))
pool.close()
pool.join()
df.to_csv(outdir+'db%.2f_alpha%.1f_neutral.csv'%(db,alpha),index=False)

max_stars_repo_name: jessiesrr/VTdyn

max_issues_repo_name: jessiesrr/VTdyn

max_forks_repo_name: jessiesrr/VTdyn


text:
import torch
from transformers import *
torch.cuda.set_device(0)
import torch.nn.functional as F
import pandas as pd
import numpy as np
from torch.utils.data import TensorDataset, DataLoader, SequentialSampler
from tensorflow.keras.preprocessing.sequence import pad_sequences

#不應該有ground_truth
class Review_Sentiment:
    def __init__(self,model_path,MAX_LEN=100):
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
        self.MAX_LEN = 100
        self.model = BertForSequenceClassification.from_pretrained(model_path)
        self.data = []
        self.ground_truth = {'label':[],'comm':[]}
    def to_ids(self):
        inputs = []
        masks = []
        self.ground_truth['label'].clear()
        self.ground_truth['comm'].clear()
        for sentence,label in zip(self.data['comm'],self.data['label']):
            if (sentence == sentence):
                data = self.tokenizer.encode(sentence, add_special_tokens=True)
                if len(data) <= self.MAX_LEN:
                    self.ground_truth['label'].append(label)
                    self.ground_truth['comm'].append(sentence)

                    data = pad_sequences([data], maxlen=self.MAX_LEN, dtype="long", truncating="post", padding="post")
                    attention_masks = [[float(i>0) for i in ii] for ii in data]
                    inputs.append(data)
                    masks.append(attention_masks)
        self.inputs= np.array(inputs).reshape(len(inputs),self.MAX_LEN)
        self.masks = np.array(masks).reshape(len(inputs),self.MAX_LEN)
    def prediction(self,data,batch_size=300):
        self.data = data
        self.to_ids()
        self.model.cuda()
        self.model.eval()
        inputs = torch.tensor(self.inputs)
        masks = torch.tensor(self.masks)
        inputs = inputs.cuda()
        masks = masks.cuda()
        valid_data = TensorDataset(inputs, masks)
        valid_sampler = SequentialSampler(valid_data)
        valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=batch_size)

        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        pred = []
        for batch in valid_dataloader:
            batch = tuple(t.to(device) for t in batch)
            b_input_ids, b_input_mask = batch
            with torch.no_grad():
                outputs = self.model(b_input_ids, token_type_ids=None,attention_mask=b_input_mask)
                prediction = torch.max(F.softmax(outputs[0]),  dim=1)[1]
                prediction = prediction.cpu()
                pred.extend(prediction.numpy().tolist())
        return pred
    def to_csv(self,pred,hotel_name):
        raw_data= {'label':[],'comm':[]}
        for label,comm in zip(pred,self.ground_truth['comm']):
            raw_data['label'].append( int(label>0.5) )
            raw_data['comm'].append(comm)

        df = pd.DataFrame(raw_data, columns = ['label','comm'])
        df.to_csv('bots/cache/'+hotel_name+'_label.csv',index = False)

        #### for test
        df = pd.DataFrame(self.ground_truth, columns = ['label','comm'])
        df.to_csv('bots/cache/'+hotel_name+'ground_truth.csv',index = False)

        return raw_data


text:
__author__ = "Steve Ataucuri"
__copyright__ = "Sprace.org.br"
__version__ = "1.0.0"

import sys
import os
import argparse
import json

from sklearn.model_selection import train_test_split

from core.data.data_loader import *
from core.models.lstm import ModelLSTM, ModelLSTMParalel, ModelLSTMCuDnnParalel
from core.models.cnn import ModelCNN
from core.models.mlp import ModelMLP
from core.models.rnn import ModelRNN

from core.utils.metrics import *
from core.utils.utils import *

import numpy as np

def parse_args():
    """Parse arguments."""
    # Parameters settings
    parser = argparse.ArgumentParser(description="MLP Implementation")

    # Dataset setting
    parser.add_argument('--config', type=str, default="config.json", help='Configuration file')
    parser.add_argument('--dataset', type=str, help='Path to dataset')
    parser.add_argument('--cylindrical', type=str, help='Type of Coordenates system')
    # parse the arguments
    args = parser.parse_args()

    return args

def manage_models(config):

    type_model = config['model']['name']
    model = None

    if type_model == 'lstm': #simple LSTM
        model = ModelLSTM(config)
    elif type_model == 'lstm-paralel':
        model = ModelLSTMParalel(config)
    elif type_model == 'cnn':
        model = ModelCNN(config)
    elif type_model == 'mlp':
        model = ModelMLP(config)
    elif type_model == 'simplernn':
        model = ModelRNN(config)

    return model

def main():

    args = parse_args()

    # load configurations of model and others
    configs = json.load(open(args.config, 'r'))

    # create defaults dirs
    output_path = configs['paths']['save_dir']
    output_logs = configs['paths']['log_dir']
    data_file = configs['data']['filename']

    if os.path.isdir(output_path) == False:
        os.mkdir(output_path)

    if os.path.isdir(output_logs) == False:
        os.mkdir(output_logs)

    time_steps =  configs['model']['layers'][0]['input_timesteps']  # the number of points or hits
    num_features = configs['model']['layers'][0]['input_features']  # the number of features of each hits

    split = configs['data']['train_split']  # the number of features of each hits
    cylindrical = configs['data']['cylindrical']  # set to polar or cartesian coordenates
    normalise = configs['data']['normalise']
    num_hits = configs['data']['num_hits']

    if args.dataset is not None:
        data_file = args.dataset
        configs['data']['filename'] = data_file
    if args.cylindrical is not None:
        cylindrical = True if args.cylindrical == "True" else False
        configs['data']['cylindrical'] = cylindrical

    # prepare data set
    data = Dataset(data_file, KindNormalization.Zscore)

    dataset = data.get_training_data(cylindrical=cylindrical, hits=num_hits)
    #dataset = dataset.iloc[0:2640,0:]
    #dataset = dataset.iloc[0:31600,0:]
    print('[Data] new shape :', dataset.shape)

    print("[Data] Converting to supervised ...")
    X, y = data.convert_to_supervised(dataset.values, n_hit_in=time_steps,
                                n_hit_out=1, n_features=num_features, normalise=normalise)

    print('[Data] shape supervised: X%s y%s :' % (X.shape, y.shape))

    # shuffle is True for default value
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=1-split, shuffle=False, random_state=42)

    print('[Data] shape data X_train.shape:', X_train.shape)
    print('[Data] shape data X_test.shape:', X_test.shape)
    print('[Data] shape data y_train.shape:', y_train.shape)
    print('[Data] shape data y_test.shape:', y_test.shape)

    model = manage_models(configs)

    if model is None:
        print('Please instance model')
        return

    loadModel = configs['training']['load_model']

    if loadModel == False:

        model.build_model()

        # in-memory training
        history = model.train(
            x=X_train,
            y=y_train,
            epochs = configs['training']['epochs'],
            batch_size = configs['training']['batch_size']
        )

        evaluate_training(history, output_path)

    elif loadModel == True:
        if not model.load_model():
            print ('[Error] please change the config file : load_model')
            return

    print('[Data] Predicting dataset with input ...', X_test.shape)
    predicted = model.predict_one_hit(X_test)
    print('[Data] shape predicted output ', predicted.shape)
    print('[Data] shape y_test ', y_test.shape)

    y_predicted = np.reshape(predicted, (predicted.shape[0]*predicted.shape[1], 1))
    y_true_ = data.reshape2d(y_test, 1)

    print('[Data] new shape y_test, y_true_ ', y_true_.shape)
    print('[Data] new shape y_predicted ', y_predicted.shape)

    # we need to transform to original data
    if normalise:
        y_test_orig = data.inverse_transform_y(y_test)
        y_predicted_orig = data.inverse_transform_y(predicted)
    else:
        y_test_orig = y_test
        y_predicted_orig = predicted

    if cylindrical:
        coord = 'cylin'
    else:
        coord = 'xyz'

    # save results in a file
    orig_stdout = sys.stdout
    f = open('results/results.txt', 'a')
    sys.stdout = f

    print("[Output] Results ")
    print("---Parameters--- ")
    print("\t Model Name    : ", model.name)
    print("\t Dataset       : ", model.orig_ds_name)
    print("\t Tracks        : ", len(dataset))
    print("\t Model saved   : ", model.save_fnameh5)
    print("\t Coordenates   : ", coord)
    print("\t Model stand   : ", model.normalise)

    # calculing scores
    result = calc_score(y_true_, y_predicted, report=True)
    #r2, rmse, rmses = evaluate_forecast(y_test, predicted)
    r2, rmse, rmses = evaluate_forecast(y_test_orig, y_predicted_orig)
    summarize_scores(r2, rmse,rmses)

    sys.stdout = orig_stdout
    f.close()

    print('[Data] shape y_test ', y_test.shape)
    print('[Data] shape predicted ', predicted.shape)

    print('[Data] shape y_test_orig ', y_test_orig.shape)
    print('[Data] shape y_predicted_orig ', y_predicted_orig.shape)

    #X = data.get_training_data(cylindrical=False, hit=10)
    X_train, X_test_new = train_test_split(dataset, test_size=1-split, shuffle=False, random_state=42)
    #X_train, X_test = data.train_test_split(dataset, y, train_size=split)

    # 18 fields
    y_predicted_orig = pd.DataFrame(y_predicted_orig)
    y_predicted_orig = data.convert_supervised_to_normal(y_predicted_orig.values, n_hit_in=4, n_hit_out=1, hits=10)
    y_predicted_orig = pd.DataFrame(y_predicted_orig)
    print('y_predicted_orig shape ', y_predicted_orig.shape)

    y_true_orig = pd.DataFrame(y_test_orig)
    y_true_orig = data.convert_supervised_to_normal(y_true_orig.values, n_hit_in=4, n_hit_out=1, hits=10)
    y_true_orig = pd.DataFrame(y_true_orig)
    print('y_true_orig shape ', y_true_orig.shape)

    x_true = X_test_new.iloc[:,0:time_steps*num_features]
    y_true = X_test_new.iloc[:,time_steps*num_features:]

    if cylindrical:

        y_true_orig.to_csv(os.path.join(output_path, 'y_true_%s_cylin.csv' % configs['model']['name']),
                    header=False, index=False)
        y_predicted_orig.to_csv(os.path.join(output_path, 'y_pred_%s_cylin.csv' % configs['model']['name']),
                    header=False, index=False)
        x_true.to_csv(os.path.join(output_path, 'x_true_%s_cylin.csv' % configs['model']['name']),
                    header=False, index=False)
    else:

        y_true_orig.to_csv(os.path.join(output_path, 'y_true_%s_xyz.csv' % configs['model']['name']),
                    header=False, index=False)
        y_predicted_orig.to_csv(os.path.join(output_path, 'y_pred_%s_xyz.csv' % configs['model']['name']),
                    header=False, index=False)
        x_true.to_csv(os.path.join(output_path, 'x_true_%s_xyz.csv' % configs['model']['name']),
                    header=False, index=False)

    print('[Output] Results saved in files %', output_path)

if __name__=='__main__':
    main()

max_stars_repo_name: SPRACE/track-ml

max_issues_repo_name: SPRACE/track-ml

max_forks_repo_name: SPRACE/track-ml


text:
import gensim
from gensim.models import doc2vec
from gensim.models import word2vec
from gensim.models import fasttext
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
import numpy as np
import jaconv
import pickle
import re
from mldatautils.logger import logger_config

class TokenFilter(object):
    @staticmethod
    def use_all_for_sentences(
        parsed_sentences,
        use_jaconv=False,
        pos_list=[],
        except_words=[],
        exclude_numbers=False,
        exclude_symbols=False
    ):
        '''
        Extract lemmatized? tokens from tokenized sentence.
        Currently this function suports only MeCab default format as input.
        Args:
            parsed_sentences(list): list of output of MeCab.Tagger().parse(sentence)
            use_jaconv(bool): normalize tokens with jaconv or not
            pos_list(list): if we specify `pos_list=['名詞', '動詞']`,
                            it returns only the token which is `名詞` or `動詞`.
            except_words(list): if we specify `except_words=['基本', 'きほん']`,
                                it returns token list without word '基本' and 'きほん'.
            exclude_numbers(boot): if we specify `True`, it returns token list
                                   without `名詞 数` such as `2019`.
            exclude_symbols(bool): if we specify `True`, it returns token list
                                   without `記号`.
        Returns:
            word_list(list): tokens
        '''
        word_lists = [[]] * len(parsed_sentences)
        for i, parsed_sentence in enumerate(parsed_sentences):
            word_lists[i] = TokenFilter.use_all(
                parsed_sentence, use_jaconv, pos_list, except_words, exclude_numbers, exclude_symbols)
        return word_lists

    @staticmethod
    def use_all(
        parsed_sentence,
        use_jaconv=False,
        pos_list=[],
        except_words=[],
        exclude_numbers=False,
        exclude_symbols=False
    ):
        '''
        Extract lemmatized? tokens from tokenized sentence.
        Currently this function suports only MeCab default format as input.
        Args:
            parsed_sentence(str): output of MeCab.Tagger().parse(sentence)
            use_jaconv(bool): normalize tokens with jaconv or not
            pos_list(list): if we specify `pos_list=['名詞', '動詞']`,
                            it returns only the token which is `名詞` or `動詞`.
            except_words(list): if we specify `except_words=['基本', 'きほん']`,
                                it returns token list without word '基本' and 'きほん'.
            exclude_numbers(boot): if we specify `True`, it returns token list
                                   without `名詞 数` such as `2019`.
            exclude_symbols(bool): if we specify `True`, it returns token list
                                   without `記号`.
        Returns:
            word_list(list): tokens
        '''

        word_infos = parsed_sentence.split('\n')
        word_list = TokenFilter._extract_lemmatized_words(
            word_infos,
            use_jaconv,
            pos_list,
            except_words,
            exclude_numbers,
            exclude_symbols)
        return word_list

    @staticmethod
    def _extract_lemmatized_words(
        word_infos,
        use_jaconv=False,
        pos_list=[],
        except_words=[],
        exclude_numbers=False,
        exclude_symbols=False
    ):
        word_list = []
        for word_info in word_infos:
            word = TokenFilter._extract_lemmatized_word(
                word_info,
                use_jaconv,
                pos_list,
                exclude_numbers,
                exclude_symbols)
            except_words = ['', 'EOS'] + except_words
            if word not in except_words:
                word_list.append(word)
        return word_list

    @staticmethod
    def _extract_lemmatized_word(
        word_info,
        use_jaconv=False,
        pos_list=[],
        exclude_numbers=False,
        exclude_symbols=False
    ):
        word_info = word_info.split('\t')
        if len(word_info) > 1:
            word_details = word_info[1].split(',')
            if pos_list != [] and word_details[0] not in pos_list:
                return ''
            if exclude_numbers and word_details[1] == '数':
                return ''
            if exclude_symbols and word_details[0] == '記号':
                return ''
            if len(word_details) > 6 and word_details[6] != '*':
                word = word_details[6]
            else:
                word = word_info[0]
        else:
            word = word_info[0]
        if use_jaconv:
            word = jaconv.z2h(word, digit=True, ascii=True)
            word = jaconv.normalize(word)
        return word

    @staticmethod
    def remove_special_characters(text):
        non_CJK_patterns = re.compile(
            "[^"
            u"\U00003040-\U0000309F"  # Hiragana
            u"\U000030A0-\U000030FF"  # Katakana
            u"\U0000FF65-\U0000FF9F"  # Half width Katakana
            u"\U0000FF10-\U0000FF19"  # Full width digits
            u"\U0000FF21-\U0000FF3A"  # Full width Upper case  English Alphabets
            u"\U0000FF41-\U0000FF5A"  # Full width Lower case English Alphabets
            u"\U00000030-\U00000039"  # Half width digits
            u"\U00000041-\U0000005A"  # Half width  Upper case English Alphabets
            u"\U00000061-\U0000007A"  # Half width Lower case English Alphabets
            u"\U00003190-\U0000319F"  # Kanbun
            u"\U00004E00-\U00009FFF"  # CJK unified ideographs. kanjis
            "]+", flags=re.UNICODE)
        return non_CJK_patterns.sub(r"", text)

class Vectorizer(object):
    def __init__(self, model_type, model_file, log_level='WARN'):
        """
        model_type(str): `word2vec` or `doc2vec` or `fasttext` or `tfidf`
        model_file(str): file path of the model
        log_level(str): log level
        """
        self.logger = logger_config('vectorizer', log_level)
        self.model, self.model_dim = self._load_model(model_type, model_file)
        self.model_type = model_type

    def _load_model(self, model_type, model_file):
        if model_type == 'doc2vec':
            model = doc2vec.Doc2Vec.load(model_file)
            model_dim = model.vector_size
        elif model_type == 'word2vec':
            model = word2vec.Word2Vec.load(model_file)
            model_dim = model.vector_size
        elif model_type == 'fasttext':
            model = fasttext.FastText.load(model_file)
            model_dim = model.vector_size
        else:  # tfidf
            with open(model_file, mode='rb') as f:
                model = pickle.load(f)
                model_dim = len(model.vocabulary_)
        return model, model_dim

    def transform_sentences(self, words_list, vector_dim=None, pooling='avg'):
        """
        Args:
            words_list(str): a list or pd.Series of word list such as [['今日', 'は', 'いい', '天気'],[...]]
            vector_dim(int): dimension of output vector
            pooling(str): 'avg' or 'max'. vector summalize method when we use `word2vec` or `fasttext`
                          for sentence vectorizer
        """
        if vector_dim is None:
            vector_dim = self.model_dim
        if self.model_dim < vector_dim:
            self.logger.error('vector size should be bigger than size.')
            return None

        # TODO: transform sentences in one time when use doc2vec
        if self.model_type == 'tfidf':
            words_list = [' '.join(word_list) for word_list in words_list]
            vectors = self.model.transform(words_list)
        else:
            vectors = np.zeros((len(words_list), self.model_dim), dtype='float32')
            for i, word_list in enumerate(words_list):
                vectors[i] = self.transform_sentence(word_list, pooling='avg')
        if self.model_dim > vector_dim:
            self.logger.info('decompose vector with TruncatedSVD')
            vectors = TruncatedSVD(n_components=vector_dim).fit_transform(vectors)
        return vectors

    def transform_sentence(self, word_list, pooling='avg'):
        """
        Args:
            word_list(str): a list or pd.Series of words such as ['今日', 'は', 'いい', '天気']
            vector_dim(int): dimension of output vector
            pooling(str): 'avg' or 'max'. vector summalize method when we use `word2vec` or `fasttext`
                          for sentence vectorizer
        """
        vector = np.zeros(self.model_dim, dtype='float32')
        word_num = 0
        self.logger.info('tokens: {}'.format(word_list))
        if not word_list:
            return vector

        if self.model_type == 'tfidf':
            vector = self.model.transform([' '.join(word_list)]).toarray().astype(np.float32)[0]
        elif self.model_type == 'doc2vec':
            vector = model.infer_vector(word_list)
        elif self.model_type == 'fasttext':
            if pooling == 'avg':
                vector = self.model.wv[word_list].mean(axis=0)
            else:  # max
                vector = self.model.wv[word_list].max(axis=0)
        else:  # word2vec
            if pooling == 'avg':
                vector = self.model.wv[filter(lambda x: x in self.model.wv.vocab, word_list)].mean(axis=0)
            else:  # max
                vector = self.model.wv[filter(lambda x: x in self.model.wv.vocab, word_list)].max(axis=0)
        return vector


text:
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from scipy.signal import savgol_filter

dtype={'names':('date','weight'), 'formats':('datetime64[s]', 'float32')}
date, weight = np.loadtxt('../data/william.dat', dtype, usecols=(0,1), unpack=True, skiprows=1)

print(date)
print(weight)
yhat = savgol_filter(weight, 7, 1)

plt.title("William's weight ")
plt.plot(date, weight, 'green')
plt.yticks(np.arange(min(date), max(date), np.timedelta64(7, 'D')))
plt.xlabel('date (yyyy-mm-dd)')

plt.plot(date, yhat, 'blue')
plt.yticks(np.arange(min(weight), max(weight)+1, 1.))
plt.ylabel('weight (lbs)')

plt.gca().xaxis.grid(True)
plt.gca().yaxis.grid(True)

daily_legend = mpatches.Patch(color='green', label='Daily')
weekly_legend = mpatches.Patch(color='blue', label='Weekly')
plt.legend(handles=[daily_legend, weekly_legend])
plt.show()

max_stars_repo_name: williamfgc/charts

max_issues_repo_name: williamfgc/charts

max_forks_repo_name: williamfgc/charts

reason: import numpy,from scipy


text:
import numpy as np
import torch
import time

from offpolicy.runner.rnn.base_runner import RecRunner

class MPERunner(RecRunner):
    """Runner class for Multiagent Particle Envs (MPE). See parent class for more information."""
    def __init__(self, config):
        super(MPERunner, self).__init__(config)
        self.collecter = self.shared_collect_rollout if self.share_policy else self.separated_collect_rollout
        # fill replay buffer with random actions
        # fill replay buffer with random actions
        num_warmup_episodes = max((self.batch_size, self.args.num_random_episodes))
        self.warmup(num_warmup_episodes)
        self.start = time.time()
        self.log_clear()

    def eval(self):
        """Collect episodes to evaluate the policy."""
        self.trainer.prep_rollout()
        eval_infos = {}
        eval_infos['average_episode_rewards'] = []

        for _ in range(self.args.num_eval_episodes):
            env_info = self.collecter( explore=False, training_episode=False, warmup=False)
            for k, v in env_info.items():
                eval_infos[k].append(v)

        self.log_env(eval_infos, suffix="eval_")

    # for mpe-simple_spread and mpe-simple_reference
    @torch.no_grad()
    def shared_collect_rollout(self, explore=True, training_episode=True, warmup=False):
        """
        Collect a rollout and store it in the buffer. All agents share a single policy.
        :param explore: (bool) whether to use an exploration strategy when collecting the episoide.
        :param training_episode: (bool) whether this episode is used for evaluation or training.
        :param warmup: (bool) whether this episode is being collected during warmup phase.

        :return env_info: (dict) contains information about the rollout (total rewards, etc).
        """
        env_info = {}
        # only 1 policy since all agents share weights
        p_id = "policy_0"
        policy = self.policies[p_id]

        env = self.env if training_episode or warmup else self.eval_env

        obs = env.reset()

        rnn_states_batch = np.zeros((self.num_envs * self.num_agents, self.hidden_size), dtype=np.float32)
        last_acts_batch = np.zeros((self.num_envs * self.num_agents, policy.output_dim), dtype=np.float32)

        # initialize variables to store episode information.
        episode_obs = {p_id : np.zeros((self.episode_length + 1, self.num_envs, self.num_agents, policy.obs_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_share_obs = {p_id : np.zeros((self.episode_length + 1, self.num_envs, self.num_agents, policy.central_obs_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_acts = {p_id : np.zeros((self.episode_length, self.num_envs, self.num_agents, policy.output_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_rewards = {p_id : np.zeros((self.episode_length, self.num_envs, self.num_agents, 1), dtype=np.float32) for p_id in self.policy_ids}
        episode_dones = {p_id : np.ones((self.episode_length, self.num_envs, self.num_agents, 1), dtype=np.float32) for p_id in self.policy_ids}
        episode_dones_env = {p_id : np.ones((self.episode_length, self.num_envs, 1), dtype=np.float32) for p_id in self.policy_ids}
        episode_avail_acts = {p_id : None for p_id in self.policy_ids}

        t = 0
        while t < self.episode_length:
            share_obs = obs.reshape(self.num_envs, -1)
            # group observations from parallel envs into one batch to process at once
            obs_batch = np.concatenate(obs)
            # get actions for all agents to step the env
            if warmup:
                # completely random actions in pre-training warmup phase
                acts_batch = policy.get_random_actions(obs_batch)
                # get new rnn hidden state
                _, rnn_states_batch, _ = policy.get_actions(obs_batch,
                                                            last_acts_batch,
                                                            rnn_states_batch)
            else:
                # get actions with exploration noise (eps-greedy/Gaussian)
                acts_batch, rnn_states_batch, _ = policy.get_actions(obs_batch,
                                                                    last_acts_batch,
                                                                    rnn_states_batch,
                                                                    t_env=self.total_env_steps,
                                                                    explore=explore)
            acts_batch = acts_batch if isinstance(acts_batch, np.ndarray) else acts_batch.cpu().detach().numpy()
            # update rnn hidden state
            rnn_states_batch = rnn_states_batch if isinstance(rnn_states_batch, np.ndarray) else rnn_states_batch.cpu().detach().numpy()
            last_acts_batch = acts_batch

            env_acts = np.split(acts_batch, self.num_envs)
            # env step and store the relevant episode information
            next_obs, rewards, dones, infos = env.step(env_acts)
            if training_episode:
                self.total_env_steps += self.num_envs

            dones_env = np.all(dones, axis=1)
            terminate_episodes = np.any(dones_env) or t == self.episode_length - 1

            episode_obs[p_id][t] = obs
            episode_share_obs[p_id][t] = share_obs
            episode_acts[p_id][t] = np.stack(env_acts)
            episode_rewards[p_id][t] = rewards
            episode_dones[p_id][t] = dones
            episode_dones_env[p_id][t] = dones_env
            t += 1

            obs = next_obs

            if terminate_episodes:
                break

        episode_obs[p_id][t] = obs
        episode_share_obs[p_id][t] = obs.reshape(self.num_envs, -1)

        if explore:
            self.num_episodes_collected += self.num_envs
            # push all episodes collected in this rollout step to the buffer
            self.buffer.insert(self.num_envs,
                               episode_obs,
                               episode_share_obs,
                               episode_acts,
                               episode_rewards,
                               episode_dones,
                               episode_dones_env,
                               episode_avail_acts)

        average_episode_rewards = np.mean(np.sum(episode_rewards[p_id], axis=0))
        env_info['average_episode_rewards'] = average_episode_rewards

        return env_info

    # for mpe-simple_speaker_listener
    @torch.no_grad()
    def separated_collect_rollout(self, explore=True, training_episode=True, warmup=False):
        """
        Collect a rollout and store it in the buffer. Each agent has its own policy.
        :param explore: (bool) whether to use an exploration strategy when collecting the episoide.
        :param training_episode: (bool) whether this episode is used for evaluation or training.
        :param warmup: (bool) whether this episode is being collected during warmup phase.

        :return env_info: (dict) contains information about the rollout (total rewards, etc).
        """
        env_info = {}
        env = self.env if training_episode or warmup else self.eval_env

        obs = env.reset()

        rnn_states = np.zeros((self.num_agents, self.num_envs, self.hidden_size), dtype=np.float32)

        last_acts = {p_id : np.zeros((self.num_envs, len(self.policy_agents[p_id]), self.policies[p_id].output_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_obs = {p_id : np.zeros((self.episode_length + 1, self.num_envs, len(self.policy_agents[p_id]), self.policies[p_id].obs_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_share_obs = {p_id : np.zeros((self.episode_length + 1, self.num_envs, len(self.policy_agents[p_id]), self.policies[p_id].central_obs_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_acts = {p_id : np.zeros((self.episode_length, self.num_envs, len(self.policy_agents[p_id]), self.policies[p_id].output_dim), dtype=np.float32) for p_id in self.policy_ids}
        episode_rewards = {p_id : np.zeros((self.episode_length, self.num_envs, len(self.policy_agents[p_id]), 1), dtype=np.float32) for p_id in self.policy_ids}
        episode_dones = {p_id : np.ones((self.episode_length, self.num_envs, len(self.policy_agents[p_id]), 1), dtype=np.float32) for p_id in self.policy_ids}
        episode_dones_env = {p_id : np.ones((self.episode_length, self.num_envs, 1), dtype=np.float32) for p_id in self.policy_ids}
        episode_avail_acts = {p_id : None for p_id in self.policy_ids}

        t = 0
        while t < self.episode_length:
            for agent_id, p_id in zip(self.agent_ids, self.policy_ids):
                policy = self.policies[p_id]
                agent_obs = np.stack(obs[:, agent_id])
                share_obs = np.concatenate([obs[0, i] for i in range(self.num_agents)]).reshape(self.num_envs,
                                                                                -1).astype(np.float32)
                # get actions for all agents to step the env
                if warmup:
                    # completely random actions in pre-training warmup phase
                    # [parallel envs, agents, dim]
                    act = policy.get_random_actions(agent_obs)
                    # get new rnn hidden state
                    _, rnn_state, _ = policy.get_actions(agent_obs,
                                                        last_acts[p_id][:, 0],
                                                        rnn_states[agent_id])
                else:
                    # get actions with exploration noise (eps-greedy/Gaussian)
                    if self.algorithm_name == "rmasac":
                        act, rnn_state, _ = policy.get_actions(agent_obs,
                                                                last_acts[p_id],
                                                                rnn_states[agent_id],
                                                                sample=explore)
                    else:
                        act, rnn_state, _ = policy.get_actions(agent_obs,
                                                                last_acts[p_id].squeeze(axis=0),
                                                                rnn_states[agent_id],
                                                                t_env=self.total_env_steps,
                                                                explore=explore)
                # update rnn hidden state
                rnn_states[agent_id] = rnn_state if isinstance(rnn_state, np.ndarray) else rnn_state.cpu().detach().numpy()
                last_acts[p_id] = np.expand_dims(act, axis=1) if isinstance(act, np.ndarray) else np.expand_dims(act.cpu().detach().numpy(), axis=1)

                episode_obs[p_id][t] = agent_obs
                episode_share_obs[p_id][t] = share_obs
                episode_acts[p_id][t] = act

            env_acts = []
            for i in range(self.num_envs):
                env_act = []
                for p_id in self.policy_ids:
                    env_act.append(last_acts[p_id][i, 0])
                env_acts.append(env_act)

            # env step and store the relevant episode information
            next_obs, rewards, dones, infos = env.step(env_acts)

            dones_env = np.all(dones, axis=1)
            terminate_episodes = np.any(dones_env) or t == self.episode_length - 1
            if terminate_episodes:
                dones_env = np.ones_like(dones_env).astype(bool)

            for agent_id, p_id in zip(self.agent_ids, self.policy_ids):
                episode_rewards[p_id][t] = np.expand_dims(rewards[:, agent_id], axis=1)
                episode_dones[p_id][t] = np.expand_dims(dones[:, agent_id], axis=1)
                episode_dones_env[p_id][t] = dones_env

            obs = next_obs
            t += 1

            if training_episode:
                self.total_env_steps += self.num_envs

            if terminate_episodes:
                break

        for agent_id, p_id in zip(self.agent_ids, self.policy_ids):
            episode_obs[p_id][t] = np.stack(obs[:, agent_id])
            episode_share_obs[p_id][t] = np.concatenate([obs[0, i] for i in range(self.num_agents)]).reshape(self.num_envs,
                                                                                -1).astype(np.float32)

        if explore:
            self.num_episodes_collected += self.num_envs
            self.buffer.insert(self.num_envs, episode_obs, episode_share_obs, episode_acts, episode_rewards, episode_dones, episode_dones_env, episode_avail_acts)

        average_episode_rewards = []
        for p_id in self.policy_ids:
            average_episode_rewards.append(np.mean(np.sum(episode_rewards[p_id], axis=0)))

        env_info['average_episode_rewards'] = np.mean(average_episode_rewards)

        return env_info

    def log(self):
        """See parent class."""
        end = time.time()
        print("\n Env {} Algo {} Exp {} runs total num timesteps {}/{}, FPS {}.\n"
              .format(self.args.scenario_name,
                      self.algorithm_name,
                      self.args.experiment_name,
                      self.total_env_steps,
                      self.num_env_steps,
                      int(self.total_env_steps / (end - self.start))))
        for p_id, train_info in zip(self.policy_ids, self.train_infos):
            self.log_train(p_id, train_info)

        self.log_env(self.env_infos)
        self.log_clear()

    def log_clear(self):
        """See parent class."""
        self.env_infos = {}

        self.env_infos['average_episode_rewards'] = []

max_stars_repo_name: Maxtoq/off-policy

max_issues_repo_name: jc-bao/off-policy

max_forks_repo_name: jc-bao/off-policy


text:
import numpy as np
from keras.preprocessing.image import ImageDataGenerator, img_to_array, load_img
from keras.models import Sequential, Model, load_model
from keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, Input
from keras.applications.inception_v3 import InceptionV3
from keras import optimizers
from keras.utils.np_utils import to_categorical
from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
import math
import configparser
from datetime import datetime, timedelta
import os

config = configparser.ConfigParser()
config.read('conf/application.ini')

app_config = config['training']

# get the initial epoch number from the checkpoint name
def get_init_epoch(checkpoint_path):
    filename = os.path.basename(checkpoint_path)
    filename = os.path.splitext(filename)[0]
    init_epoch = filename.split("-")[1]
    return int(init_epoch)

# dimensions of our images.
img_width = app_config.getint('img_width')
img_height = app_config.getint('img_height')

top_model_weights_path = app_config.get('top_model_weights_path')
train_data_dir = 'data/train'
validation_data_dir = 'data/validation'

final_model_path = app_config.get('final_model_path')

# number of epochs to train top model
# epochs = app_config.getint('fine_tune_epochs')
epochs = 100

# batch size used by flow_from_directory and predict_generator
batch_size = app_config.getint('fine_tune_batch_size')

# the checkpoint to load and continue from
checkpoint_to_load = "data/models/checkpoints/model-74-0.81.h5"
# get the epoch number to continue from
init_epoch = get_init_epoch(checkpoint_to_load)

# prepare data augmentation configuration
datagen = ImageDataGenerator(
                    rescale=1. / 255,
                    rotation_range=40,
                    shear_range=0.2,
                    zoom_range=0.2,
                    horizontal_flip=True,
                    fill_mode='nearest',
                    validation_split=0.25)

train_generator = datagen.flow_from_directory(
                    train_data_dir,
                    target_size=(img_width, img_height),
                    batch_size=batch_size,
                    class_mode='categorical',
                    interpolation='lanczos',
                    subset='training')

validation_generator = datagen.flow_from_directory(
                    train_data_dir,
                    target_size=(img_width, img_height),
                    batch_size=batch_size,
                    class_mode='categorical',
                    interpolation='lanczos',
                    subset='validation')

nb_train_samples = len(train_generator.filenames)
nb_validation_samples = len(validation_generator.filenames)
num_classes = len(train_generator.class_indices)

print("[Info] Dataset stats: Training: {} Validation: {}".format(nb_train_samples, nb_validation_samples))
print("[Info] Number of Classes: {}".format(num_classes))
print("[Info] Class Labels: {}".format(train_generator.class_indices))

train_steps = int(math.ceil(nb_train_samples / batch_size))
validation_steps = int(math.ceil(nb_validation_samples / batch_size))

model = load_model(checkpoint_to_load)

filepath = "data/models/checkpoints/model-{epoch:02d}-{val_acc:.2f}.h5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='max')
callbacks_list = [checkpoint]

ft_start_time = datetime.now()
print("[Info] Model Fine-tuning started at: {}".format(ft_start_time))

history = model.fit_generator(
    train_generator,
    steps_per_epoch=train_steps,
    epochs=epochs,
    validation_data=validation_generator,
    validation_steps=validation_steps,
    initial_epoch=init_epoch,
    callbacks=callbacks_list)

(eval_loss, eval_accuracy) = model.evaluate_generator(
    validation_generator,
    steps=validation_steps)

ft_end_time = datetime.now()
print("[Info] Model Fine-tuning completed at: {}".format(ft_end_time))
ft_duration = ft_end_time - ft_start_time
print("[Info] Total time for Fine-tuning: {}".format(ft_duration))

print("[Info] Saving final model to disk: {}".format(final_model_path))
model.save(final_model_path)

print("\n")

print("[INFO] accuracy: {:.2f}%".format(eval_accuracy * 100))
print("[INFO] Loss: {}".format(eval_loss))

plt.style.use('ggplot')

plt.figure(1)

# summarize history for accuracy

plt.subplot(211)
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='lower right')

# summarize history for loss

plt.subplot(212)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training', 'Validation'], loc='upper right')

plt.tight_layout()
# plt.show()
plt.savefig('data/models/fine-tune-031.png', bbox_inches='tight', dpi=300)


text:
"""The second stage of SMNAS."""
import copy
import logging
from random import SystemRandom
from vega.common import ClassFactory, ClassType
from vega.core.search_algs import SearchAlgorithm
import numpy as np
from .conf import SpNasConfig
from vega.report import ReportServer

@ClassFactory.register(ClassType.SEARCH_ALGORITHM)
class SpNasS(SearchAlgorithm):
    """First Stage of SPNAS."""

    config = SpNasConfig()

    def __init__(self, search_space=None):
        super(SpNasS, self).__init__(search_space)
        self.sample_count = 0
        self.max_sample = self.config.max_sample

    @property
    def is_completed(self):
        """Check sampling if finished."""
        return self.sample_count >= self.max_sample

    def search(self):
        """Search a sample."""
        pareto_records = ReportServer().get_pareto_front_records(choice='normal')
        best_record = pareto_records[0] if pareto_records else None
        desc = self.search_space.sample()
        arch_code = desc.get('network.backbone.code')
        if best_record:
            desc['network.backbone.weight_file'] = best_record.weights_file
        desc['network.backbone.code'] = self._mutate_serialnet(arch_code)
        self.sample_count += 1
        logging.info("desc:{}".format(desc))
        return dict(worker_id=self.sample_count, encoded_desc=desc)

    @property
    def max_samples(self):
        """Return the max number of samples."""
        return self.max_sample

    def _mutate_serialnet(self, arch):
        """Swap & Expend operation in Serial-level searching."""

        def is_valid(arch):
            stages = arch.split('-')
            for stage in stages:
                if len(stage) == 0:
                    return False
            return True

        def expend(arc):
            idx = np.random.randint(low=1, high=len(arc))
            arc = arc[:idx] + '1' + arc[idx:]
            return arc, idx

        def swap(arc, len_step=3):
            is_not_valid = True
            arc_origin = copy.deepcopy(arc)
            temp = arc.split('-')
            num_insert = len(temp) - 1
            while is_not_valid or arc == arc_origin:
                next_start = 0
                arc = list(''.join(temp))
                for i in range(num_insert):
                    pos = arc_origin[next_start:].find('-') + next_start
                    if arc_origin[pos] == '-':
                        max_step = min(len_step, max(len(temp[i]), len(temp[i + 1])))
                        step_range = list(range(-1 * max_step, max_step))
                        step = SystemRandom().choice(step_range)
                        next_start = pos + 1
                        pos = pos + step
                        arc.insert(pos, '-')
                    else:
                        raise ValueError("Wrong '-' is found!")
                arc = ''.join(arc)
                is_not_valid = (not is_valid(arc))
            return arc

        arch_origin = arch
        success = False
        k = 0
        while not success:
            k += 1
            arch = arch_origin
            ops = []
            for i in range(self.config.num_mutate):
                op_idx = np.random.randint(low=0, high=2)
                adds_thresh_ = self.config.add_stage_ratio if len(arch.split('-')) < self.config.max_stages else 1
                if op_idx == 0 and SystemRandom().random() > self.config.expend_ratio:
                    arch, idx = expend(arch)
                    arch, idx = expend(arch)
                    ops.append('expend')
                elif op_idx == 1:
                    arch = swap(arch)
                    ops.append('swap')
                elif op_idx == 2 and SystemRandom().random() > adds_thresh_:
                    arch = arch + '-1'
                    ops.append('add stage')
                else:
                    ops.append('Do Nothing')
            success = arch != arch_origin
            flag = 'Success' if success else 'Failed'
            logging.info('Serial-level Sample{}: {}. {}.'.format(k, ' -> '.join(ops), flag))
        return arch

max_stars_repo_name: sisrfeng/vega

max_issues_repo_name: NiuRc/vega

max_forks_repo_name: NiuRc/vega


text:
import cv2
import numpy as np
import random
from image_stitching.get_homography import calculateHomography

def RANSAC_filter(ptsA, ptsB):

#randomly pick 4 pairs

    #number of pairs before filtering
  num=ptsA.shape[0]

  maxInliers=[]
  for it in range (0,200):
      picked_four=[]
      for i in range(0,4):
          idx=random.randrange(0,num)
          pair=(ptsA[idx], ptsB[idx])
          picked_four.append(pair)
      #put the 4 pairs into homography calculation
      h=calculateHomography(picked_four)

      inliers = []
      #pick one pair
      for j in range (0,num):
          pair=(ptsA[j], ptsB[j])
          error=difference(pair,h)
          if error <5:
             inliers.append(pair)

      if len(inliers) > len(maxInliers):
          maxInliers = inliers
          finalH = h

      print ("number of pairs: ",num, "max inliers: ", len(maxInliers) )

  print ('ok')
  return (h, maxInliers)

def difference(pair,h):

        pta=pair[0]
        ptb=pair[1]

        p1 = np.transpose(np.matrix([pta[0], pta[1], 1]))
        est_p2 = np.dot(h, p1)
        est_p2 = (1 / est_p2.item(2)) * est_p2

        p2 = np.transpose(np.matrix([ptb[0], ptb[1], 1]))
        diff=p2 - est_p2

        error=np.linalg.norm(diff)
        return  error