text:
# ==================================================================================================
# Copyright (c) 2021, Jennifer Williams and Yamagishi Laboratory, National Institute of Informatics
# Author: Jennifer Williams (j.williams@ed.ac.uk)
# All rights reserved.
# ==================================================================================================

import math, pickle, os
import numpy as np
import torch
from torch.autograd import Variable
from torch import optim
from torch.utils.data import DataLoader
import torch.nn as nn
import torch.nn.functional as F
from utils.dsp import *
import sys
import time
from layers.overtone import Overtone
from layers.vector_quant import VectorQuant
from layers.downsampling_encoder import DownsamplingEncoder
import utils.env as env
import utils.logger as logger
import random
import pytorch_warmup as warmup

class Model(nn.Module) :
    def __init__(self, rnn_dims, fc_dims, global_decoder_cond_dims, upsample_factors, normalize_vq=False,noise_x=False, noise_y=False):
        super().__init__()
        self.n_classes = 256
        self.overtone = Overtone(rnn_dims, fc_dims, 128, global_decoder_cond_dims)
        self.vq = VectorQuant(1, 512, 128, normalize=normalize_vq)
        self.noise_x = noise_x
        self.noise_y = noise_y
        encoder_layers = [
            (2, 4, 1),
            (2, 4, 1),
            (2, 4, 1),
            (1, 4, 1),
            (2, 4, 1),
            (1, 4, 1),
            (2, 4, 1),
            (1, 4, 1),
            (2, 4, 1),
            (1, 4, 1),
            ]
        self.encoder = DownsamplingEncoder(128, encoder_layers)
        self.frame_advantage = 15
        self.num_params()
        self.left = self.pad_left()
        self.win = 16 * self.total_scale()
        self.right = self.pad_right()

    def forward(self, global_decoder_cond, x, samples):
        continuous = self.encoder(samples)
        discrete, vq_pen, encoder_pen, entropy, phn_codes = self.vq(continuous.unsqueeze(2))
        return self.overtone(x, discrete.squeeze(2), global_decoder_cond), vq_pen.mean(), encoder_pen.mean(), entropy

    def after_update(self):
        self.overtone.after_update()
        self.vq.after_update()

    def generate(self, samples, global_decoder_cond, deterministic=False, use_half=False, verbose=False):
        self.eval()
        with torch.no_grad() :
            continuous = self.encoder(samples)
            discrete, vq_pen, encoder_pen, entropy = self.vq(continuous.unsqueeze(2))
            code_x = discrete.squeeze(2)
            output = self.overtone.generate(code_x, global_decoder_cond, use_half=use_half, verbose=verbose)
        return output

    def forward_validate(self, global_decoder_cond, x, samples):
        self.eval()
        with torch.no_grad() :
            continuous = self.encoder(samples)
            discrete, vq_pen, encoder_pen, entropy, phn_codes = self.vq(continuous.unsqueeze(2))
        self.train()
        return self.overtone(x, discrete.squeeze(2), global_decoder_cond), vq_pen.mean(), encoder_pen.mean()

    def num_params(self) :
        parameters = filter(lambda p: p.requires_grad, self.parameters())
        parameters = sum([np.prod(p.size()) for p in parameters]) / 1_000_000
        logger.log('Trainable Parameters: %.3f million' % parameters)

    def load_state_dict(self, dict, strict=True):
        if strict:
            return super().load_state_dict(self.upgrade_state_dict(dict))
        else:
            my_dict = self.state_dict()
            new_dict = {}
            for key, val in dict.items():
                if key not in my_dict:
                    logger.log(f'Ignoring {key} because no such parameter exists')
                elif val.size() != my_dict[key].size():
                    logger.log(f'Ignoring {key} because of size mismatch')
                else:
                    logger.log(f'Loading {key}')
                    new_dict[key] = val
            return super().load_state_dict(new_dict, strict=False)

    def upgrade_state_dict(self, state_dict):
        out_dict = state_dict.copy()
        return out_dict

    def freeze_encoder(self):
        for name, param in self.named_parameters():
            if name.startswith('encoder.') or name.startswith('vq.'):
                logger.log(f'Freezing {name}')
                param.requires_grad = False
            else:
                logger.log(f'Not freezing {name}')

    def pad_left(self):
        return max(self.pad_left_decoder(), self.pad_left_encoder())

    def pad_left_decoder(self):
        return self.overtone.pad()

    def pad_left_encoder(self):
        return self.encoder.pad_left + (self.overtone.cond_pad - self.frame_advantage) * self.encoder.total_scale

    def pad_right(self):
        return self.frame_advantage * self.encoder.total_scale

    def total_scale(self):
        return self.encoder.total_scale

    def tmp_func(self, batch):
        return env.collate_multispeaker_samples(self.left, self.win, self.right, batch)

    def tmp_func2(self, batch):
        return env.collate_multispeaker_samples_forward(self.left, self.win, self.right, batch)

    def do_train(self, paths, dataset, optimiser, epochs, batch_size, num_workers, step, train_sampler, device, lr=1e-4,  spk_lr=0.01, valid_index=[], use_half=False, do_clip=False):

        if use_half:
            import apex
            optimiser = apex.fp16_utils.FP16_Optimizer(optimiser, dynamic_loss_scale=True)
        for p in optimiser.param_groups : p['lr'] = lr
        criterion = nn.NLLLoss().cuda()
        k = 0
        saved_k = 0
        pad_left = self.pad_left()
        pad_left_encoder = self.pad_left_encoder()
        pad_left_decoder = self.pad_left_decoder()
        if self.noise_x:
            extra_pad_right = 127
        else:
            extra_pad_right = 0
        pad_right = self.pad_right() + extra_pad_right
        window = 16 * self.total_scale()
        logger.log(f'pad_left={pad_left_encoder}|{pad_left_decoder}, pad_right={pad_right}, total_scale={self.total_scale()}')

        # from haoyu: slow start for the first 10 epochs
        lr_lambda = lambda epoch: min((epoch) / 10 , 1)
        lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimiser, lr_lambda=lr_lambda)
        ## warmup is useful !!!
        # warmup_scheduler = warmup.UntunedLinearWarmup(optimiser)
        # warmup_scheduler.last_step = -1

        for e in range(epochs) :
            self.left = self.pad_left()
            self.win = 16 * self.total_scale()
            self.right = pad_right

            trn_loader = DataLoader(dataset, collate_fn=self.tmp_func, batch_size=batch_size,
                num_workers=num_workers, shuffle=(train_sampler is None),  sampler=train_sampler, pin_memory=True)

            start = time.time()
            running_loss_c = 0.
            running_loss_f = 0.
            running_loss_vq = 0.
            running_loss_vqc = 0.
            running_entropy = 0.
            running_max_grad = 0.
            running_max_grad_name = ""

            iters = len(trn_loader)

            for i, (speaker, wave16) in enumerate(trn_loader) :

                speaker = speaker.cuda(device)
                wave16 = wave16.cuda(device)

                coarse = (wave16 + 2**15) // 256
                fine = (wave16 + 2**15) % 256

                coarse_f = coarse.float() / 127.5 - 1.
                fine_f = fine.float() / 127.5 - 1.
                total_f = (wave16.float() + 0.5) / 32767.5

                if self.noise_y:
                    noisy_f = total_f * (0.02 * torch.randn(total_f.size(0), 1).cuda(device)).exp() + 0.003 * torch.randn_like(total_f)
                else:
                    noisy_f = total_f

                if use_half:
                    coarse_f = coarse_f.half()
                    fine_f = fine_f.half()
                    noisy_f = noisy_f.half()

                x = torch.cat([
                    coarse_f[:, pad_left-pad_left_decoder:-pad_right].unsqueeze(-1),
                    fine_f[:, pad_left-pad_left_decoder:-pad_right].unsqueeze(-1),
                    coarse_f[:, pad_left-pad_left_decoder+1:1-pad_right].unsqueeze(-1),
                    ], dim=2)
                y_coarse = coarse[:, pad_left+1:1-pad_right]
                y_fine = fine[:, pad_left+1:1-pad_right]

                if self.noise_x:
                    # Randomly translate the input to the encoder to encourage
                    # translational invariance
                    total_len = coarse_f.size(1)
                    translated = []
                    for j in range(coarse_f.size(0)):
                        shift = random.randrange(256) - 128
                        translated.append(noisy_f[j, pad_left-pad_left_encoder+shift:total_len-extra_pad_right+shift])
                    translated = torch.stack(translated, dim=0)
                else:
                    translated = noisy_f[:, pad_left-pad_left_encoder:]
                p_cf, vq_pen, encoder_pen, entropy = self(speaker, x, translated)
                p_c, p_f = p_cf
                loss_c = criterion(p_c.transpose(1, 2).float(), y_coarse)
                loss_f = criterion(p_f.transpose(1, 2).float(), y_fine)
                encoder_weight = 0.01 * min(1, max(0.1, step / 1000 - 1))
                loss = loss_c + loss_f + vq_pen + encoder_weight * encoder_pen

                optimiser.zero_grad()
                if use_half:
                    optimiser.backward(loss)
                    if do_clip:
                        raise RuntimeError("clipping in half precision is not implemented yet")
                else:
                    loss.backward()
                    if do_clip:
                        max_grad = 0
                        max_grad_name = ""
                        for name, param in self.named_parameters():
                            if param.grad is not None:
                                param_max_grad = param.grad.data.abs().max()
                                if param_max_grad > max_grad:
                                    max_grad = param_max_grad
                                    max_grad_name = name
                                if 1000000 < param_max_grad:
                                    logger.log(f'Very large gradient at {name}: {param_max_grad}')
                        if 100 < max_grad:
                            for param in self.parameters():
                                if param.grad is not None:
                                    if 1000000 < max_grad:
                                        param.grad.data.zero_()
                                    else:
                                        param.grad.data.mul_(100 / max_grad)
                        if running_max_grad < max_grad:
                            running_max_grad = max_grad
                            running_max_grad_name = max_grad_name

                        if 100000 < max_grad:
                            torch.save(self.state_dict(), "bad_model.pyt")
                            raise RuntimeError("Aborting due to crazy gradient (model saved to bad_model.pyt)")
                optimiser.step()
                if e==0 and i==0:
                     lr_scheduler.step()
                     print("schedulre!")
                # lr_scheduler.step()
                # warmup_scheduler.dampen()
                running_loss_c += loss_c.item()
                running_loss_f += loss_f.item()
                running_loss_vq += vq_pen.item()
                running_loss_vqc += encoder_pen.item()
                running_entropy += entropy

                self.after_update()

                speed = (i + 1) / (time.time() - start)
                avg_loss_c = running_loss_c / (i + 1)
                avg_loss_f = running_loss_f / (i + 1)
                avg_loss_vq = running_loss_vq / (i + 1)
                avg_loss_vqc = running_loss_vqc / (i + 1)
                avg_entropy = running_entropy / (i + 1)

                step += 1
                k = step // 1000
                logger.status(f'Epoch:{e+1}/{epochs} -- Bt:{i+1}/{iters}'
f' -- Lc={avg_loss_c:#.4} -- Lf={avg_loss_f:#.4} -- Lvq={avg_loss_vq:#.4} -- Lvqc={avg_loss_vqc:#.4}'
f' -- E:{avg_entropy:#.4}'
f' -- G:{running_max_grad:#.1} {running_max_grad_name}-- S:{speed:#.4} steps/sec -- Step: {k}k\n\n')
            os.makedirs(paths.checkpoint_dir, exist_ok=True)
            torch.save(self.state_dict(), paths.model_path())
            np.save(paths.step_path(), step)
            logger.log_current_status()
            logger.log(f' <saved>; w[0][0] = {self.overtone.wavernn.gru.weight_ih_l0[0][0]}')
            if k > saved_k + 5:
                torch.save(self.state_dict(), paths.model_hist_path(step))
                saved_k = k
                val_loss_c, val_loss_f,val_loss_vq, val_loss_vqc = self.validate2(paths, step, dataset.path, valid_index, device)
                logger.status(f'Valid: -- Lc={val_loss_c:#.4} -- Lf={val_loss_f:#.4} -- Lvq={val_loss_vq:#.4} -- Lvqc={val_loss_vqc:#.4}')

    def validate(self, paths, step, data_path, test_index, device, deterministic=False, use_half=False, verbose=False):
        k = step // 1000
        os.makedirs(paths.gen_path(), exist_ok=True)
        all_files = [(i, name) for (i, speaker) in enumerate(test_index) for name in speaker]
        batch_size = 2
        num_batches, leftovers = divmod(len(all_files), batch_size)
        num_batches = 1
        print("Generating: ", num_batches*batch_size)
        for i in range(0, num_batches):
            start = i*batch_size
            end = (i+1)*batch_size+10
            # get a different speaker
            batch = all_files[start:end:10]
            speakers_onehot = [(np.arange(109) == speaker_id).astype(np.long) for speaker_id, name in batch]
            audio_files = [np.load(f'{data_path}/{speaker_id}/{name}.npy') for speaker_id, name in batch]
            n_points = len(audio_files)
            gt = [(x.astype(np.float32) + 0.5) / (2**15 - 0.5) for x in audio_files]
            extended = [np.concatenate([np.zeros(self.pad_left_encoder(), dtype=np.float32), x, np.zeros(self.pad_right(), dtype=np.float32)]) for x in gt]
            speakers = [torch.FloatTensor(speaker.astype(np.float32)) for speaker in speakers_onehot]
            maxlen = max([len(x) for x in extended])
            aligned = [torch.cat([torch.FloatTensor(x).cuda(), torch.zeros(maxlen-len(x)).cuda()]) for x in extended]
            A = torch.stack(speakers+ list(reversed(speakers)), dim=0).cuda(device)
            B = torch.stack(aligned + aligned, dim=0).cuda(device)
            out = self.forward_generate(A, B, verbose=verbose, use_half=use_half)
#            logger.log(f'out: {out.size()}')
            for i, x in enumerate(gt) :
                librosa.output.write_wav(f'{paths.gen_path()}/{k}k_steps_{i}_target.wav', x, sr=sample_rate)
                audio = out[i][:len(x)].cpu().numpy()
                librosa.output.write_wav(f'{paths.gen_path()}/{k}k_steps_{i}_generated.wav', audio, sr=sample_rate)
                audio_tr = out[n_points+i][:len(x)].cpu().numpy()
                librosa.output.write_wav(f'{paths.gen_path()}/{k}k_steps_{i}_transferred.wav', audio_tr, sr=sample_rate)


text:
import torch
import torch.nn as nn
from torch.utils.data import Dataset
from torch.autograd import Variable
import os
import cv2
import numpy as np
from torchvision import datasets, models, transforms

label_len = 36
# vocab="<,.+:-?$ <aAàÀảẢãÃáÁạẠăĂằẰẳẲẵẴắẮặẶâÂầẦẩẨẫẪấẤậẬbBcCdDđĐeEèÈẻẺẽẼéÉẹẸêÊềỀểỂễỄếẾệỆfFgGhHiIìÌỉỈĩĨíÍịỊjJkKlLmMnNoOòÒỏỎõÕóÓọỌôÔồỒổỔỗỖốỐộỘơƠờỜởỞỡỠớỚợỢpPqQrRsStTuUùÙủỦũŨúÚụỤưƯừỪửỬữỮứỨựỰvVwWxXyYỳỲỷỶỹỸýÝỵỴzZ0123456789!>"
vocab='<aAàÀảẢãÃáÁạẠăĂằẰẳẲẵẴắẮặẶâÂầẦẩẨẫẪấẤậẬbBcCdDđĐeEèÈẻẺẽẼéÉẹẸêÊềỀểỂễỄếẾệỆfFgGhHiIìÌỉỈĩĨíÍịỊjJkKlLmMnNoOòÒỏỎõÕóÓọỌôÔồỒổỔỗỖốỐộỘơƠờỜởỞỡỠớỚợỢpPqQrRsStTuUùÙủỦũŨúÚụỤưƯừỪửỬữỮứỨựỰvVwWxXyYỳỲỷỶỹỸýÝỵỴzZ0123456789!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ >'
# start symbol <
# end symbol >
char2token = {"PAD":0}
token2char = {0:"PAD"}
for i, c in enumerate(vocab):
    char2token[c] = i+1
    token2char[i+1] = c

def illegal(label):
    if len(label) > label_len-1:
        return True
    for l in label:
        if l not in vocab[1:-1]:
            return True
    return False

class ListDataset(Dataset):
    def __init__(self, fname):
        self.lines = []
        # if not isinstance(fname, list):
        #     fname = [fname]
        # for f in fname:
        lines = open('train_line_annotation.txt','r',encoding='utf8').readlines()
        self.lines += [i for i in lines if not illegal(i.strip('\n').split('\t')[1])]

    def __len__(self):
        return len(self.lines)

    def __getitem__(self, index):
        '''
        line: image path\tlabel
        '''
        line = self.lines[index]
        img_path, label_y_str = line.strip('\n').split('\t')
        img = cv2.imread(img_path) / 255.
        img=cv2.resize(img,(500,40))
        # cv2.imshow('s',img)
        # cv2.waitKey(0)
        # Channels-first
        img = np.transpose(img, (2, 0, 1))
        # As pytorch tensor
        img = torch.from_numpy(img).float()
        label = np.zeros(label_len, dtype=int)
        for i, c in enumerate('<'+label_y_str):
            label[i] = char2token[c]
        label = torch.from_numpy(label)

        label_y = np.zeros(label_len, dtype=int)
        for i, c in enumerate(label_y_str+'>'):
            label_y[i] = char2token[c]
        label_y = torch.from_numpy(label_y)

        return img, label_y, label

def subsequent_mask(size):
    "Mask out subsequent positions."
    attn_shape = (1, size, size)
    subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')
    return torch.from_numpy(subsequent_mask) == 0

def make_std_mask(tgt, pad):
        "Create a mask to hide padding and future words."
        tgt_mask = (tgt != pad).unsqueeze(-2)
        tgt_mask = tgt_mask & Variable(
            subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data))
        return tgt_mask

class Batch:
    "Object for holding a batch of data with mask during training."
    def __init__(self, imgs, trg_y, trg, pad=0):
        self.imgs = Variable(imgs.cuda(), requires_grad=False)
        self.src_mask = Variable(torch.from_numpy(np.ones([imgs.size(0), 1, 96], dtype=np.bool)).cuda())
        if trg is not None:
            self.trg = Variable(trg.cuda(), requires_grad=False)
            self.trg_y = Variable(trg_y.cuda(), requires_grad=False)
            self.trg_mask = \
                self.make_std_mask(self.trg, pad)
            self.ntokens = (self.trg_y != pad).data.sum()

    @staticmethod
    def make_std_mask(tgt, pad):
        "Create a mask to hide padding and future words."
        tgt_mask = (tgt != pad).unsqueeze(-2)
        tgt_mask = tgt_mask & Variable(
            subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data))
        return Variable(tgt_mask.cuda(), requires_grad=False)

class FeatureExtractor(nn.Module):
    def __init__(self, submodule, name):
        super(FeatureExtractor, self).__init__()
        self.submodule = submodule
        self.name = name
    def forward(self, x):
        for name, module in self.submodule._modules.items():
            x = module(x)
            if name is self.name:
                b = x.size(0)
                c = x.size(1)
                return x.view(b, c, -1).permute(0, 2, 1)
        return None

if __name__=='__main__':
    listdataset = ListDataset('your-lines')
    dataloader = torch.utils.data.DataLoader(listdataset, batch_size=2, shuffle=False, num_workers=0)
    for epoch in range(1):
        for batch_i, (imgs, labels_y, labels) in enumerate(dataloader):
            continue

ext: py

max_stars_repo_name: nguyenvanducb/ocr

max_issues_repo_name: nguyenvanducb/ocr

max_forks_repo_name: nguyenvanducb/ocr

reason: import numpy


text:
#!/usr/bin/env python3

import sys
import numpy
import pylab

pylab.rcParams["font.size"]=8
pylab.rcParams["legend.fontsize"]=8
#pylab.rcParams["lines.linewidth"]=1
#pylab.rcParams["axes.linewidth"]=2
#pylab.rcParams["axes.labelsize"]="large"
#pylab.rcParams["axes.labelweight"]="bold"
pylab.rcParams["xtick.major.size"]=0
pylab.rcParams["xtick.minor.size"]=0
pylab.rcParams["ytick.major.size"]=0
pylab.rcParams["ytick.minor.size"]=0
#pylab.rcParams["xtick.direction"]="out"
#pylab.rcParams["ytick.direction"]="out"

spike1=numpy.loadtxt("taudelta5s/spikeE.csv", delimiter=",")
wbias1=numpy.loadtxt("taudelta5s/wbias.csv", delimiter=",")
spike2=numpy.loadtxt("taudelta0.5s/spikeE.csv", delimiter=",")
wbias2=numpy.loadtxt("taudelta0.5s/wbias.csv", delimiter=",")
spike3=numpy.loadtxt("taudelta0.5s_weakSTP/spikeE.csv", delimiter=",")
wbias3=numpy.loadtxt("taudelta0.5s_weakSTP/wbias.csv", delimiter=",")

maxval=numpy.max([numpy.max(spike1[:, 1:]), numpy.max(spike2[:, 1:]), numpy.max(spike3[:, 1:])])

pylab.clf()
pylab.figure(figsize=(4,5))

#activity
colormap="hot" #"jet", "bwr"

pylab.subplot2grid((5,1),(0,0), rowspan=1)
pylab.imshow(spike1[:, 1:].T, aspect="auto", interpolation="none", cmap=colormap, extent=[spike1[0,0]/1000.0, spike1[-1,0]/1000.0, len(spike1[0,1:]), 1])
pylab.clim([0.0, maxval])
pylab.xticks([])
#pylab.xlabel("Time [s]")
pylab.ylabel("Neuron #")
pylab.title("Condition 1: Slow weight change, strong STP", fontsize=8, color="red")

pylab.subplot2grid((5,1),(1,0), rowspan=1)
pylab.imshow(spike2[:, 1:].T, aspect="auto", interpolation="none", cmap=colormap, extent=[spike2[0,0]/1000.0, spike2[-1,0]/1000.0, len(spike2[0,1:]), 1])
pylab.clim([0.0, maxval])
pylab.xticks([])
#pylab.xlabel("Time [s]")
pylab.ylabel("Neuron #")
pylab.title("Condition 2: Fast weight change, strong STP", fontsize=8, color="blue")

pylab.subplot2grid((5,1),(2,0), rowspan=1)
pylab.imshow(spike3[:, 1:].T, aspect="auto", interpolation="none", cmap=colormap, extent=[spike3[0,0]/1000.0, spike3[-1,0]/1000.0, len(spike3[0,1:]), 1])
pylab.clim([0.0, maxval])
pylab.xticks([])
#pylab.xlabel("Time [s]")
pylab.ylabel("Neuron #")
pylab.title("Condition 3: Fast weight change, weak STP", fontsize=8, color="green")

#wbias
time=wbias1[:,0]/1000.0
wlim=70.0

pylab.subplot2grid((5,1),(3,0), rowspan=1)
pylab.plot(time, wbias1[:,1], label="Condition 1", color="red")
pylab.plot(time, wbias2[:,1], label="Condition 2", color="blue")
pylab.plot(time, wbias3[:,1], label="Condition 3", color="green")
pylab.plot(time, time*0.0, "--", color="black")
pylab.ylabel("Weight bias\n(Neuron #100)")
pylab.ylim([-wlim, wlim])
pylab.yticks([0.0])
pylab.xticks([])
#pylab.xlabel("Time [s]")

pylab.subplot2grid((5,1),(4,0), rowspan=1)
pylab.plot(time, wbias1[:,2], label="Condition 1", color="red")
pylab.plot(time, wbias2[:,2], label="Condition 2", color="blue")
pylab.plot(time, wbias3[:,2], label="Condition 3", color="green")
pylab.plot(time, time*0.0, "--", color="black")
pylab.ylabel("Weight bias\n(Neuron #400)")
pylab.ylim([-wlim, wlim])
pylab.yticks([0.0])
#pylab.xticks([])
pylab.xlabel("Time [s]")

pylab.tight_layout()
pylab.savefig("wbias_change.pdf")


text:
from pygame import constants
from .guicomponent import GUIComponent
from video.videofeed import VideoFeed
import pygame
import cv2
import numpy as np
from typing import Text, Tuple

class TextComponent(GUIComponent):
    DEFAULT_FONT_SIZE = 25
    DEFAULT_FONT = "Arial"

    def __init__(self, format_string: str, position: Tuple[int, int], color: Tuple[int, int, int]=(0,0,255),
                size: int=None, font: str=None, is_constant: bool=True):
        if size is None:
            size = TextComponent.DEFAULT_FONT_SIZE
        if font is None:
            font = TextComponent.DEFAULT_FONT

        self.format_string = format_string
        self.position = position
        self.size = size
        self.font = font
        self.color = color
        self.is_constant = is_constant
        self.variables = {}

    def setVariables(self, variables: dict):
        if (self.is_constant):
            raise RuntimeError('Cannot set variables for constant TextComponent')
        self.variables = variables

    def setKeyValue(self, key, value):
        if (self.is_constant):
            raise RuntimeError('Cannot set variable for constant TextComponent')
        self.variables[key] = value

    def get_surface(self):
        if (not self.is_constant and not self.variables):
            raise RuntimeError('Cannot get formatted text without initialized variables')
        if (not self.pygame_font):
            raise RuntimeError('TextComponent has not been initialized')
        if (self.is_constant):
            text = self.format_string
        else:
            text = self.format_string.format(**self.variables)
        return self.pygame_font.render(text, True, self.color)

    def get_position(self):
        return self.position

    def initialize(self):
        self.pygame_font = pygame.font.SysFont(self.font, self.size)

max_stars_repo_name: TUPilotsClub/TUTello

max_issues_repo_name: TUPilotsClub/TUTello

max_forks_repo_name: TUPilotsClub/TUTello


text:
import sys
import time
import numpy as np
from optparse import OptionParser

from deep_sea_treasure import DeepSeaTreasure
from agent import DCRACSAgent, DCRACAgent, DCRACSEAgent, DCRAC0Agent, CNAgent, CN0Agent
from utils import mkdir_p, get_weights_from_json
from stats import rebuild_log, print_stats, compute_log

AGENT_DICT = {
    'DCRAC': DCRACAgent,
    'DCRACS': DCRACSAgent,
    'DCRACSE': DCRACSEAgent,
    'DCRAC0': DCRAC0Agent,
    'CN': CNAgent,
    'CN0': CN0Agent,
}

mkdir_p('output')
mkdir_p('output/logs')
mkdir_p('output/networks')
# mkdir_p('output/pred')
# mkdir_p('output/imgs')

parser = OptionParser()
parser.add_option('-a', '--agent', dest='agent', choices=['DCRAC', 'DCRACS', 'DCRACSE', 'DCRAC0', 'CN', 'CN0'], default='DCRACS')
parser.add_option('-n', '--net-type', dest='net_type', choices=['R', 'M', 'F'], default='R', help='Agent architecture type: [R]ecurrent, [M]emNN or [F]C')
parser.add_option('-r', '--replay', dest='replay', default='DER', choices=['STD', 'DER'], help='Replay type, one of "STD","DER"')
parser.add_option('-s', '--buffer-size', dest='buffer_size', default='2500', help='Replay buffer size', type=int)
parser.add_option('-m', '--memnn-size', dest='memnn_size', default='9', help='Memory network memory size', type=int)
parser.add_option('-d', '--dup', dest='dup', action='store_false', default=True, help='Extra training')
parser.add_option('-t', '--timesteps', dest='timesteps', default='10', help='Recurrent timesteps', type=int)
parser.add_option('-e', '--end_e', dest='end_e', default='0.01', help='Final epsilon value', type=float)
parser.add_option('-l', '--lr-c', dest='lr_c', default='0.02', help='Critic learning rate', type=float)
parser.add_option('-L', '--lr-a', dest='lr_a', default='0.25', help='Actor learning rate', type=float)
parser.add_option('--buffer-a', dest='buffer_a', default='2.', help='reply buffer error exponent', type=float)
parser.add_option('--buffer-e', dest='buffer_e', default='0.01', help='reply buffer error offset', type=float)
parser.add_option('-u', '--update-period', dest='updates', default='4', help='Update interval', type=int)
parser.add_option('-f', '--frame-skip', dest='frame_skip', default='4', help='Frame skip', type=int)
parser.add_option('-b', '--batch-size', dest='batch_size', default='16', help='Sample batch size', type=int)
parser.add_option('-g', '--discount', dest='discount', default='0.95', help='Discount factor', type=float)
parser.add_option('--anneal-steps', dest='steps', default='10000', help='steps',  type=int)
parser.add_option('-p', '--mode', dest='mode', choices=['regular', 'sparse'], default='sparse')
parser.add_option('-v', '--obj-func', dest='obj_func', choices=['a', 'ap', 'td', 'q', 'y'], default='a')
parser.add_option('--no-action', dest='action_conc', action='store_false', default=True)
parser.add_option('--no-embd', dest='feature_embd', action='store_false', default=True)
parser.add_option('--gpu', dest='gpu_setting', choices=['1', '2', '3'], default='2', help='1 for CPU, 2 for GPU, 3 for CuDNN')
parser.add_option('--log-game', dest='log_game', action='store_true', default=False)
parser.add_option('--dst', dest='dst_view', choices=['3', '5'], default='5')

(options, args) = parser.parse_args()

hyper_info = '{}_{}-r{}{}-d{}-t{}-batsiz{}-{}steps-lr{}-lr2{}-{}-acteval_{}'.format(
    options.agent, options.net_type, options.replay, str(options.buffer_size), options.dup, options.timesteps,
    options.batch_size, options.steps, str(options.lr_c), str(options.lr_a), options.mode, options.obj_func)

# create evironment
env = DeepSeaTreasure(view=(5,5), scale=9) if options.dst_view == '5' else DeepSeaTreasure(view=(3,3), scale=15)

all_weights = get_weights_from_json('./train_weights_dst.json') if options.mode == 'sparse' else get_weights_from_json('./train_weights_dst_r.json')
timestamp = time.strftime('%m%d_%H%M', time.localtime())
deep_agent = AGENT_DICT[options.agent]
agent = deep_agent(env,
                   gamma=options.discount,
                   weights=None,
                   timesteps=options.timesteps,
                   batch_size=options.batch_size,
                   replay_type=options.replay,
                   buffer_size=options.buffer_size,
                   buffer_a=options.buffer_a,
                   buffer_e=options.buffer_e,
                   memnn_size=options.memnn_size,
                   end_e=options.end_e,
                   net_type=options.net_type,
                   obj_func=options.obj_func,
                   lr=options.lr_c,
                   lr_2=options.lr_a,
                   frame_skip=options.frame_skip,
                   update_interval=options.updates,
                   dup=options.dup,
                   action_conc=options.action_conc,
                   feature_embd=options.feature_embd,
                   extra='{}_{}'.format(timestamp, hyper_info),
                   gpu_setting=options.gpu_setting)

steps_per_weight = 5000 if options.mode == 'sparse' else 1

log_file_name = 'output/logs/{}_dst{}_rewards_{}.log'.format(timestamp, options.dst_view, hyper_info)
with open(log_file_name, 'w', 1) as log_file:
    agent.train(log_file, options.steps, all_weights, steps_per_weight, options.steps*10, log_game_step=options.log_game)

# print stats
length, step_rewards, step_regrets, step_nb_episode = rebuild_log(total=options.steps*10, log_file=log_file_name)
print_stats(length, step_rewards, step_regrets, step_nb_episode, write_to_file=True, timestamp=timestamp)

# # run test
# with open('output/test1.log', 'w', 1) as test_log:
#     all_weights = get_weights_from_json('./train_weights_dst.json')
#     test_weights = all_weights[:20]
#     agent.test(weights=test_weights, stoch_policy=False, log_file=test_log)
# compute_log('output/test1.log', 'output/test1_result.csv')
# with open('output/test2.log', 'w', 1) as test_log:
#     test_weights = np.array(list(zip(range(1,20), range(19,0,-1))))/20
#     agent.test(weights=test_weights, stoch_policy=False, log_file=test_log)
# compute_log('output/test2.log', 'output/test2_result.csv')

max_stars_repo_name: xdnian/DCRAC

max_issues_repo_name: xdnian/DCRAC

max_forks_repo_name: xdnian/DCRAC


text:
"""
This script is modified from the horovod benchmark script(https://github.com/horovod/horovod/blob/master/examples/pytorch/pytorch_synthetic_benchmark.py).
You can compare the performance with the horovod script directly.
"""

import argparse
import timeit
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data.distributed
from torchvision import models
import numpy as np
import bagua.torch_api as bagua

# Benchmark settings
parser = argparse.ArgumentParser(
    description="PyTorch Synthetic Benchmark",
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)

parser.add_argument("--model", type=str, default="resnet50", help="model to benchmark")
parser.add_argument("--batch-size", type=int, default=32, help="input batch size")

parser.add_argument(
    "--num-warmup-batches",
    type=int,
    default=10,
    help="number of warm-up batches that don't count towards benchmark",
)
parser.add_argument(
    "--num-batches-per-iter",
    type=int,
    default=10,
    help="number of batches per benchmark iteration",
)
parser.add_argument(
    "--num-iters", type=int, default=10, help="number of benchmark iterations"
)

parser.add_argument(
    "--no-cuda", action="store_true", default=False, help="disables CUDA training"
)

parser.add_argument(
    "--deterministic",
    action="store_true",
    default=False,
    help="deterministic reproducible training",
)

# bagua args
parser.add_argument(
    "--algorithm",
    type=str,
    default="gradient_allreduce",
    help="gradient_allreduce, bytegrad, decentralized, low_precision_decentralized or qadam",
)
parser.add_argument(
    "--amp",
    action="store_true",
    default=False,
    help="use amp",
)

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()

if not args.cuda:
    raise RuntimeError("bagua currently not supporting non-GPU mode")

if args.cuda:
    torch.cuda.set_device(bagua.get_local_rank())
bagua.init_process_group()

scaler = torch.cuda.amp.GradScaler(enabled=args.amp)

cudnn.benchmark = True
if args.deterministic:
    cudnn.benchmark = False
    cudnn.deterministic = True
    np.random.seed(bagua.get_rank())
    torch.manual_seed(bagua.get_rank())
    torch.cuda.manual_seed(bagua.get_rank())

# Set up standard model.
model = getattr(models, args.model)()

if args.cuda:
    # Move model to GPU.
    model.cuda()

optimizer = optim.SGD(model.parameters(), lr=0.01 * bagua.get_world_size())

if args.algorithm == "gradient_allreduce":
    from bagua.torch_api.algorithms import gradient_allreduce

    algorithm = gradient_allreduce.GradientAllReduceAlgorithm()
elif args.algorithm == "decentralized":
    from bagua.torch_api.algorithms import decentralized

    algorithm = decentralized.DecentralizedAlgorithm()
elif args.algorithm == "low_precision_decentralized":
    from bagua.torch_api.algorithms import decentralized

    algorithm = decentralized.LowPrecisionDecentralizedAlgorithm()
elif args.algorithm == "bytegrad":
    from bagua.torch_api.algorithms import bytegrad

    algorithm = bytegrad.ByteGradAlgorithm()
elif args.algorithm == "qadam":
    from bagua.torch_api.algorithms import q_adam

    optimizer = q_adam.QAdamOptimizer(model.parameters())
    algorithm = q_adam.QAdamAlgorithm(optimizer, 10)
else:
    raise NotImplementedError

model = model.with_bagua([optimizer], algorithm)

# Set up fixed fake data
data = torch.randn(args.batch_size, 3, 224, 224)
target = torch.LongTensor(args.batch_size).random_() % 1000
if args.cuda:
    data, target = data.cuda(), target.cuda()

def benchmark_step():
    optimizer.zero_grad()
    output = model(data)
    loss = F.cross_entropy(output, target)

    if args.amp:
        scaled_loss = args.scaler.scale(loss)
        scaled_loss.backward()
        scaler.step(optimizer)
        scaler.update()
        return

    loss.backward()
    optimizer.step()

def log(s, nl=True):
    if bagua.get_rank() != 0:
        return
    print(s, end="\n" if nl else "")

log("Model: %s" % args.model)
log("Batch size: %d" % args.batch_size)
device = "GPU" if args.cuda else "CPU"
log("Number of %ss: %d" % (device, bagua.get_world_size()))

# Warm-up
log("Running warmup...")
timeit.timeit(benchmark_step, number=args.num_warmup_batches)

# Benchmark
log("Running benchmark...")
img_secs = []
for x in range(args.num_iters):
    time = timeit.timeit(benchmark_step, number=args.num_batches_per_iter)
    img_sec = args.batch_size * args.num_batches_per_iter / time
    log("Iter #%d: %.1f img/sec %s" % (x, img_sec * bagua.get_world_size(), device))
    img_secs.append(img_sec)

# Results
img_sec_mean = np.mean(img_secs)
img_sec_conf = 1.96 * np.std(img_secs)
log("Img/sec per %s: %.1f +-%.1f" % (device, img_sec_mean, img_sec_conf))
log(
    "Total img/sec on %d %s(s): %.1f +-%.1f"
    % (
        bagua.get_world_size(),
        device,
        bagua.get_world_size() * img_sec_mean,
        bagua.get_world_size() * img_sec_conf,
    )
)

max_stars_repo_name: ganshaoduo/examples

max_issues_repo_name: ganshaoduo/examples

max_forks_repo_name: ganshaoduo/examples


text:
# -*- coding: utf-8 -*-

# Filename: data_handler.py
# Author: Julian Betz
# Created: 2018-12-23
# Version: 2019-01-13
#
# Description: A class for converting and loading the dataset.

import json
import os

import numpy as np
import tensorflow as tf
from matplotlib import pyplot as plt
from matplotlib.patches import Rectangle
from numpy.random import RandomState
from sklearn.model_selection import KFold, train_test_split
from sklearn.utils import shuffle
from tensorflow.data import Dataset

N_FEATURES = 40  # TODO Make non-hardcoded
TRAIN_MODE = 0
EVAL_MODE = 1
PRED_MODE = 2

class DataLoader:
    TAGS_ALL = [None, 'B', 'I', 'E', 'S']
    TAG_COLORS = [(0, 0, 0, 0), (0, 1, 0, 0.2), (0, 0, 1, 0.2), (1, 0, 0, 0.2), (1, 1, 0, 0.2)]

    def __init__(self, data_dir, tst_size, seed=None):
        """A data management utility.

        Args:
            data_dir (str): The directory where the dataset is stored.
            tst_size (int/float): The number/proportion of samples from the complete dataset to reserve for testing.
            seed (int/None): The seed used for shuffling the dataset. If None, a random seed is generated.
        """
        self.data_dir = os.path.abspath(data_dir)
        with open(self.data_dir + '/tags.json', 'r') as file_handle:
            self.tags = json.load(file_handle)
        with open(self.data_dir + '/utterances.json', 'r') as file_handle:
            self.ids = json.load(file_handle)
        self.tst_size = tst_size if type(tst_size) == int else int(tst_size * len(self.ids))
        assert self.tst_size >= 0 and self.tst_size <= len(self.ids)
        self.seed = RandomState().randint((2 ** 31 - 1) + 2 ** 31) if seed is None else seed

    def load(self, ids):
        """Loads the data for the specified IDs from disk and returns it.

        A single ID by itself can be provided as a string.

        Args:
            ids (str/iterable<str>): The ID / the collection of IDs that should be loaded. Data is returned in the order in which the IDs are provided.
        Returns:
            feat_seqs (numpy.ndarray/list<numpy.ndarray>): The features for the specified IDs.
            align_seqs (numpy.ndarray/list<numpy.ndarray>): The alignment data for the specified IDs.
            phone_seqs (list<str>/list<list<str>>): Phonetic data for the specified IDs.
        """
        tags_all_inv = {v: i for i, v in enumerate(DataLoader.TAGS_ALL)}
        if type(ids) == str:
            feat_seq = np.load(self.data_dir + '/utterances/%s.npy' % (ids,))
            with open(self.data_dir + '/utterances/%s.json' % (ids,), 'r') as file_handler:
                align_seq = np.array(json.load(file_handler), dtype=np.uint32)
            phone_seq = []
            for token in align_seq:
                tag = self.tags[token[2]]
                token[2] = tags_all_inv[tag[0]]
                phone_seq.append(tag[1])
            return feat_seq, align_seq, phone_seq
        else:
            feat_seqs = [np.load(self.data_dir + '/utterances/%s.npy' % (i,)) for i in ids]
            align_seqs = []
            phone_seqs = []
            for i in ids:
                with open(self.data_dir + '/utterances/%s.json' % (i,), 'r') as file_handler:
                    align_seqs.append(np.array(json.load(file_handler), dtype=np.uint32))
                phone_seqs.append([])
                for token in align_seqs[-1]:
                    tag = self.tags[token[2]]
                    token[2] = tags_all_inv[tag[0]]
                    phone_seqs[-1].append(tag[1])
            return feat_seqs, align_seqs, phone_seqs

    def plot(self, ids):
        """Displays the data for the specified IDs as a plot.

        A single ID by itself can be provided as a string.

        Args:
            ids (str/iterable<str>): The ID / the collection of IDs that should be displayed. Data is plotted in the order in which the IDs are provided.
        """
        for i, feat_seq, align_seq, phone_seq in (
                ((ids, *self.load(ids)),) if type(ids) == str else zip(ids, *self.load(ids))):
            plt.imshow(feat_seq.transpose(), origin='lower', cmap='gray_r')
            for token in align_seq:
                rect = Rectangle((token[0], 0), token[1], feat_seq.shape[1], linewidth=0,
                                 facecolor=DataLoader.TAG_COLORS[token[2]])
                # plt.axvline(token[0], c=['black', 'green', 'blue', 'red', 'yellow'][token[2]])
                plt.gca().add_patch(rect)
            plt.xticks(align_seq[:, 0], phone_seq)
            plt.yticks([], [])
            plt.title(i)
            plt.tight_layout()
            plt.get_current_fig_manager().window.showMaximized()
            plt.show()

    def test_set_ids(self):
        """Returns the IDs for the test set.
        """
        r = RandomState(self.seed)
        ids = shuffle(self.ids, random_state=r, n_samples=self.tst_size)
        print(ids, type(ids))
        return ids

    def test_set(self):
        tst_ids = self.test_set_ids()
        return (tst_ids, *self.load(tst_ids))

    def kfolds_ids(self, n_samples=None, n_splits=10, trn_size=0.8):
        """Iterates over folds to provide the IDs with which to index the appropriate data.

        Shuffles the data IDs using a random number generator seeded with this class's seed, splits off the testing portion and provides training, evaluation and validation data IDs per fold from a portion of the remaining data.

        Args:
            n_samples (int/None): The maximum number of samples to use in k-fold cross-validation from the non-testing portion of the dataset. If None, the whole non-testing portion is used.
            n_splits (int): The number k of splits for cross-validation.
            trn_size (float): The proportion of training data per fold. The remaining data of this fold is provided as evaluation data.
        Yields:
            trn_ids (list<str>): A list of sequence IDs for training.
            evl_ids (list<str>): A list of sequence IDs for evaluation.
            val_ids (list<str>): A list of sequence IDs for validation.
        Example:
            for i, (trn_ids, evl_ids, val_ids) in enumerate(dl.kfolds()):
                trn_data, evl_data, val_data = dl.load(trn_ids), dl.load(evl_ids), dl.load(val_ids)
                pass # Use data for training/evaluation/validation on fold i
        """
        r = RandomState(self.seed)
        n_samples = min(float('inf') if n_samples is None else n_samples, len(self.ids) - self.tst_size)
        ids = shuffle(self.ids, random_state=r, n_samples=self.tst_size + n_samples)[self.tst_size:]
        kfolder = KFold(n_splits=n_splits, shuffle=False)
        for dev_indices, val_indices in kfolder.split(ids):
            trn_indices, evl_indices = train_test_split(dev_indices, test_size=None, train_size=trn_size, shuffle=True,
                                                        random_state=r)
            yield ([ids[trn_index] for trn_index in trn_indices],
                   [ids[evl_index] for evl_index in evl_indices],
                   [ids[val_index] for val_index in val_indices])

    def kfolds(self, n_samples=None, n_splits=10, trn_size=0.8):
        """Iterates over folds to provide the appropriate data.

        Shuffles the data using a random number generator seeded with this class's seed, splits off the testing portion and provides training, evaluation and validation data per fold from a portion of the remaining data.
        Only use this method if the fold data is not expected to take up large amounts of memory.

        Args:
            n_samples (int/None): The maximum number of samples to use in k-fold cross-validation from the non-testing portion of the dataset. If None, the whole non-testing portion is used.
            n_splits (int): The number k of splits for cross-validation.
            trn_size (float): The proportion of training data per fold. The remaining data of this fold is provided as evaluation data.
        Yields:
            trn_data (tuple<list>): Data for training, consisting of:
                trn_ids (list<str>): A list of sequence IDs for training.
                trn_feat_seqs (list<numpy.ndarray>): A list of feature sequences for training.
                trn_align_seqs (list<numpy.ndarray>): A list of alignment data for training.
                trn_phone_seqs (list<list<str>>): A list of phone sequences for training.
            evl_data (tuple<list>): Data for evaluation, consisting of:
                evl_ids (list<str>): A list of sequence IDs for evaluation.
                evl_feat_seqs (list<numpy.ndarray>): A list of feature sequences for evaluation.
                evl_align_seqs (list<numpy.ndarray>): A list of alignment data for evaluation.
                evl_phone_seqs (list<list<str>>): A list of phone sequences for evaluation.
            val_data (tuple<list>): Data for validation, consisting of:
                val_ids (list<str>): A list of sequence IDs for validation.
                val_feat_seqs (list<numpy.ndarray>): A list of feature sequences for validation.
                val_align_seqs (list<numpy.ndarray>): A list of alignment data for validation.
                val_phone_seqs (list<list<str>>): A list of phone sequences for validation.
        Example:
            for i, ((ti, tf, ta, tp), (ei, ef, ea, ep), (vi, vf, va, vp)) in enumerate(dl.kfolds()):
                pass # Use data for training/evaluation/validation on fold i
        """
        for trn_ids, evl_ids, val_ids in self.kfolds_ids(n_samples=n_samples, n_splits=n_splits, trn_size=trn_size):
            yield ((trn_ids, *self.load(trn_ids)),
                   (evl_ids, *self.load(evl_ids)),
                   (val_ids, *self.load(val_ids)))

def align_seqs_to_alternating_labels(align_seqs, lengths):
    """Converts the specified alignment data to labels.

    Labels encode word boundaries in checkerboard representation.
    A single align sequence by itself can be provided as a list of rank 2.

    Args:
        align_seqs (numpy.ndarray/list<numpy.ndarray>): The alignment data.
        lengths (int/list<int>): The total number of frames for the alignment sequences.
    Returns:
        feat_seqs (numpy.ndarray/list<numpy.ndarray>): The labels corresponding to the specified alignment data.
    """
    if type(align_seqs[0][0]) not in [list, tuple, np.ndarray]:  # Single alignment sequence
        label_seq = np.full((lengths,), 0.5, dtype=np.float32)
        current_class = 0.0
        for i, (start, duration, tag) in enumerate(align_seqs):
            label_seq[start:start + duration] = current_class
            if (DataLoader.TAGS_ALL[tag] == 'E'
                    or DataLoader.TAGS_ALL[tag] == 'S'
                    or (DataLoader.TAGS_ALL[tag] == None
                        # Assumes that all non-phone tags can be treated as the same token
                        and align_seqs.shape[0] > i + 1
                        and DataLoader.TAGS_ALL[align_seqs[i + 1][2]] != None)):
                current_class = 1.0 - current_class
        return label_seq
    else:  # List of alignment sequences
        label_seqs = [align_seqs_to_alternating_labels(align_seq, length) for align_seq, length in
                      zip(align_seqs, lengths)]
        return label_seqs

def align_seqs_to_breaking_labels(align_seqs, lengths):
    """Converts the specified alignment data to labels.

    Labels encode word boundaries directly: Class 1 represents end-of-word tags.
    A single align sequence by itself can be provided as a list of rank 2.

    Args:
        align_seqs (numpy.ndarray/list<numpy.ndarray>): The alignment data.
        lengths (int/list<int>): The total number of frames for the alignment sequences.
    Returns:
        feat_seqs (numpy.ndarray/list<numpy.ndarray>): The labels corresponding to the specified alignment data.
    """
    if type(align_seqs[0][0]) not in [list, tuple, np.ndarray]:  # Single alignment sequence
        label_seq = np.full((lengths,), 0.0, dtype=np.float32)
        for i, (start, duration, tag) in enumerate(align_seqs):
            if (DataLoader.TAGS_ALL[tag] == 'E'
                    or DataLoader.TAGS_ALL[tag] == 'S'
                    or (DataLoader.TAGS_ALL[tag] == None
                        # Assumes that all non-phone tags can be treated as the same token
                        and align_seqs.shape[0] > i + 1
                        and DataLoader.TAGS_ALL[align_seqs[i + 1][2]] != None)):
                label_seq[start + duration - 1] = 1.0
        return label_seq
    else:  # List of alignment sequences
        label_seqs = [align_seqs_to_breaking_labels(align_seq, length) for align_seq, length in
                      zip(align_seqs, lengths)]
        return label_seqs

def input_fn(loader, ids, batch_size=None, random_state=None, mode=TRAIN_MODE):
    """Provides the input data for training, evaluation or prediction.

    Data is returned in the format used by tf.estimator.Estimator.

    Args:
        loader: The DataLoader instance to handle the loading of datapoints.
        ids (list<str>): The IDs of the datapoints to that will be needed.
        batch_size (int): The number of samples that comprise a batch. None if not used (in pred and eval mode)
        random_state (numpy.random.RandomState): Random state instance to shuffle the dataset prior to batching. Random states are used to enable different permutations of the training set for different epochs while ensuring that data is not prefetched.
        mode (int) The mode of the estimator to load the correct data. Defined as TRAIN_MODE = 0, EVAL_MODE = 1, PRED_MODE = 2
    Returns:
        next_batch: A nested structure of tensors that iterate over the dataset.
            Every iteration contains a batch of data.
    """

    def load():
        for i in ids:
            feat_seq, align_seq, _ = loader.load(i)
            length = feat_seq.shape[0]
            label_seq = align_seqs_to_breaking_labels(align_seq, length)
            # print('<Loaded %s>' % (i,))
            if mode == TRAIN_MODE or mode == EVAL_MODE:
                yield {'features': feat_seq, 'length': length}, label_seq
            elif mode == PRED_MODE:
                yield {'features': feat_seq, 'length': length}

    dtypes = ({'features': tf.float32, 'length': tf.int32}, tf.float32)
    shapes = ({'features': tf.TensorShape([None, N_FEATURES]), 'length': tf.TensorShape([])}, tf.TensorShape([None]))

    if mode == TRAIN_MODE and random_state is not None:
        ids = shuffle(ids, random_state=random_state)
    elif mode == EVAL_MODE or mode == PRED_MODE or batch_size is None:
        batch_size = len(ids)

    if mode == PRED_MODE:
        dtypes = dtypes[0]
        shapes = shapes[0]

    dataset = Dataset.from_generator(load, dtypes, shapes)
    dataset = dataset.padded_batch(batch_size, shapes)
    return dataset.make_one_shot_iterator().get_next()

# data_handler.py ends here

reason: import numpy,from numpy


text:
from typing import Optional, Tuple, Iterable, Callable, Union
import itertools
import numpy as np
from overrides import overrides
from checklist.test_suite import TestSuite
from checklist.test_types import MFT
from checklist.perturb import Perturb
from allennlp.confidence_checks.task_checklists.task_suite import TaskSuite
from allennlp.confidence_checks.task_checklists import utils

def _wrap_apply_to_each(perturb_fn: Callable, both: bool = False, *args, **kwargs):
    """
    Wraps the perturb function so that it is applied to
    both elements in the (premise, hypothesis) tuple.
    """

    def new_fn(pair, *args, **kwargs):
        premise, hypothesis = pair
        ret = []
        fn_premise = perturb_fn(premise, *args, **kwargs)
        fn_hypothesis = perturb_fn(hypothesis, *args, **kwargs)
        if type(fn_premise) != list:
            fn_premise = [fn_premise]
        if type(fn_hypothesis) != list:
            fn_hypothesis = [fn_hypothesis]
        ret.extend([(x, str(hypothesis)) for x in fn_premise])
        ret.extend([(str(premise), x) for x in fn_hypothesis])
        if both:
            ret.extend([(x, x2) for x, x2 in itertools.product(fn_premise, fn_hypothesis)])

        # The perturb function can return empty strings, if no relevant perturbations
        # can be applied. Eg. if the sentence is "This is a good movie", a perturbation
        # which toggles contractions will have no effect.
        return [x for x in ret if x[0] and x[1]]

    return new_fn

@TaskSuite.register("textual-entailment")
class TextualEntailmentSuite(TaskSuite):
    def __init__(
        self,
        suite: Optional[TestSuite] = None,
        entails: int = 0,
        contradicts: int = 1,
        neutral: int = 2,
        premise: str = "premise",
        hypothesis: str = "hypothesis",
        probs_key: str = "probs",
        **kwargs,
    ):

        self._entails = entails
        self._contradicts = contradicts
        self._neutral = neutral

        self._premise = premise
        self._hypothesis = hypothesis

        self._probs_key = probs_key

        super().__init__(suite, **kwargs)

    def _prediction_and_confidence_scores(self, predictor):
        def preds_and_confs_fn(data):
            labels = []
            confs = []

            data = [{self._premise: pair[0], self._hypothesis: pair[1]} for pair in data]
            predictions = predictor.predict_batch_json(data)
            for pred in predictions:
                label = np.argmax(pred[self._probs_key])
                labels.append(label)
                confs.append(pred[self._probs_key])
            return np.array(labels), np.array(confs)

        return preds_and_confs_fn

    @overrides
    def _format_failing_examples(
        self,
        inputs: Tuple,
        pred: int,
        conf: Union[np.array, np.ndarray],
        label: Optional[int] = None,
        *args,
        **kwargs,
    ):
        """
        Formatting function for printing failed test examples.
        """
        labels = {
            self._entails: "Entails",
            self._contradicts: "Contradicts",
            self._neutral: "Neutral",
        }
        ret = "Premise: %s\nHypothesis: %s" % (inputs[0], inputs[1])
        if label is not None:
            ret += "\nOriginal: %s" % labels[label]
        ret += "\nPrediction: Entails (%.1f), Contradicts (%.1f), Neutral (%.1f)" % (
            conf[self._entails],
            conf[self._contradicts],
            conf[self._neutral],
        )

        return ret

    @classmethod
    def contractions(cls):
        return _wrap_apply_to_each(Perturb.contractions, both=True)

    @classmethod
    def typos(cls):
        return _wrap_apply_to_each(Perturb.add_typos, both=False)

    @classmethod
    def punctuation(cls):
        return _wrap_apply_to_each(utils.toggle_punctuation, both=False)

    @overrides
    def _setup_editor(self):
        super()._setup_editor()

        antonyms = [
            ("progressive", "conservative"),
            ("positive", "negative"),
            ("defensive", "offensive"),
            ("rude", "polite"),
            ("optimistic", "pessimistic"),
            ("stupid", "smart"),
            ("negative", "positive"),
            ("unhappy", "happy"),
            ("active", "passive"),
            ("impatient", "patient"),
            ("powerless", "powerful"),
            ("visible", "invisible"),
            ("fat", "thin"),
            ("bad", "good"),
            ("cautious", "brave"),
            ("hopeful", "hopeless"),
            ("insecure", "secure"),
            ("humble", "proud"),
            ("passive", "active"),
            ("dependent", "independent"),
            ("pessimistic", "optimistic"),
            ("irresponsible", "responsible"),
            ("courageous", "fearful"),
        ]

        self.editor.add_lexicon("antonyms", antonyms, overwrite=True)

        comp = [
            "smarter",
            "better",
            "worse",
            "brighter",
            "bigger",
            "louder",
            "longer",
            "larger",
            "smaller",
            "warmer",
            "colder",
            "thicker",
            "lighter",
            "heavier",
        ]

        self.editor.add_lexicon("compare", comp, overwrite=True)

        nouns = [
            "humans",
            "cats",
            "dogs",
            "people",
            "mice",
            "pigs",
            "birds",
            "sheep",
            "cows",
            "rats",
            "chickens",
            "fish",
            "bears",
            "elephants",
            "rabbits",
            "lions",
            "monkeys",
            "snakes",
            "bees",
            "spiders",
            "bats",
            "puppies",
            "dolphins",
            "babies",
            "kittens",
            "children",
            "frogs",
            "ants",
            "butterflies",
            "insects",
            "turtles",
            "trees",
            "ducks",
            "whales",
            "robots",
            "animals",
            "bugs",
            "kids",
            "crabs",
            "carrots",
            "dragons",
            "mosquitoes",
            "cars",
            "sharks",
            "dinosaurs",
            "horses",
            "tigers",
        ]
        self.editor.add_lexicon("nouns", nouns, overwrite=True)

        professions = [
            "journalist",
            "historian",
            "secretary",
            "nurse",
            "waitress",
            "accountant",
            "engineer",
            "attorney",
            "artist",
            "editor",
            "architect",
            "model",
            "interpreter",
            "analyst",
            "actor",
            "actress",
            "assistant",
            "intern",
            "economist",
            "organizer",
            "author",
            "investigator",
            "agent",
            "administrator",
            "executive",
            "educator",
            "investor",
            "DJ",
            "entrepreneur",
            "auditor",
            "advisor",
            "instructor",
            "activist",
            "consultant",
            "apprentice",
            "reporter",
            "expert",
            "psychologist",
            "examiner",
            "painter",
            "manager",
            "contractor",
            "therapist",
            "programmer",
            "musician",
            "producer",
            "associate",
            "intermediary",
            "designer",
            "cook",
            "salesperson",
            "dentist",
            "attorney",
            "detective",
            "banker",
            "researcher",
            "cop",
            "driver",
            "counselor",
            "clerk",
            "professor",
            "tutor",
            "coach",
            "chemist",
            "scientist",
            "veterinarian",
            "firefighter",
            "baker",
            "psychiatrist",
            "prosecutor",
            "director",
            "technician",
        ]
        self.editor.add_lexicon("professions", professions, overwrite=True)

    @overrides
    def _default_tests(self, data: Optional[Iterable[Tuple]], num_test_cases=100):
        super()._default_tests(data, num_test_cases)
        self._setup_editor()
        self._default_vocabulary_tests(data, num_test_cases)
        self._default_ner_tests(data, num_test_cases)
        self._default_temporal_tests(data, num_test_cases)
        self._default_logic_tests(data, num_test_cases)
        self._default_negation_tests(data, num_test_cases)

    def _default_vocabulary_tests(self, data: Optional[Iterable[Tuple]], num_test_cases=100):

        template = self.editor.template(
            (
                "{first_name1} is more {antonyms[0]} than {first_name2}",
                "{first_name2} is more {antonyms[1]} than {first_name1}",
            ),
            remove_duplicates=True,
            nsamples=num_test_cases,
        )

        test = MFT(
            **template,
            labels=self._entails,
            name='"A is more COMP than B" entails "B is more antonym(COMP) than A"',
            capability="Vocabulary",
            description="Eg. A is more active than B implies that B is more passive than A",
        )

        self.add_test(test)

    def _default_logic_tests(self, data: Optional[Iterable[Tuple]], num_test_cases=100):
        template = self.editor.template(
            ("{nouns1} are {compare} than {nouns2}", "{nouns2} are {compare} than {nouns1}"),
            nsamples=num_test_cases,
            remove_duplicates=True,
        )

        test = MFT(
            **template,
            labels=self._contradicts,
            name='"A is COMP than B" contradicts "B is COMP than A"',
            capability="Logic",
            description='Eg. "A is better than B" contradicts "B is better than A"',
        )

        self.add_test(test)

        if data:
            template = Perturb.perturb(
                data, lambda x: (x[0], x[0]), nsamples=num_test_cases, keep_original=False
            )
            template += Perturb.perturb(
                data, lambda x: (x[1], x[1]), nsamples=num_test_cases, keep_original=False
            )

            test = MFT(
                **template,
                labels=self._entails,
                name="A entails A (premise == hypothesis)",
                capability="Logic",
                description="If premise and hypothesis are the same, then premise entails the hypothesis",
            )

            self.add_test(test)

    def _default_negation_tests(self, data: Optional[Iterable[Tuple]], num_test_cases=100):

        template = self.editor.template(
            (
                "{first_name1} is {compare} than {first_name2}",
                "{first_name1} is not {compare} than {first_name2}",
            ),
            nsamples=num_test_cases,
            remove_duplicates=True,
        )

        test = MFT(
            **template,
            labels=self._contradicts,
            name='"A is COMP than B" contradicts "A is not COMP than B"',
            capability="Negation",
            description="Eg. A is better than B contradicts A is not better than C",
        )

        self.add_test(test)

    def _default_ner_tests(self, data: Optional[Iterable[Tuple]], num_test_cases=100):
        template = self.editor.template(
            (
                "{first_name1} is {compare} than {first_name2}",
                "{first_name1} is {compare} than {first_name3}",
            ),
            nsamples=num_test_cases,
            remove_duplicates=True,
        )

        test = MFT(
            **template,
            labels=self._neutral,
            name='"A is COMP than B" gives no information about "A is COMP than C"',
            capability="NER",
            description='Eg. "A is better than B" gives no information about "A is better than C"',
        )

        self.add_test(test)

    def _default_temporal_tests(self, data: Optional[Iterable[Tuple]], num_test_cases=100):
        template = self.editor.template(
            (
                "{first_name} works as {a:professions}",
                "{first_name} used to work as a {professions}",
            ),
            nsamples=num_test_cases,
            remove_duplicates=True,
        )

        template += self.editor.template(
            (
                "{first_name} {last_name} is {a:professions}",
                "{first_name} {last_name} was {a:professions}",
            ),
            nsamples=num_test_cases,
            remove_duplicates=True,
        )

        test = MFT(
            **template,
            labels=self._neutral,
            name='"A works as P" gives no information about "A used to work as P"',
            capability="Temporal",
            description='Eg. "A is a writer" gives no information about "A was a writer"',
        )

        self.add_test(test)

        template = self.editor.template(
            (
                "{first_name} was {a:professions1} before they were {a:professions2}",
                "{first_name} was {a:professions1} after they were {a:professions2}",
            ),
            nsamples=num_test_cases,
            remove_duplicates=True,
        )

        test = MFT(
            **template,
            labels=self._contradicts,
            name="Before != After",
            capability="Temporal",
            description='Eg. "A was a writer before they were a journalist" '
            'contradicts "A was a writer after they were a journalist"',
        )

        self.add_test(test)

max_stars_repo_name: kophil/allennlp

max_issues_repo_name: kophil/allennlp

max_forks_repo_name: kophil/allennlp


text:
#
# From https://github.com/marcusschiesser/intraday - modified
# https://pythoninoffice.com/get-values-rows-and-columns-in-pandas-dataframe/
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.append.html
#
from pprint import pprint
import yfinance as yf
import pandas as pd
import numpy as np
import os
import ast
from decimal import Decimal
from datetime import datetime, date, timedelta, timezone

# def to_utc(d):
#    return d.tz_convert(timezone.utc)
#
#
# def df_to_utc(df):
#    df.index = df.index.map(to_utc)

def get_tickerfile(ticker):
    dirname = os.path.dirname(__file__)
    return dirname + '/data/' + ticker + '.csv'

def get_cache(ticker):
    filename = get_tickerfile(ticker)
    #print("get_cache " + ticker + " " + filename)
    try:
        #print("read_csv " + ticker + " " + filename)
        return pd.read_csv(filename, delimiter=",", parse_dates=True, index_col='Date')

    except ValueError:
        print("get_cache ValueError " + ticker)
        return pd.DataFrame()

    except FileNotFoundError:
        print("Ticker cache file '{}' not found.\n".format(filename))
        # return empty data frame on error
        return pd.DataFrame()

# get last date we have in our dataframe
def get_lastdate(df):
    if len(df) == 0:
        return date.today() - timedelta(days=5*365)
    max = df.index.max()
    return max.date()

# Get date/close pair at a specific date
def get(df, date):
    max = df.index.max()
    while date < max and date not in df.index:
        date = date + timedelta(days=1)

    if date in df.index:
        return {"date": date.date(), "close": Decimal(df.loc[date, 'Close'])}
    else:
        return {"date": date.date(), "close": Decimal(0)}

# Get last date/close pair
def get_last(df):
    max = df.index.max()
    # print(df.index)
    # print(df.columns)
    # print(df.Close)
    # print(df['Close'])
    # print(max.date(), df.loc[max, 'Close'])
    return {"date": max.date(), "close": Decimal(df.loc[max, 'Close'])}

# update the dataset for a ticker, getting new data from Yahoo Finance if needed
def update_ticker(ticker):
    existing_df = get_cache(ticker)

    # Figure out if we need to get more data from Yahoo Finance.
    # We do not get data from today as the market may still be open (could add check for that)
    last_date = get_lastdate(existing_df)
    start_date = last_date + timedelta(days=1)
    today = date.today()
    end_date = today - timedelta(days=1)
    #print("Ticker[{}] Update - Last[{}] Start[{}] Today[{}] End[{}]".format(ticker, last_date, start_date, today, end_date))
    if start_date < today and end_date >= start_date:
        print("Ticker[{}] Getting data Start[{}] End[{}]".format(ticker, start_date, end_date))
        t = yf.Ticker(ticker)
        updated_df = t.history(start=start_date, end=end_date)
        if not updated_df.empty:
            existing_df = pd.concat([existing_df, updated_df], sort=True)
            existing_df.to_csv(get_tickerfile(ticker))

    return existing_df

max_stars_repo_name: roarflolo/portfolio-tracker

max_issues_repo_name: roarflolo/portfolio-tracker

max_forks_repo_name: roarflolo/portfolio-tracker


text:
import numpy as np

class Cluster():
    def __init__(self, data = None, labels = None, index = None):
        self.data = data
        if type(self.data) is np.ndarray:
            self.labels = [0] * len(self.data)
            self.index = [i for i in range(0, len(data))]

        else:
            self.data = np.array([])

    def __str__(self):
        print_str = "{:>5}{:>12}{:>11}\n".format("Index", "Value", "Label")
        for i in range(len(self.data)):
            print_str += "{:>5}{:>12}{:>11}\n".format(str(self.index[i]), str(self.data[i]), str(self.labels[i]))
        return print_str

    def __getitem__(self, i):
        return self.data[i]

    def __setitem__(self, i, data_val):
        """
        Set the data value and, optionally, the label for a given index.
        Input should be a numpy array or a tuple of an array and an integer.
        If a label is not provided, the default value of 0 is used.
        """
        if type(data_val) is tuple:

            if type(data_val[0]) is not np.ndarray:
                raise TypeError(f"Expected type {np.ndarray}, got {type(data_val[0])} instead.")
            self.data[i] = data_val[0]

            if (type(data_val[1]) is not int) and (type(data_val[1]) is not str):
                raise TypeError(f"Expected type {int} or {str}, got {type(data_val[1])} instead.")
            self.labels[i] = data_val[1]

        else:
            if type(data_val) is not np.ndarray:
                raise TypeError(f"Expected type {np.ndarray} or {tuple}, got {type(data_val[0])} instead.")
            self.data[i] = data_val
            self.labels[i] = 0

    def set_label(self, value, label, assign = 'All'):
        """
        Set the label for a given index or data value.
        If multiple points have the same value, the method defaults to setting the label for each one.
        This can be overridden by setting the 'assign' parameter to integer, n.
        The method will then assign only the first n points with the same value.
        """
        if type(value) is int:
            self.labels[value] = label

        elif type(value) is np.ndarray:

            # Get the index for every data value that matches the input value
            indices = np.all(self.data == value, axis=1).nonzero()[0]

            if assign == 'All':
                for i in indices:
                    self.labels[i] = label

            else:
                i = 0
                while i < assign and i < len(indices):
                    self.labels[i] = label
                    i+=1

        else:
            raise TypeError(f"Expected type {np.ndarray} or {int}, got {type(value)} instead.")

if __name__ == "__main__":

    # Create Object
    Array = np.array([[0,0], [0,1], [0,2], [1,1], [5,1], [9,1], [10,0], [10,1], [10,2]])
    clusters = Cluster(Array)

    # Test print function
    print(clusters)

    # # Test __setitem__
    # clusters[0] = Array[8], '1'
    # print(clusters)

    # Test set_label
    clusters.set_label(Array[8], 'test 1')
    print(clusters)

    clusters[3] = Array[8]
    clusters[4] = Array[8]
    clusters[5] = Array[8]

    clusters.set_label(Array[8], 'test 2')
    print(clusters)


text:
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A client that performs inferences on a ResNet model using the REST API.

Install nvidia-docker
https://github.com/NVIDIA/nvidia-docker#quick-start

Typical usage example:
docker run --runtime=nvidia -p 8501:8501 \
--name tfserving_segmentation \
--mount type=bind,source=/home/ubuntu/demo/model/fcn_camvid_20190125/export,target=/models/segmentation \
-e MODEL_NAME=segmentation -t tensorflow/serving:latest-gpu &

docker run --runtime=nvidia -p 8501:8501 \
--name tfserving_segmentation \
--mount type=bind,source=/home/ubuntu/demo/model/unet_camvid_20190125/export,target=/models/segmentation \
-e MODEL_NAME=segmentation -t tensorflow/serving:latest-gpu &

python client/image_segmentation_client.py --image_path=~/demo/data/camvid/test/0001TP_008550.png
"""

from __future__ import print_function

import os
import requests
import numpy as np
import json
import argparse
import skimage.io
from skimage import img_as_ubyte
import matplotlib.pyplot as plt
from PIL import Image

# The server URL specifies the endpoint of your server running the ResNet
# model with the name "resnet" and using the predict interface.
SERVER_URL = 'http://localhost:8501/v1/models/segmentation:predict'

NUM_CLASSES = 12

COLORS = np.random.randint(255, size=(NUM_CLASSES, 3))

def render_label(label, num_classes, label_colors):

  label = label.astype(int)
  r = np.zeros((label.shape[0], label.shape[1]), dtype=np.uint8)
  g = np.zeros((label.shape[0], label.shape[1]), dtype=np.uint8)
  b = np.zeros((label.shape[0], label.shape[1]), dtype=np.uint8)

  for i_color in range(0, num_classes):
    r[label == i_color] = label_colors[i_color, 0]
    g[label == i_color] = label_colors[i_color, 1]
    b[label == i_color] = label_colors[i_color, 2]

  rgb = np.zeros((label.shape[0], label.shape[1], 3), dtype=np.uint8)
  rgb[:, :, 0] = r
  rgb[:, :, 1] = g
  rgb[:, :, 2] = b

  return rgb

def main():

  parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter)

  parser.add_argument("--image_path",
                      help="path for image to run inference",
                      default="~/demo/data/camvid/test/0001TP_008550.png")

  args = parser.parse_args()

  args.image_path = os.path.expanduser(args.image_path)

  # Read the image
  image = img_as_ubyte(skimage.io.imread(args.image_path, plugin='imageio'))

  data = json.dumps({"signature_name": "predict", "instances": image.tolist()})
  headers = {"content-type": "application/json"}

  response = requests.post(SERVER_URL, data=data, headers=headers)
  response.raise_for_status()

  predictions = np.squeeze(
    np.array(response.json()["predictions"]), axis=0)

  render_image = render_label(predictions, NUM_CLASSES, COLORS)

  render_image = Image.fromarray(render_image, 'RGB')
  plt.imshow(render_image)
  plt.show()

if __name__ == '__main__':
  main()


text:
import sys
import numpy as np
import pygame
import pygame.gfxdraw

pygame.init()
pygame.display.init()
screen_size = 1080, 720
screen = pygame.display.set_mode(screen_size)
fps = 240
clock = pygame.time.Clock()

black = 0
color_a = 0x282d3f
color_b = 0xd3423e

square_size = 10

rows, cols = int(screen_size[1] / square_size), int(screen_size[0] / square_size)

grid = np.zeros((rows, cols), dtype=np.uint8)
print(f"\n{grid}\n")
print(
    f"Screen size: {screen_size[0]}x{screen_size[1]}\n"
    f"Square size: {square_size}\n"
    f"Grid size: {grid.shape[1]}x{grid.shape[0]}")

class Ant:
    ANT_UP = 0
    ANT_RIGHT = 1
    ANT_DOWN = 2
    ANT_LEFT = 3

    def __init__(self, position, direction: int):
        self.position: list[int] = [int(position[0]), int(position[1])]
        self.direction = direction

    def turn_right(self):
        self.direction += 1
        self.direction %= 4

    def turn_left(self):
        self.direction -= 1
        self.direction %= 4

    def on_zero(self):
        # print("Checking square color")
        # print(not grid[self.position[1]][self.position[0]])
        return not grid[self.position[1], self.position[0]]

    def switch_color(self):
        # print("Switching color")
        grid[self.position[1], self.position[0]] = not grid[self.position[1], self.position[0]]

    def move(self):
        if self.direction == self.ANT_UP:
            self.position[1] -= 1
        elif self.direction == self.ANT_RIGHT:
            self.position[0] += 1
        elif self.direction == self.ANT_DOWN:
            self.position[1] += 1
        elif self.direction == self.ANT_LEFT:
            self.position[0] -= 1

        if self.position[0] > cols - 1:
            self.position[0] = 0
        elif self.position[0] < 0:
            self.position[0] = cols - 1

        if self.position[1] > rows - 1:
            self.position[1] = 0
        elif self.position[1] < 0:
            self.position[1] = rows - 1

    def step(self):
        if self.on_zero():
            # print("Left")
            self.turn_right()
            self.switch_color()
            self.move()
        else:
            # print("Right")
            self.turn_left()
            self.switch_color()
            self.move()

"""
def display(surface):
    display_surf = pygame.Surface(grid.shape)
    display_grid = (grid*color_b) + (1-grid)*color_a
    pygame.surfarray.blit_array(display_surf, display_grid)
    scaled_surf = pygame.transform.scale(display_surf, surface.get_size())

    surface.blit(scaled_surf, (0, 0))
"""

def display(surface):
    x, y = 0, 0
    for row in grid:
        for col in row:
            if col:
                pygame.draw.rect(surface, color_b, (x, y, square_size, square_size))
            x += square_size
        y += square_size
        x = 0

def draw_grid():
    for x in range(0, screen_size[0], square_size):
        pygame.gfxdraw.vline(screen, x, 0, screen_size[1], (0, 0, 0))
    for y in range(0, screen_size[1], square_size):
        pygame.gfxdraw.hline(screen, 0, screen_size[0], y, (0, 0, 0))

ant = Ant((cols / 2, rows / 2), 1)

# ant1 = Ant((cols / 2.5, rows / 2), 2)
# ant2 = Ant((cols / 1.8, rows / 2), 3)
# ant3 = Ant((cols / 3, rows / 2), 4)

def main():
    steps = 0

    while True:
        clock.tick(fps)

        # Exit on ESCAPE key press or if the user presses the X button on the window
        for event in pygame.event.get():
            if event.type == pygame.QUIT or \
                    event.type == pygame.KEYDOWN and \
                    event.key == pygame.K_ESCAPE:
                print(steps)
                sys.exit()

        ant.step()
        # ant1.step()
        # ant2.step()
        # ant3.step()

        screen.fill(color_a)
        display(screen)
        draw_grid()
        pygame.display.flip()
        pygame.display.set_caption(f"Langton's ant ({round(clock.get_fps())}/{fps} fps)")
        steps += 1

        """ Performance testing
        if steps >= 100:
            print(f"\n{steps} steps in {pygame.time.get_ticks()} ms")
            sys.exit()
        """

if __name__ == "__main__":
    main()

max_stars_repo_name: IlmastroStefanuzzo/pygame-langtons-ant

max_issues_repo_name: IlmastroStefanuzzo/pygame-langtons-ant

max_forks_repo_name: IlmastroStefanuzzo/pygame-langtons-ant


text:
# py 2.x compatibility
try:
    from Tkinter import *
except ImportError:
    from tkinter import *

import numpy as np
import scipy.linalg, scipy.stats

def clamp(val, minimum=0, maximum=255):
    if val < minimum:
        return minimum
    if val > maximum:
        return maximum
    return val

def colorscale(hexstr, scalefactor):
    """
    Scales a hex string by ``scalefactor``. Returns scaled hex string.

    To darken the color, use a float value between 0 and 1.
    To brighten the color, use a float value greater than 1.

    >>> colorscale("#DF3C3C", .5)
    #6F1E1E
    >>> colorscale("#52D24F", 1.6)
    #83FF7E
    >>> colorscale("#4F75D2", 1)
    #4F75D2
    """

    hexstr = hexstr.strip('#')

    if scalefactor < 0 or len(hexstr) != 6:
        return hexstr

    r, g, b = int(hexstr[:2], 16), int(hexstr[2:4], 16), int(hexstr[4:], 16)

    r = clamp(r * scalefactor)
    g = clamp(g * scalefactor)
    b = clamp(b * scalefactor)

    return "#%02x%02x%02x" % (r, g, b)

class TKanvas(object):
    def __init__(self, draw_fn=None, tick_fn=None, event_fn=None, quit_fn=None, w=400, h=400, frame_time=20, bgcolor='black'):
        self.root = Tk()
        self.canvas = Canvas(self.root, background = bgcolor, width=w, height=h)
        self.w, self.h = w, h
        self.cx = w/2
        self.cy=h/2
        self.mouse_x = self.cx
        self.mouse_y = self.cy
        self.canvas.pack(expand=1, fill="both")
        self.draw_fn = draw_fn
        self.tick_fn = tick_fn
        self.quit_fn = quit_fn
        self.event_fn = event_fn
        self.root.iconify()
        self.root.update()
        self.root.deiconify()
        self.root.wm_title("Canvas view: Press ESC to quit")
        self.root.bind("<Any-KeyPress>", lambda ev: self.event("keypress", ev))
        self.root.bind("<Any-KeyRelease>", lambda ev: self.event("keyrelease", ev))
        self.root.bind("<Escape>", self.quit)
        self.root.protocol("WM_DELETE_WINDOW", lambda :self.quit(None))
        self.root.bind( "<Any-Button>", lambda ev: self.event("mousedown", ev))
        self.root.bind( "<Any-ButtonRelease>", lambda ev: self.event("mouseup", ev))
        self.root.bind( "<Any-Motion>", lambda ev: self.event("mousemotion", ev))
        self.frame_time = frame_time
        self.root.update()
        self.root.after(int(10), self.update)

    def quit(self, event):
        print("Exiting...")
        if self.quit_fn is not None:
            try:
                self.quit_fn(self)
            except:
                print("Error in quit routine; exiting anyway")

        self.root.destroy()

    def clear(self):
        self.canvas.delete(ALL)

    def to_front(self, obj):
        self.canvas.tag_raise(obj)

    def to_back(self, obj):
        self.canvas.tag_lower(obj)

    def rectangle(self, x1, y1, x2, y2, **kw):
        return self.canvas.create_rectangle( x1, y1, x2, y2, **kw)

    def error_ellipse(self, mean, cov, scale=1, **kw):
        r = np.linspace(-np.pi, np.pi, 12)
        cov = scipy.linalg.sqrtm(cov)
        p = np.stack([np.cos(r), -np.sin(r)]).T
        q = np.dot(p,cov)*scale*5 + mean
        self.canvas.create_polygon(*q.ravel(), **kw)

    def polygon(self, pts, **kw):
        pts = np.array(pts)
        return self.canvas.create_polygon(*pts.ravel(), **kw)

    def modify(self, item, **kw):
        self.canvas.itemconfig(item, **kw)

    def square(self, x, y, r, **kw):
        return self.rectangle(x-r, y-r, x+r, y+r, **kw)

    def arc(self, x1, y1, x2, y2, **kw):
        return self.canvas.create_rectangle( x1, y1, x2, y2, **kw)

    def line(self, x1, y1, x2, y2, **kw ):
        return self.canvas.create_line( x1, y1, x2, y2, **kw)

    def circle(self, x, y, r, **kw):
        return self.oval(x-r, y-r, x+r, y+r, **kw)

    def oval(self, x1, y1, x2, y2, **kw ):
        return self.canvas.create_oval( x1, y1, x2, y2, **kw)

    def text(self, x1, y1, **kw ):
        return self.canvas.create_text(x1,y1, **kw)

    def move_rel(self, tagOrId, dx,dy):
        print(tagOrId, dx, dy)
        self.canvas.move( tagOrId, dx, dy)

    def delete(self, tagOrId):
        self.canvas.delete(tagOrId)

    def title(self, title):
        self.root.wm_title(title)

    def event(self, event_type, event):
        if event_type=="mousemotion":
            # track mouse offset
            dx = self.mouse_x - event.x
            dy = self.mouse_y - event.y
            event.dx = -dx
            event.dy = -dy

            self.mouse_x = event.x
            self.mouse_y = event.y

        if self.event_fn is not None:
            self.event_fn(self, event_type, event)

    def normal(self, mean, cov, outline='#0000ff', ppfs=(0.55, 0.65, 0.75, 0.85, 0.9), **kw):
        for ppf in reversed(sorted(ppfs)):
            scale = scipy.stats.norm.ppf(ppf)
            cscale = 2.0*scipy.stats.norm.pdf(scale)
            self.error_ellipse(mean, cov, scale=scale, smooth=True, outline=colorscale(outline,cscale), fill=colorscale(outline,cscale), **kw)

    def update(self):
        if self.draw_fn is not None:
            self.draw_fn(self)
        self.root.update_idletasks()
        if self.tick_fn is not None:
            self.tick_fn(0.01)
        self.root.after(int(self.frame_time), self.update)

### Demo with mouse crosshairs
if __name__=="__main__":
    def track(src, etype, event):
        pass

    def draw(src):
        src.clear()
        src.line(0, src.mouse_y, src.w, src.mouse_y, fill="red")
        src.line(src.mouse_x, 0, src.mouse_x, src.h, fill="red")
        #src.normal(np.array([src.mouse_x, src.mouse_y]), np.eye(2)*30,
        #           smooth=1, outline="blue", fill='')

        pass

    c = TKanvas(draw_fn=draw)
    mainloop()

max_stars_repo_name: chwebster/jhwutils

max_issues_repo_name: chwebster/jhwutils

max_forks_repo_name: chwebster/jhwutils

reason: import numpy,import scipy


text:
from bokeh.sampledata import us_states, us_counties
from bokeh.plotting import figure, show, output_notebook, output_file, save
from bokeh import palettes
from bokeh.models import ColorBar,HoverTool,LinearColorMapper,ColumnDataSource,FixedTicker, LogColorMapper
output_notebook()
import re
import numpy as np
from modeling import fit_and_predict
import plotly.graph_objs as go
import plotly.figure_factory as ff
from plotly.offline import plot
from plotly.subplots import make_subplots
import json
import plotly.express as px
import plotly
import pandas as pd
from datetime import datetime, timedelta

credstr ='rgb(234, 51, 86)'
cbluestr = 'rgb(57, 138, 242)'

def viz_curves(df, filename='out.html',
               key_toggle='CountyName',
               keys_table=['CountyName', 'StateName'],
               keys_curves=['deaths', 'cases'],
               dropdown_suffix=' County',
               decimal_places=0,
               expl_dict=None, interval_dicts=None,
               point_id=None, show_stds=False, ):
        '''Visualize explanation for all features (table + ICE curves)
        and save to filename

        Params
        ------
        df: table of data

        '''
        color_strings = [credstr, cbluestr]

        # plot the table
        df_tab = df[keys_table]
        fig = ff.create_table(df_tab.round(decimal_places))

        # scatter plots
        traces = []
        num_traces_per_plot = len(keys_curves)
        key0 = df_tab[key_toggle].values[0] # only want this to be visible
        for i in range(df.shape[0]):
            row = df.iloc[i]
            key = row[key_toggle]

            for j, key_curve in enumerate(keys_curves):
                curve = row[key_curve]
                x = np.arange(curve.size)
                traces.append(go.Scatter(x=x,
                    y=curve,
                    showlegend=False,
                    visible=i==0, #key == key0,# False, #key == key0,
                    name=key_curve,
                    line=dict(color=color_strings[j], width=4),
                    xaxis='x2', yaxis='y2')
                )

        fig.add_traces(traces)

        # add buttons to toggle visibility
        buttons = []
        for i, key in enumerate(df[key_toggle].values):
            table_offset = 1
            visible = np.array([True] * table_offset + [False] * num_traces_per_plot * len(df[key_toggle]))
            visible[num_traces_per_plot * i + table_offset: num_traces_per_plot * (i + 1) + table_offset] = True
            buttons.append(
                dict(
                    method='restyle',
                    args=[{'visible': visible}],
                    label=key + dropdown_suffix
                ))

        # initialize xaxis2 and yaxis2
        fig['layout']['xaxis2'] = {}
        fig['layout']['yaxis2'] = {}

        fig.layout.updatemenus = [go.layout.Updatemenu(
            dict(
                active=int(np.argmax(df[key_toggle].values == key0)),
                buttons=buttons,
                x=0.8, # this is fraction of entire screen
                y=1.05,
                direction='down'
            )
        )]

        # Edit layout for subplots
        fig.layout.xaxis.update({'domain': [0, .5]})
        fig.layout.xaxis2.update({'domain': [0.6, 1.]})
        fig.layout.xaxis2.update({'title': 'Time'})

        # The graph's yaxis MUST BE anchored to the graph's xaxis
        fig.layout.yaxis.update({'domain': [0, .9]})
        fig.layout.yaxis2.update({'domain': [0, .9], 'anchor': 'x2', })
        fig.layout.yaxis2.update({'title': 'Count'})

        # Update the margins to add a title and see graph x-labels.
        fig.layout.margin.update({'t':50, 'b':120})

        fig.layout.update({
            'title': 'County-level outbreaks',
            'height': 800
        })

        # fig.layout.template = 'plotly_dark'
        plot(fig, filename=filename, config={'showLink': False,
                                             'showSendToCloud': False,
                                             'sendData': True,
                                             'responsive': True,
                                             'autosizable': True,
                                             'displaylogo': False
                                            })
#         fig.show()
        print('plot saved to', filename)

'''Interactive plots for counties/hospitals'''

'''
fig = px.scatter(df, x="tot_deaths", y="Predicted Deaths 3-day", log_x=True, log_y=True,
                 hover_name="CountyName", hover_data=["CountyName", 'StateName'])
plotly.offline.plot(fig, filename="results/pred_deaths_vs_curr_deaths.html")

fig = px.scatter(df, x="tot_deaths", y="Predicted New Deaths 3-day", log_x=True, log_y=True,
                 hover_name="CountyName", hover_data=["CountyName", 'StateName'])
plotly.offline.plot(fig, filename="results/new_deaths_vs_curr_deaths.html")
'''

def viz_index_animated(d, NUM_DAYS_LIST, by_size=False,
                       x_key='Total Deaths Hospital', # Hospital Employees
                       y_key='Predicted new deaths at hospital', # 'Predicted (cumulative) deaths at hospital'
                       hue='Severity Index', # 'Surge Index'
                       out_name="results/hospital_index_animated.html"):
    '''
    by_size: bool
        If False, make the plot of new deaths vs current deaths
        If true, make the plot of tot deaths vs hosp size
    '''
    def flat_list(list_list):
        l = []
        for ll in list_list:
            l += ll.tolist()
        return l

    N = d.shape[0]
    NUM_DAYS = len(NUM_DAYS_LIST)
    dd = pd.concat([d] * NUM_DAYS)
    # days_past = [(datetime.now() + timedelta(days=i)).strftime("%B %d") for i in NUM_DAYS_LIST]
    dd['Days in the future'] = np.repeat(NUM_DAYS_LIST, N)
    dd['Predicted new deaths at hospital'] = flat_list([d[f"Predicted New Deaths Hospital {i}-day"].values for i in NUM_DAYS_LIST])
    dd['Predicted (cumulative) deaths at hospital'] = flat_list([d[f"Predicted Deaths Hospital {i}-day"].values for i in NUM_DAYS_LIST])
    dd['Severity Index'] = flat_list([d[f"Severity Index {i}-day"].values for i in NUM_DAYS_LIST])
    dd['Surge Index'] = flat_list([d[f"Surge {i}-day"].values for i in NUM_DAYS_LIST])
    today = datetime.today().strftime("%B %d")
    todays_deaths_key = f'Total (estimated) deaths at hospital by {today}'
    dd = dd.rename(columns={'Total Deaths Hospital': todays_deaths_key})

    # decide x and y keys
    if x_key == 'Total Deaths Hospital':
        x_key = todays_deaths_key

    # make plot
    fig = px.scatter(dd, x=x_key,
                 y=y_key,
                 animation_frame="Days in the future",
                 animation_group="Hospital Name",
                 color=hue,
                 size='Hospital Employees',
                 hover_name="Hospital Name",
                 hover_data=["CountyName", 'StateName'],
                 log_x=True, log_y=True)
    fig.update_layout(
                paper_bgcolor='rgba(0,0,0,255)',
                plot_bgcolor='rgba(0,0,0,255)',
                template='plotly_dark'
            )
    plotly.offline.plot(fig, filename=out_name, auto_open=False)


text:
#!/usr/bin/python3 -O
# encoding: utf-8
"""
.. invisible:
     _   _ _____ _     _____ _____
    | | | |  ___| |   |  ___/  ___|
    | | | | |__ | |   | |__ \ `--.
    | | | |  __|| |   |  __| `--. \
    \ \_/ / |___| |___| |___/\__/ /
     \___/\____/\_____|____/\____/

Created on April 24, 2014

A unit test for local response normalization.

███████████████████████████████████████████████████████████████████████████████

Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.

███████████████████████████████████████████████████████████████████████████████
"""

import numpy

from veles.memory import Array
from veles.tests import AcceleratedTest, assign_backend
from veles.znicz.normalization import LRNormalizerForward, LRNormalizerBackward

class TestNormalization(AcceleratedTest):
    ABSTRACT = True

    def test_normalization_forward(self):
        fwd_normalizer = LRNormalizerForward(self.parent)
        fwd_normalizer.input = Array()
        in_vector = numpy.zeros(shape=(3, 2, 5, 19), dtype=self.dtype)

        for i in range(5):
            in_vector[0, 0, i, :] = numpy.linspace(10, 50, 19) * (i + 1)
            in_vector[0, 1, i, :] = numpy.linspace(10, 50, 19) * (i + 1) + 1
            in_vector[1, 0, i, :] = numpy.linspace(10, 50, 19) * (i + 1) + 2
            in_vector[1, 1, i, :] = numpy.linspace(10, 50, 19) * (i + 1) + 3
            in_vector[2, 0, i, :] = numpy.linspace(10, 50, 19) * (i + 1) + 4
            in_vector[2, 1, i, :] = numpy.linspace(10, 50, 19) * (i + 1) + 5

        fwd_normalizer.input.mem = in_vector
        fwd_normalizer.initialize(device=self.device)

        fwd_normalizer.ocl_run()
        fwd_normalizer.output.map_read()
        ocl_result = numpy.copy(fwd_normalizer.output.mem)

        fwd_normalizer.numpy_run()
        fwd_normalizer.output.map_read()
        numpy_result = numpy.copy(fwd_normalizer.output.mem)

        max_delta = numpy.fabs(numpy_result - ocl_result).max()

        self.info("FORWARD")
        self.assertLess(max_delta, 0.0001,
                        "Result differs by %.6f" % (max_delta))

        self.info("FwdProp done.")

    def test_normalization_backward(self):

        h = numpy.zeros(shape=(2, 1, 5, 5), dtype=self.dtype)
        for i in range(5):
            h[0, 0, i, :] = numpy.linspace(10, 50, 5) * (i + 1)
            h[1, 0, i, :] = numpy.linspace(10, 50, 5) * (i + 1) + 1

        err_y = numpy.zeros(shape=(2, 1, 5, 5), dtype=self.dtype)
        for i in range(5):
            err_y[0, 0, i, :] = numpy.linspace(2, 10, 5) * (i + 1)
            err_y[1, 0, i, :] = numpy.linspace(2, 10, 5) * (i + 1) + 1

        back_normalizer = LRNormalizerBackward(self.parent, n=3)
        back_normalizer.input = Array()
        back_normalizer.err_output = Array()

        back_normalizer.input.mem = h
        back_normalizer.err_output.mem = err_y

        back_normalizer.initialize(device=self.device)

        back_normalizer.numpy_run()
        numpy_result = back_normalizer.err_input.mem.copy()

        back_normalizer.err_input.map_invalidate()
        back_normalizer.err_input.mem[:] = 100

        back_normalizer.ocl_run()
        back_normalizer.err_input.map_read()
        ocl_result = back_normalizer.err_input.mem.copy()

        self.info("BACK")

        max_delta = numpy.fabs(numpy_result - ocl_result).max()
        self.assertLess(max_delta, 0.0001,
                        "Result differs by %.6f" % (max_delta))

        self.info("BackProp done.")

@assign_backend("ocl")
class OpenCLTestNormalization(TestNormalization):
    pass

@assign_backend("cuda")
class CUDATestNormalization(TestNormalization):
    pass

if __name__ == "__main__":
    AcceleratedTest.main()


text:
import numpy as np

def amari_d(W, A):
    P = np.dot(W, A)

    def s(r):
        return np.sum(np.sum(r ** 2, axis=1) / np.max(r ** 2, axis=1) - 1)

    return (s(np.abs(P)) + s(np.abs(P.T))) / (2 * P.shape[0])

max_stars_repo_name: hugorichard/ShICA

max_issues_repo_name: hugorichard/ShICA

max_forks_repo_name: hugorichard/ShICA


text:
# -*- coding: utf-8 -*-
## @package npr_sfs.np.norm
#
#  Numpy norm functions.
#  @author      tody
#  @date        2015/07/18

import numpy as np

## True if x is a vector.
def isVector(x):
    return x.size == x.shape[0]

## True if x is a matrix.
def isMatrix(x):
    return not isVector(x)

## Normalize vector.
def normalizeVector(x):
    norm = np.linalg.norm(x)
    y = x
    if norm > 0:
        y = np.ravel((1.0 / norm) * x)
    return y

## Normalize vectors (n x m matrix).
def normalizeVectors(x):
    norm = normVectors(x)
    nonZeroIDs = norm > 0
    x[nonZeroIDs] = (x[nonZeroIDs].T / norm[nonZeroIDs]).T
    return x

## Norm of vectors (n x m matrix).
def normVectors(x):
    return np.sqrt(l2NormVectors(x))

## L2 norm of vectors (n x m matrix).
#  n x 1 vector: np.square.
#  n x m vectors: np.einsum.
def l2NormVectors(x):
    if isVector(x):
        return np.square(x)
    else:
        return np.einsum('...i,...i', x, x)

max_issues_repo_name: theomission/NPR-SFS

max_forks_repo_name: theomission/NPR-SFS