text:
from __future__ import print_function
from builtins import range

from binpacking.utilities import (
            load_csv,
            save_csvs,
            print_binsizes,
            get,
            argmin,
            revargsort,
        )

def csv_to_constant_bin_number(filepath,
                               weight_column,
                               N_bin,
                               has_header=False,
                               delim=',',
                               quotechar='"',
                               lower_bound=None,
                               upper_bound=None,
                              ):
    """
    Load a csv file, binpack the rows according to one of the columns
    to a constant number of bins.
    Write a new csv file for each bin, containing
    the corresponding rows.
    """

    data, weight_column, header = load_csv(filepath,
                                           weight_column,
                                           has_header=has_header,
                                           delim=delim,
                                           quotechar=quotechar,
                                           )

    bins = to_constant_bin_number(data,
                                  N_bin,
                                  weight_pos=weight_column,
                                  lower_bound=lower_bound,
                                  upper_bound=upper_bound,
                                  )
    print_binsizes(bins, weight_column)

    save_csvs(bins,
              filepath,
              header,
              delim=delim,
              quotechar=quotechar,
              )

def to_constant_bin_number(d,
                           N_bin,
                           weight_pos=None,
                           key=None,
                           lower_bound=None,
                           upper_bound=None,
                       ):
    """
    Distributes a list of weights, a dictionary of weights or a list of tuples containing weights
    to a fixed number of bins while trying to keep the weight distribution constant.

    Parameters
    ==========
    d : iterable
        list containing weights,
        OR dictionary where each (key,value)-pair carries the weight as value,
        OR list of tuples where one entry in the tuple is the weight. The position of
        this weight has to be given in optional variable weight_pos
    N_bin : int
        Number of bins to distribute items to.
    weight_pos : int, default = None
        if d is a list of tuples, this integer number gives the position of the weight in a tuple
    key : function, default = None
        if d is a list, this key functions grabs the weight for an item
    lower_bound : float, default = None
        weights under this bound are not considered
    upper_bound : float, default = None
        weights exceeding this bound are not considered

    Returns
    =======
    bins : list
        A list of length ``N_bin``. Each entry is a list of items or
        a dict of items, depending on the type of ``d``.
    """

    isdict = isinstance(d,dict)

    if not hasattr(d,'__len__'):
        raise TypeError("d must be iterable")

    if not isdict and hasattr(d[0], '__len__'):
        if weight_pos is not None:
            key = lambda x: x[weight_pos]
        if key is None:
            raise ValueError("Must provide weight_pos or key for tuple list")

    if not isdict and key:
        new_dict = {i: val for i, val in enumerate(d)}
        d = {i: key(val) for i, val in enumerate(d)}
        isdict = True
        is_tuple_list = True
    else:
        is_tuple_list = False

    if isdict:

        #get keys and values (weights)
        keys_vals = d.items()
        keys = [ k for k, v in keys_vals ]
        vals = [ v for k, v in keys_vals ]

        #sort weights decreasingly
        ndcs = revargsort(vals)

        weights = get(vals, ndcs)
        keys = get(keys, ndcs)

        bins = [ {} for i in range(N_bin) ]
    else:
        weights = sorted(d,key=lambda x: -x)
        bins = [ [] for i in range(N_bin) ]

    #find the valid indices
    if lower_bound is not None and upper_bound is not None and lower_bound<upper_bound:
        valid_ndcs = filter(lambda i: lower_bound < weights[i] < upper_bound,range(len(weights)))
    elif lower_bound is not None:
        valid_ndcs = filter(lambda i: lower_bound < weights[i],range(len(weights)))
    elif upper_bound is not None:
        valid_ndcs = filter(lambda i: weights[i] < upper_bound,range(len(weights)))
    elif lower_bound is None and upper_bound is None:
        valid_ndcs = range(len(weights))
    elif lower_bound>=upper_bound:
        raise Exception("lower_bound is greater or equal to upper_bound")

    valid_ndcs = list(valid_ndcs)

    weights = get(weights, valid_ndcs)

    if isdict:
        keys = get(keys, valid_ndcs)

    #the total volume is the sum of all weights
    V_total = sum(weights)

    #the first estimate of the maximum bin volume is
    #the total volume divided to all bins
    V_bin_max = V_total / float(N_bin)

    #prepare array containing the current weight of the bins
    weight_sum = [0. for n in range(N_bin) ]

    #iterate through the weight list, starting with heaviest
    for item, weight in enumerate(weights):

        if isdict:
            key = keys[item]

        #put next value in bin with lowest weight sum
        b = argmin(weight_sum)

        #calculate new weight of this bin
        new_weight_sum = weight_sum[b] + weight

        found_bin = False
        while not found_bin:

            #if this weight fits in the bin
            if new_weight_sum <= V_bin_max:

                #...put it in
                if isdict:
                    bins[b][key] = weight
                else:
                    bins[b].append(weight)

                #increase weight sum of the bin and continue with
                #next item
                weight_sum[b] = new_weight_sum
                found_bin = True

            else:
                #if not, increase the max volume by the sum of
                #the rest of the bins per bin
                V_bin_max += sum(weights[item:]) / float(N_bin)

    if not is_tuple_list:
        return bins
    else:
        new_bins = []
        for b in range(N_bin):
            new_bins.append([])
            for key in bins[b]:
                new_bins[b].append(new_dict[key])
        return new_bins

if __name__=="__main__":
    import pylab as pl
    import numpy as np

    a = np.random.power(0.01,size=1000)
    N_bin = 9

    bins = to_constant_bin_number(a,N_bin)
    weight_sums = [np.sum(b) for b in bins]

    #show max values of a and weight sums of the bins
    print(np.sort(a)[-1:-11:-1],weight_sums)

    #plot distribution
    pl.plot(np.arange(N_bin),[np.sum(b) for b in bins])
    pl.ylim([0,max([np.sum(b) for b in bins])+0.1])

    b = { 'a': 10, 'b': 10, 'c':11, 'd':1, 'e': 2,'f':7 }
    bins = to_constant_bin_number(b,4)
    print("===== dict\n",b,"\n",bins)

    lower_bound = None
    upper_bound = None

    b = [ ('a', 10), ('b', 10), ('c',11), ('d',1), ('e', 2),('f',7,'foo') ]
    bins = to_constant_bin_number(b,4,weight_pos=1,lower_bound=lower_bound,upper_bound=upper_bound)
    print("===== list of tuples\n",b,"\n",bins)

    pl.show()

max_stars_repo_name: maprihoda/binpacking

max_issues_repo_name: maprihoda/binpacking

max_forks_repo_name: maprihoda/binpacking


text:
#!/usr/bin/env python
# coding=utf-8
'''
Author: Shuangchi He / Yulv
Email: yulvchi@qq.com
Date: 2022-03-20 18:17:37
Motto: Entities should not be multiplied unnecessarily.
LastEditors: Shuangchi He
LastEditTime: 2022-04-03 17:14:05
FilePath: /Awesome-Ultrasound-Standard-Plane-Detection/src/AgentSPL/agent.py
Description: Modify here please
Init from https://github.com/wulalago/AgentSPL
'''
import random
import torch
import torch.nn as nn
import numpy as np
from torchvision.models import vgg13_bn
from torch.autograd import Variable

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.feature = vgg13_bn(pretrained=False).features
        self.linear1 = nn.Linear(512*7*7, 512)
        self.relu = nn.ReLU(inplace=True)
        self.drop = nn.Dropout(0.5)

        self.fc1_adv = nn.Linear(in_features=512, out_features=128)
        self.fc1_val = nn.Linear(in_features=512, out_features=128)

        self.fc2_adv = nn.Linear(in_features=128, out_features=8)
        self.fc2_val = nn.Linear(in_features=128, out_features=1)

    def forward(self, x):
        x = self.feature(x)
        x = x.view(x.size(0), -1)
        x = self.drop(self.relu(self.linear1(x)))

        adv = self.drop(self.relu(self.fc1_adv(x)))
        val = self.drop(self.relu(self.fc1_val(x)))

        adv = self.fc2_adv(adv)
        val = self.fc2_val(val).expand(x.size(0), 8)

        x = val + adv - adv.mean(1).unsqueeze(1).expand(x.size(0), 8)

        return x

class Agent(object):
    """we define the Double-DQN agent"""
    def __init__(self, args):
        # define the eval net and target net.
        self.eval_net = Net().cuda()
        self.target_net = Net().cuda()

        # define the batch size
        self.batch_size = args.batch_size
        # define the target net weight step counter
        self.target_step_counter = args.target_step_counter
        # define the replay buffer
        self.replay_buffer = NaivePrioritizedBuffer(args.memory_capacity)
        # define the optimizer
        self.optimizer = torch.optim.Adam(
            self.eval_net.parameters(),
            lr=args.lr,
            weight_decay=args.weight_decay,
        )

        # define the loss function
        self.loss_func = nn.MSELoss()
        # define the learning step
        self.learning_step = 0

        # define the epsilon of the greedy algorithm
        self.epsilon = args.epsilon
        # define the reward decay
        self.gamma = args.gamma
        self.frame_idx = 0

    def select_action(self, x, return_q=False, random_action=True):
        self.eval_net.eval()
        x = torch.from_numpy(x[np.newaxis, :, :, :]).cuda()
        if random_action:
            if random.random() < self.epsilon:
                action = self.eval_net.forward(x)
                action_value_arr = action.data.cpu().numpy()
            else:
                # select the random action
                action_value_arr = np.array([random.random() for _ in range(8)], dtype=np.float32)
        else:
            action = self.eval_net.forward(x)
            action_value_arr = action.data.cpu().numpy()

        action_value_arr = np.squeeze(action_value_arr)
        action = action_value_arr.argmax(0)

        if return_q:
            return action, action_value_arr
        else:
            return action

    def learn(self, beta):
        self.eval_net.train()
        self.target_net.eval()
        if self.learning_step % self.target_step_counter == 0:
            # update the target weight after target step counter
            self.target_net.load_state_dict(self.eval_net.state_dict())
        self.learning_step += 1

        if len(self.replay_buffer) < self.batch_size:
            return 0

        if self.learning_step % 10000 == 0 and self.epsilon < 0.95:
            # increase the epsilon to reduce the ability of explore
            self.epsilon *= 1.01

        # sample the state from the buffer.
        state, action, reward, next_state, done, indices, weights = self.replay_buffer.sample(self.batch_size, beta)

        # create variable
        state = Variable(torch.from_numpy(state)).cuda()
        next_state = Variable(torch.from_numpy(next_state)).cuda()
        reward = Variable(torch.from_numpy(reward)).cuda()
        action = Variable(torch.from_numpy(action[:, np.newaxis])).cuda()
        weights = Variable(torch.from_numpy(weights)).cuda()

        # get the current q value
        q_eval = self.eval_net(state)
        q_eval = q_eval.gather(1, action).squeeze()

        # get the target q value and forbid the gradient
        q_next = self.target_net(next_state).detach()
        q_target = (q_next.max(1)[0] * self.gamma) + reward

        # calculate the loss function
        loss = self.loss_func(q_eval, q_target) * weights
        prios = loss + 1e-5

        loss = loss.mean()

        self.optimizer.zero_grad()
        loss.backward()

        # update the replay buffer weights
        self.replay_buffer.update_priorities(indices, prios.data.cpu().numpy())
        self.optimizer.step()

        return loss.item()

class NaivePrioritizedBuffer(object):
    """This is the naive prioritized Buffer that borrowed from the Github (forgot the source....)"""
    def __init__(self, capacity, prob_alpha=0.6):
        self.prob_alpha = prob_alpha
        self.capacity = capacity
        self.buffer = []
        self.pos = 0
        self.priorities = np.zeros((capacity,), dtype=np.float32)

    def push(self, state, action, reward, next_state, done):
        assert state.ndim == next_state.ndim
        state = np.expand_dims(state, 0)
        next_state = np.expand_dims(next_state, 0)

        max_prio = self.priorities.max() if self.buffer else 1.0

        if len(self.buffer) < self.capacity:
            self.buffer.append((state, action, reward, next_state, done))
        else:
            self.buffer[self.pos] = (state, action, reward, next_state, done)

        self.priorities[self.pos] = max_prio
        self.pos = (self.pos + 1) % self.capacity

    def sample(self, batch_size, beta=0.4):
        if len(self.buffer) == self.capacity:
            prios = self.priorities
        else:
            prios = self.priorities[:self.pos]

        probs = prios ** self.prob_alpha
        probs /= probs.sum()

        indices = np.random.choice(len(self.buffer), batch_size, p=probs)
        samples = [self.buffer[idx] for idx in indices]

        total = len(self.buffer)
        weights = (total * probs[indices]) ** (-beta)
        weights /= weights.max()
        weights = np.array(weights, dtype=np.float32)

        batch = zip(*samples)
        batch = [data for data in batch]

        states = np.concatenate(batch[0], axis=0)
        actions = np.array(batch[1], dtype=np.int64)
        rewards = np.array(batch[2], dtype=np.float32)
        next_states = np.concatenate(batch[3], axis=0)
        dones = batch[4]

        return states, actions, rewards, next_states, dones, indices, weights

    def update_priorities(self, batch_indices, batch_priorities):
        for idx, prio in zip(batch_indices, batch_priorities):
            self.priorities[idx] = prio

    def __len__(self):
        return len(self.buffer)

def beta_by_frame(frame_idx):
    beta_start = 0.4
    beta_frames = 2000
    return min(1.0, beta_start + frame_idx * (1.0 - beta_start) / beta_frames)

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf-8

# In[ ]:

from __future__ import print_function
import argparse
import os
import sys
import random
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
cudnn.benchmark = True
cudnn.fastest = True
import torch.optim as optim
import torchvision.utils as vutils
from torch.autograd import Variable

import models.UNet as net

import models.UNet as net_test
from misc import *
from torchsummary import summary
import datetime
import torch.nn.functional as F
import torch.optim as optim
import math
import numpy as np

from basic_toolfunc import normalization_test, local_area, coord_trans, number_to_coord
from models.CNN_models import ConvNet_2
from models.gan_models import G, D

def prior_update_gan_noise(particles, inputs, model_no):
    netG = net.G(5, 3, 64)
    netG.load_state_dict(torch.load('%s/netG_epoch_%d.pth' % ('/home/gtx1080/Abduallah/pix2pix.pytorch/imglog/fluid_noise', model_no)))
    netG.eval()
    netG.cuda()
    mu = np.load('/home/gtx1080/Sync/Kun/30_min_ele/non_ele_whole/train/mu.npy')
    var = np.load('/home/gtx1080/Sync/Kun/30_min_ele/non_ele_whole/train/var.npy')
    particles = np.concatenate((particles, inputs), axis = 1)
    particles, label = normalization_test(particles, particles[:,0:3,:,:], mu, var)
    particles = torch.tensor(particles, dtype = torch.float).cuda()
    with torch.no_grad():
        particles = netG(particles)
    particles = np.array(particles.cpu())
    particles[:,0,:,:] = particles[:,0,:,:]*np.sqrt(var[5,0]) + mu[5,0]
    particles[:,1,:,:] = particles[:,1,:,:]*np.sqrt(var[6,0]) + mu[6,0]
    particles[:,2,:,:] = particles[:,2,:,:]*np.sqrt(var[7,0]) + mu[7,0]
    particles[:,0,:,:] = np.maximum(particles[:,0,:,:], 0)
    return particles

def CNN_update(particles, inputs, model_no):
    model = G()
    model = torch.load('/home/gtx1080/Sync/Kun/0614CNN_Res/{0}_model'.format(model_no))
    model.eval()
    model.cuda()
    mu = np.load('/home/gtx1080/Sync/Kun/30_min_ele/non_ele_whole/train/mu.npy')
    var = np.load('/home/gtx1080/Sync/Kun/30_min_ele/non_ele_whole/train/var.npy')
    particles = np.concatenate((particles, inputs), axis = 1)
    particles, label = normalization_test(particles, particles[:,0:3,:,:], mu, var)
    particles = torch.tensor(particles, dtype = torch.float).cuda()
    with torch.no_grad():
        particles = model(particles)
    particles = np.array(particles.cpu())
    particles[:,0,:,:] = particles[:,0,:,:]*np.sqrt(var[5,0]) + mu[5,0]
    particles[:,1,:,:] = particles[:,1,:,:]*np.sqrt(var[6,0]) + mu[6,0]
    particles[:,2,:,:] = particles[:,2,:,:]*np.sqrt(var[7,0]) + mu[7,0]
    particles[:,0,:,:] = np.maximum(particles[:,0,:,:], 0)
    return particles

# the function to test the GAN models without poeterior update step
def CNNGAN_update_path(particles, inputs, model_no):
    # employed the model 1
    model = G()
    model.load_state_dict(torch.load('trainedmodels/0718_newGAN/{0}_modelG.pth'.format(model_no)))
    model.eval()
    model.cuda()
    mu = np.load('test_code/mu.npy')
    var = np.load('test_code/var.npy')
    particles = np.concatenate((particles, inputs), axis = 1)
    particles, label = normalization_test(particles, particles[:,0:3,:,:], mu, var)
    particles = torch.tensor(particles, dtype = torch.float).cuda()
    with torch.no_grad():
        particles = model(particles)
    particles = np.array(particles.cpu())
    particles[:,0,:,:] = particles[:,0,:,:]*np.sqrt(var[5,0]) + mu[5,0]
    particles[:,1,:,:] = particles[:,1,:,:]*np.sqrt(var[6,0]) + mu[6,0]
    particles[:,2,:,:] = particles[:,2,:,:]*np.sqrt(var[7,0]) + mu[7,0]
    particles[:,0,:,:] = np.maximum(particles[:,0,:,:], 0)
    return particles


text:
import json
import random
import os
from collections import Counter
from glob import glob

from itertools import repeat
from multiprocessing.pool import Pool

from exprimo import get_log_dir
from exprimo.optimizers.base import BaseOptimizer
from exprimo.optimizers.utils import evaluate_placement, apply_placement, generate_random_placement, flatten, \
    get_device_assignment
from exprimo.graph import get_flattened_layer_names, ComputationGraph

import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt
from exprimo import PLOT_STYLE, log
import seaborn as sns

from exprimo.plotting import plot_map_elites_archive, plot_archive_animation

sns.set(style=PLOT_STYLE)

def _evaluate(individual, net_string, groups, device_graph, pipeline_batches=1, batches=1,
              simulator_comp_penalty=1, simulator_comm_penalty=1,
              device_memory_utilization=1):
    description, individual = individual

    comp_graph_dict = apply_placement(net_string, individual, groups)

    score = 1 / evaluate_placement(comp_graph_dict, device_graph,
                                   pipeline_batches=pipeline_batches, batches=batches,
                                   comp_penalty=simulator_comp_penalty, comm_penalty=simulator_comm_penalty,
                                   device_memory_utilization=device_memory_utilization)

    return score, description, individual

class MapElitesOptimizer(BaseOptimizer):

    def __init__(self, dimension_sizes=(-1, -1, 10), initial_size=50,
                 steps=1000, allow_cpu=True, mutation_rate=0.05, copy_mutation_rate=0, replace_mutation_rate=0,
                 zone_mutation_rate=0, crossover_rate=0.4, selection='random', tournament_size=10,
                 benchmarking_function=None, benchmarking_steps=0, benchmark_before_selection=False,
                 benchmarking_n_keep=None, benchmarking_time_threshold=None, include_trivial_solutions=True,
                 show_score_plot=False, plot_axes=(0, 2), plot_animation=False, animation_fps=1,
                 archive_log_period=None, **kwargs):
        super().__init__(**kwargs)
        self.dimension_sizes = dimension_sizes
        self.initial_size = initial_size
        self.steps = steps
        self.allow_cpu = allow_cpu
        self.mutation_rate = mutation_rate
        self.copy_mutation_rate = copy_mutation_rate
        self.replace_mutation_rate = replace_mutation_rate
        self.zone_mutation_rate = zone_mutation_rate
        self.crossover_rate = crossover_rate
        self.selection = selection
        self.tournament_size = tournament_size
        self.include_trivial_solutions = include_trivial_solutions
        self.benchmarking_steps = benchmarking_steps
        self.benchmark_before_selection = benchmark_before_selection
        self.benchmarking_n_keep = benchmarking_n_keep
        self.benchmarking_time_threshold = benchmarking_time_threshold
        self.benchmarking_function = benchmarking_function
        self.plot_axes = plot_axes
        self.show_score_plot = show_score_plot
        self.plot_animation = plot_animation
        self.animation_fps = animation_fps
        self.archive_log_period = archive_log_period

        if self.archive_log_period is not None:
            if not os.path.exists(os.path.join(get_log_dir(), 'archive_logs')):
                os.makedirs(os.path.join(get_log_dir(), 'archive_logs'))

        self.worker_pool = None

    def optimize(self, net_string, device_graph, return_full_archive=False):

        if self.n_threads > 1:
            self.worker_pool = Pool(self.n_threads)

        n_devices = len(device_graph.devices)
        groups = self.create_colocation_groups(get_flattened_layer_names(net_string))

        if self.dimension_sizes[0] == -1:
            self.dimension_sizes[0] = n_devices

        if self.dimension_sizes[1] == -1:
            self.dimension_sizes[1] = n_devices

        if self.dimension_sizes[2] == -1:
            comp_graph = ComputationGraph()
            comp_graph.load_from_string(net_string)
            _, max_jumps = comp_graph.get_number_of_jumps(return_max_jumps=True)
            self.dimension_sizes[2] = max_jumps

        archive_scores = np.empty(self.dimension_sizes)
        archive_scores[:] = np.NaN
        archive_individuals = np.zeros(list(self.dimension_sizes) + [len(groups)], dtype=int)

        def evaluate(individual):
            return _evaluate(individual, net_string, groups, device_graph, self.dimension_sizes, self.pipeline_batches,
                             self.batches, self.simulator_comp_penalty, self.simulator_comm_penalty,
                             self.device_memory_utilization)

        def mutate(individual):
            new_individual = []
            if random.random() < self.replace_mutation_rate:
                devices_present = list(set(individual))
                i1 = random.choice(devices_present)
                i2 = random.choice(devices_present)

                new_individual = [i2 if i == i1 else i for i in individual]
            elif random.random() < self.zone_mutation_rate:
                split1 = random.randint(0, len(individual) - 1)
                split2 = split1 + min(np.random.geometric(0.2), len(individual) - split1)
                dev = random.randint(0 if self.allow_cpu else 1, n_devices - 1)
                new_individual = individual[:split1] + [dev] * (split2 - split1) + individual[split2:]
            else:
                for i, gene in enumerate(individual):
                    if random.random() < self.copy_mutation_rate and i > 0:
                        new_individual.append(individual[i - 1])
                    elif random.random() < self.mutation_rate:
                        if self.allow_cpu:
                            new_individual.append(random.randint(0, n_devices - 1))
                        else:
                            new_individual.append(random.randint(1, n_devices - 1))
                    else:
                        new_individual.append(gene)

            return new_individual

        def crossover(parent1, parent2):
            crossover_point = random.randint(1, len(parent1) - 1)
            return parent1[:crossover_point] + parent2[crossover_point:]

        def create_candidates(n, create_random=False, create_trivial=False, selectable_candidates=None):
            if n <= 0:
                return []
            candidates = []
            if create_trivial:
                candidates.extend([
                    [i] * len(groups) for i in range(1, n_devices)
                ])
                n -= n_devices - 1

                if self.allow_cpu:
                    candidates.append([0] * len(groups))
                    n -= 1

            if create_random:
                while len(candidates) < n:
                    candidates.append(generate_random_placement(len(groups), n_devices, allow_device_0=self.allow_cpu))
            else:
                selectable_indices = np.argwhere(np.isfinite(archive_scores))
                # selectable_indices = sorted(selectable_indices, key=lambda x: -archive_scores[x[0], x[1], x[2]])
                while len(candidates) < n:
                    c = []
                    if selectable_candidates:
                        for _ in range(1 + int(random.random() < self.crossover_rate)):
                            c.append(random.choice(selectable_candidates))
                    else:
                        if self.selection == 'random':
                            for _ in range(1 + int(random.random() < self.crossover_rate)):
                                idx = random.choice(selectable_indices)
                                c.append(archive_individuals[idx[0], idx[1], idx[2], :].tolist())
                        elif self.selection == 'tournament':
                            idx = []
                            t = min(self.tournament_size, len(selectable_indices))
                            while len(idx) < 1 + int(random.random() < self.crossover_rate):
                                competitors = random.sample(selectable_indices.tolist(), t)
                                winner = max(competitors, key=lambda x: archive_scores[x[0], x[1], x[2]])
                                idx.append(winner)
                            for i in idx:
                                c.append(archive_individuals[i[0], i[1], i[2], :].tolist())

                    if len(c) == 2:
                        candidate = crossover(*c)
                    else:
                        candidate = c[0]
                    candidate = mutate(candidate)
                    candidates.append(candidate)

            return candidates

        def create_description(individual):
            c = Counter(individual)
            device_mode = c.most_common(1)[0][0]
            device_mode = round((device_mode / len(device_graph.devices)) * self.dimension_sizes[0])

            used_devices = round(((len(set(individual)) - 1) / (len(device_graph.devices))) * self.dimension_sizes[1])

            comp_graph_dict = apply_placement(net_string, individual, groups)
            comp_graph = ComputationGraph()
            comp_graph.load_from_string(json.dumps(comp_graph_dict))

            num_jumps, max_jumps = comp_graph.get_number_of_jumps(return_max_jumps=True)
            num_jumps = round((num_jumps / max_jumps) * (self.dimension_sizes[2] - 1))

            return (device_mode, used_devices, num_jumps)

        def benchmark(individual, benchmarking_function):
            device_assignment = get_device_assignment(apply_placement(net_string, individual, groups))
            time, memory_overflow = benchmarking_function(device_assignment, return_memory_overflow=True)

            description = create_description(individual)

            # Time is set to -1 if memory overflows - but we check with memory_overflow instead
            time = max(time, 0)

            if memory_overflow == -1:
                memory_overflow = 1

            if memory_overflow > 0:
                time += memory_overflow * 10 ** 9 * 1

            return 1 / time, description, individual

        def reevaluate_archive(benchmarking_function=None, n_keep=None, time_threshold=None):
            indices = list(np.argwhere(np.isfinite(archive_scores)))

            if time_threshold:
                indices = [i for i in indices if archive_scores[i[0], i[1], i[2]] >= 1 / time_threshold]

            if n_keep:
                indices = sorted(indices, key=lambda i: -archive_scores[i[0], i[1], i[2]])
                indices = indices[:n_keep]

            assert len(indices), 'No solutions fulfill the specified requirements'

            archive_scores[:] = np.NaN
            if self.verbose:
                if n_keep:
                    log(f'Reevaluating {n_keep} best individuals in archive (and throwing away the rest)')
                else:
                    log('Reevaluating all individuals in archive')
                if time_threshold:
                    log(f'Time threshold: {time_threshold}ms')
            for i in tqdm(indices, disable=not self.verbose):
                individual = archive_individuals[i[0], i[1], i[2], :].tolist()
                if benchmarking_function:
                    archive_scores[i[0], i[1], i[2]] = benchmark(individual, benchmarking_function)[0]
                else:
                    archive_scores[i[0], i[1], i[2]] = evaluate(individual)[0]

        def log_archive(file_name):
            indices = list(np.argwhere(np.isfinite(archive_scores)))
            indices = sorted(indices, key=lambda i: -archive_scores[i[0], i[1], i[2]])

            with open(os.path.join(get_log_dir(), 'archive_logs', file_name), 'w') as f:
                f.write('niche; time; placement\n')
                for i in indices:
                    niche = tuple(i)
                    time = 1 / archive_scores[i[0], i[1], i[2]]
                    placement = archive_individuals[i[0], i[1], i[2]].tolist()

                    f.write(f'{niche}; {time}; {placement}\n')

        def run_optimization(steps, benchmarking_function=None, start_step=0):
            nonlocal archive_individuals, archive_scores

            if self.verbose:
                if benchmarking_function:
                    log('Optimizing with benchmarking...')
                else:
                    log('Optimizing with simulator...')

            step_size = 1 if benchmarking_function else self.n_threads

            for i in tqdm(range(0, steps, step_size), disable=not self.verbose):
                init_number = min(max(0, self.initial_size - i), self.n_threads)

                if self.include_trivial_solutions and i == 0:
                    candidates = create_candidates(init_number, create_trivial=True, create_random=True)
                else:
                    candidates = create_candidates(init_number, create_random=True)
                if init_number > 0:
                    candidates += create_candidates(self.n_threads - init_number, selectable_candidates=candidates[:])
                else:
                    candidates += create_candidates(self.n_threads - init_number)

                if benchmarking_function:
                    eval_results = [benchmark(candidates[0], benchmarking_function)]
                elif self.n_threads == 1:
                    eval_results = [evaluate(candidates[0])]
                else:
                    fn_args = zip(((create_description(c), c) for c in candidates), repeat(net_string), repeat(groups),
                                  repeat(device_graph), repeat(self.pipeline_batches), repeat(self.batches),
                                  repeat(self.simulator_comp_penalty), repeat(self.simulator_comm_penalty),
                                  repeat(self.device_memory_utilization))

                    eval_results = self.worker_pool.starmap(_evaluate, fn_args)

                for result in eval_results:
                    score, description, individual = result

                    previous_elite_score = archive_scores[description[0], description[1], description[2]]
                    if np.isnan(previous_elite_score) or previous_elite_score < score:
                        archive_scores[description[0], description[1], description[2]] = score
                        archive_individuals[description[0], description[1], description[2], :] = individual

                if self.verbose and (i + 1) % self.verbose < step_size:
                    best_time = 1 / np.nanmax(archive_scores)
                    log(f'[{i + 1}/{steps}] Best time: {best_time:.4f}ms')

                if self.score_save_period and (i % self.score_save_period == 0 or steps - i < step_size):
                    best_time = 1 / np.nanmax(archive_scores)
                    with open(os.path.join(get_log_dir(), 'time_history.csv'), 'a') as f:
                        f.write(f'{i + start_step + 1}, {best_time}\n')

                if self.archive_log_period and (i + 1) % self.archive_log_period < step_size:
                    log_archive(f'step_{i + start_step + 1:06}.csv')

        if self.score_save_period:
            with open(os.path.join(get_log_dir(), 'time_history.csv'), 'w') as f:
                f.write('step, time\n')

        run_optimization(self.steps)

        if self.worker_pool:
            self.worker_pool.close()

        if self.archive_log_period is not None:
            log_archive('1_simulation_finished.csv')

        if self.benchmarking_steps > 0 or self.benchmark_before_selection:
            reevaluate_archive(self.benchmarking_function, n_keep=self.benchmarking_n_keep,
                               time_threshold=self.benchmarking_time_threshold)

            if self.archive_log_period is not None:
                log_archive('2_reevaluated.csv')

        if self.benchmarking_steps > 0:
            run_optimization(self.benchmarking_steps, self.benchmarking_function, self.steps)
            log_archive('3_benchmarking_finished.csv')

        if self.show_score_plot:
            if self.verbose:
                log('Plotting archive scores...', end='')
            graph = ComputationGraph()
            graph.load_from_string(net_string)
            _, max_jumps = graph.get_number_of_jumps(return_max_jumps=True)
            plot_map_elites_archive(archive_scores, n_devices, max_jumps, self.plot_axes,
                                    save_path=os.path.join(get_log_dir(), 'archive_plot.pdf'))
            if self.verbose:
                log('Done')

        if self.plot_animation:
            if not self.archive_log_period and self.verbose:
                log('self.plot_animation was set to True, but archive logging was not enabled. '
                    'Skipping animation plot.')
            else:
                if self.verbose:
                    log('Plotting archive animation...', end='')
                paths = glob(os.path.join(get_log_dir(), 'archive_logs', 'step_*.csv'))
                plot_archive_animation(paths, (os.path.join(get_log_dir(), 'archive_animation.mp4'),
                                               os.path.join(get_log_dir(), 'archive_animation.gif')),
                                       self.dimension_sizes,
                                       n_devices=n_devices, max_jumps=max_jumps, axes=self.plot_axes,
                                       fps=self.animation_fps)
                if self.verbose:
                    log('Done')

        if return_full_archive:
            return archive_scores, archive_individuals

        best_index = np.nanargmax(archive_scores)
        best_individual = archive_individuals.reshape((-1, len(groups)))[best_index]

        if self.verbose:
            log(f'Best individual: {best_individual.tolist()}')

        solution = json.dumps(apply_placement(net_string, best_individual.tolist(), groups), indent=4)

        with open(os.path.join(get_log_dir(), 'me_solution.json'), 'w') as f:
            f.write(solution)

        return solution

max_stars_repo_name: Lagostra/exprimo

max_issues_repo_name: Lagostra/exprimo

max_forks_repo_name: Lagostra/exprimo


text:
import tensorflow as tf
import numpy as np
import itertools as it
import pickle
from pathlib import Path
from tophat.tasks.wrapper import FactorizationTaskWrapper
import tophat.callbacks as cbks
from tophat.evaluation.transport import items_pred_dicter
from tophat.utils.io import write_vocab
from typing import Optional, List, Sequence, Any, Union

class TophatModel(object):

    def __init__(self,
                 tasks: List[FactorizationTaskWrapper],
                 task_weights: List[float] = None,
                 sess: Optional[tf.Session] = None,
                 ):

        self.tasks = tasks
        # Assure all tasks are built
        for task in self.tasks:
            if not task.built:
                task.build()

        self.steps_per_epoch = sum((t.steps_per_epoch for t in self.tasks))
        self.task_samp_weights = np.array([
            t.steps_per_epoch / self.steps_per_epoch for t in self.tasks
        ])
        task_weights = task_weights or [1] * len(tasks)
        self.task_samp_weights *= task_weights
        self.task_samp_weights = (self.task_samp_weights /
                                  self.task_samp_weights.sum())

        self.sess = sess
        self.sess_init()

        # Assume embedding map is shared for all tasks, just grab first
        self.embedding_map = self.tasks[0].embedding_map

        self.global_step = 0
        self.loss_hists = None

    def sess_init(self):
        self.sess = self.sess or tf.Session()
        # Set session on sampler (in case of adaptive sampling)
        for task in self.tasks:
            task.sampler.sess = self.sess
        init = tf.global_variables_initializer()
        self.sess.run(init)

    def fit(self,
            n_epochs: Optional[int] = 1,
            callbacks: Optional[List[cbks.Callback]] = None,
            verbose: int = 1,
            ):
        """Alternating task fit

        Args:
            n_epochs: number of effective epochs
                This is approximate since we're randomly alternating tasks
            callbacks: list of callbacks to perform during fit loop
            verbose: periodicity (in epochs) of logging

        Returns:

        """

        n_tasks_per_epoch = int(self.steps_per_epoch)
        self.loss_hists = [cbks.TaskLossHistory(t.name) for t in self.tasks]

        _callbacks = (callbacks or []) + self.loss_hists
        if verbose:
            _callbacks.append(cbks.Monitor(self.loss_hists, verbose))

        for c in _callbacks:
            if hasattr(c, 'sess') and c.sess is None:
                c.sess = self.sess

        callbacks = cbks.CallbackList(_callbacks)

        callbacks.on_train_begin()

        for epoch_ind in range(n_epochs):
            callbacks.on_epoch_begin(epoch_ind)

            if len(self.tasks) == 1:
                sampled_tasks = it.repeat(self.tasks[0], n_tasks_per_epoch)
            else:
                sampled_tasks = np.random.choice(
                    self.tasks, n_tasks_per_epoch, p=self.task_samp_weights)

            for step_ind, task in enumerate(sampled_tasks):
                batch_logs = {'batch': step_ind, 'size': task.batch_size,
                              'task': task.name}

                callbacks.on_batch_begin(step_ind, batch_logs)

                # Perform operations
                task_loss, _ = self.sess.run([task.loss, task.train_op])
                batch_logs['loss'] = task_loss

                callbacks.on_batch_end(step_ind, batch_logs)
                self.global_step += 1

            callbacks.on_epoch_end(epoch_ind)
        callbacks.on_train_end()

    def predict(self,
                user_id: Any,
                item_ids: Sequence[Any],
                task: Optional[FactorizationTaskWrapper] = None,
                ):

        task_wrapper = task or self.tasks[0]
        data_loader = task_wrapper.data_loader
        feat_codes_df = data_loader.feats_codes_df
        num_feats_df = data_loader.num_feats_df

        input_fwd_d = task_wrapper.task.get_fwd_dict()

        input_tensors = items_pred_dicter(
            user_id=user_id,
            item_ids=item_ids,
            cat_codes_dfs=feat_codes_df,
            num_feats_dfs=num_feats_df,
            input_fwd_d=input_fwd_d,
        )

        preds_op = task_wrapper.task.forward(input_fwd_d)
        # TODO: consider moving from feed_dict to tf.data.Dataset ? (not sure)
        preds_arr = self.sess.run(preds_op, feed_dict=input_tensors)

        return preds_arr

    def write_vocab(self, dir_export: Union[str, Path]):
        write_vocab(dir_export, self.embedding_map.cats_d)

    def write_cats(self, path_export):
        pickle.dump(self.embedding_map.cats_d, open(path_export, 'wb'))

max_stars_repo_name: JasonTam/tophat

max_issues_repo_name: JasonTam/tophat

max_forks_repo_name: JasonTam/tophat


text:
from numpy.core.fromnumeric import mean
from tune_hyper import train

config = {"JK": 'cat', "dp_1": 0.6, "dp_2": 0.6, "gat_dp": 0.6, "gat_out_dim": 32, "lr": 0.001,
    "trans_dp": 0.3, "trans_expand": 2, "trans_head": 2, "trans_nlayer": 4, "wd": 0.0001, 'double_linear': True, 'scheduler': 'cosine'}
mean_acc = train(config)
print(mean_acc)

max_stars_repo_name: itsdaniele/graphtrans

max_issues_repo_name: itsdaniele/graphtrans

max_forks_repo_name: itsdaniele/graphtrans

reason: from numpy


text:
import os
import csv
import sys
import json
import pickle
import logging
import argparse
import numpy as np
from tqdm import tqdm
from pygtrie import CharTrie
from collections import defaultdict
from nltk.translate import bleu_score
from nltk.translate.bleu_score import SmoothingFunction

logging.basicConfig(
    stream=sys.stdout,
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s ",
    datefmt="%m/%d/%Y %H:%M:%S",
    level=logging.INFO,
)
LOGGER = logging.getLogger(__name__)

def calc_mrr(gt_data_iterator, pred_data_iterator):
    all_mrr = []
    all_bleu = []
    for j, (gt_label, preds) in tqdm(
        enumerate(zip(gt_data_iterator, pred_data_iterator)), total=len(gt_data_iterator)
    ):
        preds = [p for p in json.loads(preds.lower())]
        mrr = 0.0
        for i, curr_pred in enumerate(preds):
            if gt_label == curr_pred:
                mrr = 1.0 / (i + 1)
                break

        all_mrr.append(mrr)
        wgtd_bleu_score = 0.0
        normalizer = 0.0
        for i, curr_pred in enumerate(preds):
            wgtd_bleu_score += (
                bleu_score.sentence_bleu(
                    [gt_label.split()],
                    curr_pred.split(),
                    smoothing_function=SmoothingFunction().method1,
                )
                / (i + 1)
            )
            normalizer += 1.0 / (i + 1)
        wgtd_bleu_score = wgtd_bleu_score / normalizer if normalizer > 0 else wgtd_bleu_score
        all_bleu.append(wgtd_bleu_score)

    return {
        "mrr": np.mean(all_mrr) if len(all_mrr) > 0 else 0.0,
        "mrr_err": np.std(all_mrr) if len(all_mrr) > 0 else 0.0,
        "bleu": np.mean(all_bleu) if len(all_bleu) > 0 else 0.0,
        "bleu_err": np.std(all_bleu) if len(all_bleu) > 0 else 0.0,
        "num_points": len(all_mrr),
    }

def get_data_by_freq_and_pref_len(gt_filename, pred_filename, label_trie, only_seen_labels):

    freq_to_data = defaultdict(list)
    pref_len_to_data = defaultdict(list)
    with open(gt_filename, "r") as all_gt_file, open(pred_filename, "r") as main_pred_file:
        for line_ctr, (line_gt, line_pred) in enumerate(zip(all_gt_file, main_pred_file)):
            gt_dict = json.loads(line_gt)
            gt_label = gt_dict["next_query"].lower()  # ground truth next_query
            prefix = gt_dict["prefix"].lower()  # prefix

            curr_label_freq = label_trie[gt_label] if gt_label in label_trie else 0

            freq_to_data[curr_label_freq] += [(gt_label, line_pred)]
            if only_seen_labels:
                if curr_label_freq > 0:
                    pref_len_to_data[len(prefix)] += [(gt_label, line_pred)]
                else:
                    pass
            else:
                pref_len_to_data[len(prefix)] += [(gt_label, line_pred)]

    new_freq_to_data = defaultdict(lambda: ([], []))
    for freq in freq_to_data:
        gt_labels, preds = zip(*freq_to_data[freq])
        new_freq_to_data[freq] = list(gt_labels), list(preds)

    new_pref_len_to_data = defaultdict(lambda: ([], []))
    for pref_len in pref_len_to_data:
        gt_labels, preds = zip(*pref_len_to_data[pref_len])
        new_pref_len_to_data[pref_len] = list(gt_labels), list(preds)

    return new_freq_to_data, new_pref_len_to_data

def write_results(pred_fname, gt_fname, feat_name, feat_to_scores, res_file):
    eval_metrics = list(calc_mrr([], []).keys())
    fieldnames = [feat_name] + eval_metrics + ["pred_fname", "gt_fname"]

    with open(res_file, "w") as f:
        csv_writer = csv.DictWriter(f, fieldnames=fieldnames)
        csv_writer.writeheader()
        for feat in sorted(feat_to_scores):
            temp_dict = {feat_name: feat, "pred_fname": pred_fname, "gt_fname": gt_fname}
            temp_dict.update(feat_to_scores[feat])
            csv_writer.writerow(temp_dict)

def get_bucketed_scores(feat_to_data, feat_buckets):
    feat_bkt_to_scores = {}
    for min_feat_val, max_feat_val in zip(feat_buckets[:-1], feat_buckets[1:]):
        gt_data = []
        pred_data = []
        for curr_feat_val in range(min_feat_val, max_feat_val):
            gt_data += feat_to_data[curr_feat_val][0]
            pred_data += feat_to_data[curr_feat_val][1]

        feat_bkt_to_scores[min_feat_val, max_feat_val] = calc_mrr(
            gt_data_iterator=gt_data, pred_data_iterator=pred_data
        )

    return feat_bkt_to_scores

def analyse(gt_filename, pred_filename, label_trie, only_seen_labels):

    freq_to_data, pref_len_to_data = get_data_by_freq_and_pref_len(
        gt_filename=gt_filename,
        pred_filename=pred_filename,
        label_trie=label_trie,
        only_seen_labels=only_seen_labels,
    )

    freq_to_scores = {
        freq: calc_mrr(gt_data_iterator=gt_data, pred_data_iterator=pred_data)
        for freq, (gt_data, pred_data) in freq_to_data.items()
    }

    pref_len_to_scores = {
        pref_len: calc_mrr(gt_data_iterator=gt_data, pred_data_iterator=pred_data)
        for pref_len, (gt_data, pred_data) in pref_len_to_data.items()
    }

    out_dir = os.path.dirname(pred_filename)

    test_fname = gt_filename.split("/")[-1]
    # Write results for each label freq to csv file
    res_file = f"{out_dir}/{test_fname}_eval_res_by_seen_in_train_bkt_{only_seen_labels}.csv"
    bktd_freq_to_scores = get_bucketed_scores(feat_to_data=freq_to_data, feat_buckets=[0, 1, 80000])
    write_results(
        pred_fname=pred_filename,
        feat_name="freq_bkt",
        feat_to_scores=bktd_freq_to_scores,
        res_file=res_file,
        gt_fname=gt_filename,
    )

    res_file = f"{out_dir}/{test_fname}_eval_res_by_freq_{only_seen_labels}.csv"
    write_results(
        pred_fname=pred_filename,
        feat_name="freq",
        feat_to_scores=freq_to_scores,
        res_file=res_file,
        gt_fname=gt_filename,
    )

    res_file = f"{out_dir}/{test_fname}_eval_res_by_freq_bkt_{only_seen_labels}.csv"
    bktd_freq_to_scores = get_bucketed_scores(
        feat_to_data=freq_to_data, feat_buckets=[0, 1, 32, 128, 512, 2048, 8192, 32000, 80000]
    )
    write_results(
        pred_fname=pred_filename,
        feat_name="freq_bkt",
        feat_to_scores=bktd_freq_to_scores,
        res_file=res_file,
        gt_fname=gt_filename,
    )

    # Write results for each label prefix-length to csv file
    res_file = f"{out_dir}/{test_fname}_eval_res_by_pref_len_{only_seen_labels}.csv"
    write_results(
        pred_fname=pred_filename,
        feat_name="pref_len",
        feat_to_scores=pref_len_to_scores,
        res_file=res_file,
        gt_fname=gt_filename,
    )

    # Write results for each label prefix-length to csv file bucketed
    max_pref_bkt = 26
    max_pref = max(pref_len_to_scores.keys())
    pref_feat_bkts = list(range(max_pref_bkt))
    if max_pref > max_pref_bkt:
        pref_feat_bkts = pref_feat_bkts + [max_pref + 1]
    res_file = (
        f"{out_dir}/{test_fname}_eval_res_by_pref_len_bkt_{max_pref_bkt}_{only_seen_labels}.csv"
    )
    bktd_pref_len_to_scores = get_bucketed_scores(
        feat_to_data=pref_len_to_data, feat_buckets=pref_feat_bkts
    )
    write_results(
        pred_fname=pred_filename,
        feat_name="pref_len_bkt",
        feat_to_scores=bktd_pref_len_to_scores,
        res_file=res_file,
        gt_fname=gt_filename,
    )

def main():
    parser = argparse.ArgumentParser(
        description="Breakdown performance of model given prediction file"
    )
    parser.add_argument(
        "--gt_file", type=str, required=True, help="file containing ground-truth data"
    )
    parser.add_argument(
        "--pred_file", type=str, required=True, help="file containing prediction data"
    )
    parser.add_argument(
        "--label_file", type=str, default=None, help="file containing label freq trie"
    )

    np.random.seed(0)

    args = parser.parse_args()
    _gt_file = args.gt_file
    _pred_file = args.pred_file
    _label_file = args.label_file if args.label_file != "" else None

    LOGGER.info("Loading label freq trie from {}".format(_label_file))
    if _label_file is None:
        label_trie = defaultdict(int)
    else:
        with open(_label_file, "rb") as f:
            label_trie = pickle.load(f)
        assert isinstance(label_trie, CharTrie)
    LOGGER.info("Loaded label freq trie")

    analyse(gt_filename=_gt_file, pred_filename=_pred_file, label_trie=label_trie)

def final_paper_analysis():
    parser = argparse.ArgumentParser(
        description="Breakdown performance of model given prediction file"
    )
    parser.add_argument("--opt", type=int, default=-1, help="option")
    parser.add_argument(
        "--base_dir", type=str, required=True, help="dir with all relevant result folder"
    )
    parser.add_argument(
        "--label_file",
        type=str,
        default="",
        help="pickle file containing dict/trie that maps " "label to its frequency",
    )
    parser.add_argument("--gt_file", type=str, required=True, help="json file containing gt data")
    args = parser.parse_args()
    opt = args.opt
    base_dir = args.base_dir
    label_file = args.label_file
    gt_file = args.gt_file

    pecos_08 = f"{base_dir}/7_ReproduceRes/d=aol_v=poswgtd_c-tfidf_i=hybridindexer_s=0_08"
    freq = f"{base_dir}/8_FinalResults/PrefFreqSuggester"
    seq2seqGRU = f"{base_dir}/8_Baseline/d=aol_m=HRED_s=0"
    seq2seqGRU_freq = f"{base_dir}/8_Baseline/d=aol_m=HRED_s=0_rerank"

    LOGGER.info("Loading label freq trie from {}".format(label_file))
    if label_file is None:
        label_trie = defaultdict(int)
        only_seen_labels = False
    else:
        label_trie = pickle.load(open(label_file, "rb"))
        only_seen_labels = True
        assert isinstance(label_trie, CharTrie)
    LOGGER.info("Loaded label freq trie")

    if opt == 1:
        all_dirs = [pecos_08]
    elif opt == 2:
        all_dirs = [freq]
    elif opt == 3:
        all_dirs = [seq2seqGRU]
    elif opt == 4:
        all_dirs = [seq2seqGRU_freq]
    else:
        all_dirs = [pecos_08, freq, seq2seqGRU, seq2seqGRU_freq]

    for curr_dir in tqdm(all_dirs):
        print("Running for dir = {}".format(curr_dir))
        try:
            pred_file = f"{curr_dir}/eval/{curr_dir.split('/')[-1]}_test.json.pred_data"
            analyse(
                gt_filename=gt_file,
                pred_filename=pred_file,
                label_trie=label_trie,
                only_seen_labels=only_seen_labels,
            )
            analyse(
                gt_filename=gt_file,
                pred_filename=pred_file,
                label_trie=defaultdict(int),
                only_seen_labels=False,
            )
        except Exception as e:
            print(f"1) Error processing {curr_dir} {e}")

if __name__ == "__main__":
    # main()
    final_paper_analysis()

max_stars_repo_name: xeisberg/pecos

max_issues_repo_name: xeisberg/pecos

max_forks_repo_name: xeisberg/pecos


text:
import numpy as np

def rms(data, reference):
    rms = np.sqrt(np.sum(np.ndarray.flatten(np.square(data - reference)))/np.size(data))
    return rms

def rms_array(data, reference):
    rms_array = rms(data, reference)
    rms_array = np.average(np.ndarray.flatten(rms_array))
    return rms_array

max_stars_repo_name: shmouses/SpectrumImageAnalysisPy

max_issues_repo_name: shmouses/SpectrumImageAnalysisPy

max_forks_repo_name: icbicket/SpectrumImageAnalysisPy


text:
import numpy as np
import copy
from scipy.special import expit, softmax

from common.wbf_postprocess import weighted_boxes_fusion

def yolo_decode(prediction, anchors, num_classes, input_dims, scale_x_y=None, use_softmax=False):
    '''Decode final layer features to bounding box parameters.'''
    batch_size = np.shape(prediction)[0]
    num_anchors = len(anchors)

    grid_size = np.shape(prediction)[1:3]
    #check if stride on height & width are same
    assert input_dims[0]//grid_size[0] == input_dims[1]//grid_size[1], 'model stride mismatch.'
    stride = input_dims[0] // grid_size[0]

    prediction = np.reshape(prediction,
                            (batch_size, grid_size[0] * grid_size[1] * num_anchors, num_classes + 5))

    ################################
    # generate x_y_offset grid map
    grid_y = np.arange(grid_size[0])
    grid_x = np.arange(grid_size[1])
    x_offset, y_offset = np.meshgrid(grid_x, grid_y)

    x_offset = np.reshape(x_offset, (-1, 1))
    y_offset = np.reshape(y_offset, (-1, 1))

    x_y_offset = np.concatenate((x_offset, y_offset), axis=1)
    x_y_offset = np.tile(x_y_offset, (1, num_anchors))
    x_y_offset = np.reshape(x_y_offset, (-1, 2))
    x_y_offset = np.expand_dims(x_y_offset, 0)

    ################################

    # Log space transform of the height and width
    anchors = np.tile(anchors, (grid_size[0] * grid_size[1], 1))
    anchors = np.expand_dims(anchors, 0)

    box_xy = (expit(prediction[..., :2]) + x_y_offset) / np.array(grid_size)[::-1]
    box_wh = (np.exp(prediction[..., 2:4]) * anchors) / np.array(input_dims)[::-1]

    # Sigmoid objectness scores
    objectness = expit(prediction[..., 4])  # p_o (objectness score)
    objectness = np.expand_dims(objectness, -1)  # To make the same number of values for axis 0 and 1

    if use_softmax:
        # Softmax class scores
        class_scores = softmax(prediction[..., 5:], axis=-1)
    else:
        # Sigmoid class scores
        class_scores = expit(prediction[..., 5:])

    return np.concatenate([box_xy, box_wh, objectness, class_scores], axis=2)

def yolo_correct_boxes(predictions, img_shape, model_image_size):
    '''rescale predicition boxes back to original image shape'''
    box_xy = predictions[..., :2]
    box_wh = predictions[..., 2:4]
    objectness = np.expand_dims(predictions[..., 4], -1)
    class_scores = predictions[..., 5:]

    # model_image_size & image_shape should be (height, width) format
    model_image_size = np.array(model_image_size, dtype='float32')
    image_shape = np.array(img_shape, dtype='float32')
    height, width = image_shape

    new_shape = np.round(image_shape * np.min(model_image_size/image_shape))
    offset = (model_image_size-new_shape)/2./model_image_size
    scale = model_image_size/new_shape
    # reverse offset/scale to match (w,h) order
    offset = offset[..., ::-1]
    scale = scale[..., ::-1]

    box_xy = (box_xy - offset) * scale
    box_wh *= scale

    # Convert centoids to top left coordinates
    box_xy -= box_wh / 2

    # Scale boxes back to original image shape.
    image_wh = image_shape[..., ::-1]
    box_xy *= image_wh
    box_wh *= image_wh

    return np.concatenate([box_xy, box_wh, objectness, class_scores], axis=2)

def yolo_handle_predictions(predictions, image_shape, max_boxes=100, confidence=0.1, iou_threshold=0.4, use_cluster_nms=False, use_wbf=False):
    boxes = predictions[:, :, :4]
    box_confidences = np.expand_dims(predictions[:, :, 4], -1)
    box_class_probs = predictions[:, :, 5:]

    # filter boxes with confidence threshold
    box_scores = box_confidences * box_class_probs
    box_classes = np.argmax(box_scores, axis=-1)
    box_class_scores = np.max(box_scores, axis=-1)
    pos = np.where(box_class_scores >= confidence)

    boxes = boxes[pos]
    classes = box_classes[pos]
    scores = box_class_scores[pos]

    if use_cluster_nms:
        # use Fast/Cluster NMS for boxes postprocess
        n_boxes, n_classes, n_scores = fast_cluster_nms_boxes(boxes, classes, scores, iou_threshold, confidence=confidence)
    elif use_wbf:
        # use Weighted-Boxes-Fusion for boxes postprocess
        n_boxes, n_classes, n_scores = weighted_boxes_fusion([boxes], [classes], [scores], image_shape, weights=None, iou_thr=iou_threshold)
    else:
        # Boxes, Classes and Scores returned from NMS
        n_boxes, n_classes, n_scores = nms_boxes(boxes, classes, scores, iou_threshold, confidence=confidence)

    if n_boxes:
        boxes = np.concatenate(n_boxes)
        classes = np.concatenate(n_classes).astype('int32')
        scores = np.concatenate(n_scores)
        boxes, classes, scores = filter_boxes(boxes, classes, scores, max_boxes)

        return boxes, classes, scores

    else:
        return [], [], []

def box_iou(boxes):
    """
    Calculate IoU value of 1st box with other boxes of a box array

    Parameters
    ----------
    boxes: bbox numpy array, shape=(N, 4), xywh
           x,y are top left coordinates

    Returns
    -------
    iou: numpy array, shape=(N-1,)
         IoU value of boxes[1:] with boxes[0]
    """
    # get box coordinate and area
    x = boxes[:, 0]
    y = boxes[:, 1]
    w = boxes[:, 2]
    h = boxes[:, 3]
    areas = w * h

    # check IoU
    inter_xmin = np.maximum(x[1:], x[0])
    inter_ymin = np.maximum(y[1:], y[0])
    inter_xmax = np.minimum(x[1:] + w[1:], x[0] + w[0])
    inter_ymax = np.minimum(y[1:] + h[1:], y[0] + h[0])

    inter_w = np.maximum(0.0, inter_xmax - inter_xmin + 1)
    inter_h = np.maximum(0.0, inter_ymax - inter_ymin + 1)

    inter = inter_w * inter_h
    iou = inter / (areas[1:] + areas[0] - inter)
    return iou

def box_diou(boxes):
    """
    Calculate DIoU value of 1st box with other boxes of a box array
    Reference Paper:
        "Distance-IoU Loss: Faster and Better Learning for Bounding Box Regression"
        https://arxiv.org/abs/1911.08287

    Parameters
    ----------
    boxes: bbox numpy array, shape=(N, 4), xywh
           x,y are top left coordinates

    Returns
    -------
    diou: numpy array, shape=(N-1,)
         IoU value of boxes[1:] with boxes[0]
    """
    # get box coordinate and area
    x = boxes[:, 0]
    y = boxes[:, 1]
    w = boxes[:, 2]
    h = boxes[:, 3]
    areas = w * h

    # check IoU
    inter_xmin = np.maximum(x[1:], x[0])
    inter_ymin = np.maximum(y[1:], y[0])
    inter_xmax = np.minimum(x[1:] + w[1:], x[0] + w[0])
    inter_ymax = np.minimum(y[1:] + h[1:], y[0] + h[0])

    inter_w = np.maximum(0.0, inter_xmax - inter_xmin + 1)
    inter_h = np.maximum(0.0, inter_ymax - inter_ymin + 1)

    inter = inter_w * inter_h
    iou = inter / (areas[1:] + areas[0] - inter)

    # box center distance
    x_center = x + w/2
    y_center = y + h/2
    center_distance = np.power(x_center[1:] - x_center[0], 2) + np.power(y_center[1:] - y_center[0], 2)

    # get enclosed area
    enclose_xmin = np.minimum(x[1:], x[0])
    enclose_ymin = np.minimum(y[1:], y[0])
    enclose_xmax = np.maximum(x[1:] + w[1:], x[0] + w[0])
    enclose_ymax = np.maximum(x[1:] + w[1:], x[0] + w[0])
    enclose_w = np.maximum(0.0, enclose_xmax - enclose_xmin + 1)
    enclose_h = np.maximum(0.0, enclose_ymax - enclose_ymin + 1)
    # get enclosed diagonal distance
    enclose_diagonal = np.power(enclose_w, 2) + np.power(enclose_h, 2)
    # calculate DIoU, add epsilon in denominator to avoid dividing by 0
    diou = iou - 1.0 * (center_distance) / (enclose_diagonal + np.finfo(float).eps)

    return diou

def nms_boxes(boxes, classes, scores, iou_threshold, confidence=0.1, use_diou=True, is_soft=False, use_exp=False, sigma=0.5):
    nboxes, nclasses, nscores = [], [], []
    for c in set(classes):
        # handle data for one class
        inds = np.where(classes == c)
        b = boxes[inds]
        c = classes[inds]
        s = scores[inds]

        # make a data copy to avoid breaking
        # during nms operation
        b_nms = copy.deepcopy(b)
        c_nms = copy.deepcopy(c)
        s_nms = copy.deepcopy(s)

        while len(s_nms) > 0:
            # pick the max box and store, here
            # we also use copy to persist result
            i = np.argmax(s_nms, axis=-1)
            nboxes.append(copy.deepcopy(b_nms[i]))
            nclasses.append(copy.deepcopy(c_nms[i]))
            nscores.append(copy.deepcopy(s_nms[i]))

            # swap the max line and first line
            b_nms[[i,0],:] = b_nms[[0,i],:]
            c_nms[[i,0]] = c_nms[[0,i]]
            s_nms[[i,0]] = s_nms[[0,i]]

            if use_diou:
                iou = box_diou(b_nms)
                #iou = box_diou_matrix(b_nms, b_nms)[0][1:]
            else:
                iou = box_iou(b_nms)
                #iou = box_iou_matrix(b_nms, b_nms)[0][1:]

            # drop the last line since it has been record
            b_nms = b_nms[1:]
            c_nms = c_nms[1:]
            s_nms = s_nms[1:]

            if is_soft:
                # Soft-NMS
                if use_exp:
                    # score refresh formula:
                    # score = score * exp(-(iou^2)/sigma)
                    s_nms = s_nms * np.exp(-(iou * iou) / sigma)
                else:
                    # score refresh formula:
                    # score = score * (1 - iou) if iou > threshold
                    depress_mask = np.where(iou > iou_threshold)[0]
                    s_nms[depress_mask] = s_nms[depress_mask]*(1-iou[depress_mask])
                keep_mask = np.where(s_nms >= confidence)[0]
            else:
                # normal Hard-NMS
                keep_mask = np.where(iou <= iou_threshold)[0]

            # keep needed box for next loop
            b_nms = b_nms[keep_mask]
            c_nms = c_nms[keep_mask]
            s_nms = s_nms[keep_mask]

    # reformat result for output
    nboxes = [np.array(nboxes)]
    nclasses = [np.array(nclasses)]
    nscores = [np.array(nscores)]
    return nboxes, nclasses, nscores

def box_iou_matrix(boxes1, boxes2):
    """
    Calculate IoU matrix for two box array.
    Both sets of boxes are expected to be in (x, y, w, h) format.
    Reference implementation:
        https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py

    Arguments:
        boxes1 (np.array[N, 4])
        boxes2 (np.array[M, 4])
    Returns:
        iou (np.array[N, M]): the NxM matrix containing the pairwise
            IoU values for every element in boxes1 and boxes2
    """

    def box_area(box):
        # box = 4xN
        return box[2] * box[3]

    area1 = box_area(boxes1.T)
    area2 = box_area(boxes2.T)

    inter_min = np.maximum(boxes1[:, None, :2], boxes2[:, :2])  # [N,M,2]
    inter_max = np.minimum(boxes1[:, None, :2]+boxes1[:, None, 2:], boxes2[:, :2]+boxes2[:, 2:])  # [N,M,2]
    inter = np.maximum(inter_max - inter_min, 0).prod(axis=-1)  # [N,M]

    iou = inter / (area1[:, None] + area2 - inter)  # iou = inter / (area1 + area2 - inter)
    return iou

def box_diou_matrix(boxes1, boxes2):
    """
    Calculate DIoU matrix for two box array.
    Both sets of boxes are expected to be in (x, y, w, h) format.

    Arguments:
        boxes1 (np.array[N, 4])
        boxes2 (np.array[M, 4])
    Returns:
        diou (np.array[N, M]): the NxM matrix containing the pairwise
            IoU values for every element in boxes1 and boxes2
    """
    iou = box_iou_matrix(boxes1, boxes2)

    # box center distance
    center_distance = (boxes1[:, None, :2]+boxes1[:, None, 2:]/2) - (boxes2[:, :2]+boxes2[:, 2:]/2)  # [N,M,2]
    center_distance = np.power(center_distance[..., 0], 2) + np.power(center_distance[..., 1], 2)  # [N,M]

    # get enclosed area
    enclose_min = np.minimum(boxes1[:, None, :2], boxes2[:, :2])  # [N,M,2]
    enclose_max = np.maximum(boxes1[:, None, :2]+boxes1[:, None, 2:], boxes2[:, :2]+boxes2[:, 2:])  # [N,M,2]

    enclose_wh = np.maximum(enclose_max - enclose_min, 0) # [N,M,2]
    enclose_wh = np.maximum(enclose_max - enclose_min, 0) # [N,M,2]

    # get enclosed diagonal distance matrix
    enclose_diagonal = np.power(enclose_wh[..., 0], 2) + np.power(enclose_wh[..., 1], 2)  # [N,M]

    # calculate DIoU, add epsilon in denominator to avoid dividing by 0
    diou = iou - 1.0 * np.true_divide(center_distance, enclose_diagonal + np.finfo(float).eps)

    return diou

def fast_cluster_nms_boxes(boxes, classes, scores, iou_threshold, confidence=0.1, use_cluster=True, use_diou=True, use_weighted=True, use_matrix_nms=False, use_spm=False):
    """
    Fast NMS/Cluster NMS/Matrix NMS bbox post process
    Reference Paper:
        1. "YOLACT: Real-time Instance Segmentation"
           https://arxiv.org/abs/1904.02689

        2. "Enhancing Geometric Factors in Model Learning and Inference for Object Detection and Instance Segmentation"
           https://arxiv.org/abs/2005.03572

        3. "SOLOv2: Dynamic, Faster and Stronger"
           https://arxiv.org/abs/2003.10152

        4. Blogpost on zhihu:
           https://zhuanlan.zhihu.com/p/157900024

    Parameters
    ----------
    boxes:   bbox numpy array, shape=(N, 4), xywh
             x,y are top left coordinates
    classes: bbox class index numpy array, shape=(N, 1)
    scores:  bbox score numpy array, shape=(N, 1)
    iou_threshold:

    Returns
    -------
    nboxes:   NMSed bbox numpy array, shape=(N, 4), xywh
              x,y are top left coordinates
    nclasses: NMSed bbox class index numpy array, shape=(N, 1)
    nscores:  NMSed bbox score numpy array, shape=(N, 1)
    """
    nboxes, nclasses, nscores = [], [], []
    for c in set(classes):
        # handle data for one class
        inds = np.where(classes == c)
        b = boxes[inds]
        c = classes[inds]
        s = scores[inds]

        # make a data copy to avoid breaking
        # during nms operation
        b_nms = copy.deepcopy(b)
        c_nms = copy.deepcopy(c)
        s_nms = copy.deepcopy(s)

        # ascend sort boxes according to scores
        sorted_indices = np.argsort(s_nms)
        sorted_indices = sorted_indices[::-1]
        b_nms = b_nms[sorted_indices]
        c_nms = c_nms[sorted_indices]
        s_nms = s_nms[sorted_indices]

        # number of boxes for one class
        num_boxes = b_nms.shape[0]

        # get IoU/DIoU matrix (upper triangular matrix)
        if use_diou:
            iou_matrix = box_diou_matrix(b_nms, b_nms)
        else:
            iou_matrix = box_iou_matrix(b_nms, b_nms)
        iou_matrix = np.triu(iou_matrix, k=1)
        max_iou = np.max(iou_matrix, axis=0)
        updated_iou_matrix = copy.deepcopy(iou_matrix)

        # Cluster loop
        if use_cluster:
            for i in range(200):
                prev_iou_matrix = copy.deepcopy(updated_iou_matrix)
                max_iou = np.max(prev_iou_matrix, axis=0)
                keep_diag = np.diag((max_iou < iou_threshold).astype(np.float32))
                updated_iou_matrix = np.dot(keep_diag, iou_matrix)
                if (prev_iou_matrix == updated_iou_matrix).all():
                    break

        if use_matrix_nms:
            # Matrix NMS
            max_iou_expand = np.tile(max_iou, (num_boxes, 1)).T  #(num_boxes)x(num_boxes)

            def get_decay_factor(method='gauss', sigma=0.5):
                if method == 'gauss':
                    # gaussian decay
                    decay_factor = np.exp(-(iou_matrix**2 - max_iou_expand**2) / sigma)
                else:
                    # linear decay
                    decay_factor = (1 - iou_matrix) / (1 - max_iou_expand)

                # decay factor: 1xN
                decay_factor = np.min(decay_factor, axis=0)
                # clamp decay factor to <= 1
                decay_factor = np.minimum(decay_factor, 1.0)
                return decay_factor

            # decay factor for box score
            decay_factor = get_decay_factor()

            # apply decay factor to punish box score,
            # and filter box with confidence threshold
            s_matrix_decay = s_nms * decay_factor
            keep_mask = s_matrix_decay >= confidence

        elif use_spm:
            # apply SPM(Score Penalty Mechanism)
            if use_diou:
                # TODO: Cluster SPM distance NMS couldn't achieve good result, may need to double check
                # currently we fallback to normal SPM
                #
                # Reference:
                # https://github.com/Zzh-tju/CIoU/blob/master/layers/functions/detection.py
                # https://zhuanlan.zhihu.com/p/157900024

                #diou_matrix = box_diou_matrix(b_nms, b_nms)
                #flag = (updated_iou_matrix >= 0).astype(np.float32)
                #penalty_coef = np.prod(np.minimum(np.exp(-(updated_iou_matrix**2)/0.2) + diou_matrix*((updated_iou_matrix>0).astype(np.float32)), flag), axis=0)
                penalty_coef = np.prod(np.exp(-(updated_iou_matrix**2)/0.2), axis=0)
            else:
                penalty_coef = np.prod(np.exp(-(updated_iou_matrix**2)/0.2), axis=0)
            s_spm = penalty_coef * s_nms
            keep_mask = s_spm >= confidence

        else:
            # filter low score box with iou_threshold
            keep_mask = max_iou < iou_threshold

        if use_weighted:
            # generate weights matrix with box score and final IoU matrix
            weights = (updated_iou_matrix*(updated_iou_matrix>iou_threshold).astype(np.float32) + np.eye(num_boxes)) * (s_nms.reshape((1, num_boxes)))

            # convert box format to (xmin,ymin,xmax,ymax) for weighted average,
            # and expand to NxN array
            xmin_expand = np.tile(b_nms[:,0], (num_boxes, 1))  #(num_boxes)x(num_boxes)
            ymin_expand = np.tile(b_nms[:,1], (num_boxes, 1))  #(num_boxes)x(num_boxes)
            xmax_expand = np.tile(b_nms[:,0]+b_nms[:,2], (num_boxes, 1))  #(num_boxes)x(num_boxes)
            ymax_expand = np.tile(b_nms[:,1]+b_nms[:,3], (num_boxes, 1))  #(num_boxes)x(num_boxes)

            # apply weighted average to all the candidate boxes
            weightsum = weights.sum(axis=1)
            xmin_expand = np.true_divide((xmin_expand*weights).sum(axis=1), weightsum)
            ymin_expand = np.true_divide((ymin_expand*weights).sum(axis=1), weightsum)
            xmax_expand = np.true_divide((xmax_expand*weights).sum(axis=1), weightsum)
            ymax_expand = np.true_divide((ymax_expand*weights).sum(axis=1), weightsum)

            # stack the weighted average boxes and convert back to (x,y,w,h)
            b_nms = np.stack([xmin_expand, ymin_expand, xmax_expand-xmin_expand, ymax_expand-ymin_expand], axis=1)

        # keep NMSed boxes
        b_nms = b_nms[keep_mask]
        c_nms = c_nms[keep_mask]
        s_nms = s_nms[keep_mask]

        # merge NMSed boxes to final result
        if len(nboxes) == 0:
            nboxes = np.asarray(copy.deepcopy(b_nms))
            nclasses = np.asarray(copy.deepcopy(c_nms))
            nscores = np.asarray(copy.deepcopy(s_nms))
        else:
            nboxes = np.append(nboxes, copy.deepcopy(b_nms), axis=0)
            nclasses = np.append(nclasses, copy.deepcopy(c_nms), axis=0)
            nscores = np.append(nscores, copy.deepcopy(s_nms), axis=0)

    # reformat result for output
    nboxes = [np.array(nboxes)]
    nclasses = [np.array(nclasses)]
    nscores = [np.array(nscores)]
    return nboxes, nclasses, nscores

def filter_boxes(boxes, classes, scores, max_boxes):
    '''
    Sort the prediction boxes according to score
    and only pick top "max_boxes" ones
    '''
    # sort result according to scores
    sorted_indices = np.argsort(scores)
    sorted_indices = sorted_indices[::-1]
    nboxes = boxes[sorted_indices]
    nclasses = classes[sorted_indices]
    nscores = scores[sorted_indices]

    # only pick max_boxes
    nboxes = nboxes[:max_boxes]
    nclasses = nclasses[:max_boxes]
    nscores = nscores[:max_boxes]

    return nboxes, nclasses, nscores

def yolo_adjust_boxes(boxes, img_shape):
    '''
    change box format from (x,y,w,h) top left coordinate to
    (xmin,ymin,xmax,ymax) format
    '''
    if boxes is None or len(boxes) == 0:
        return []

    image_shape = np.array(img_shape, dtype='float32')
    height, width = image_shape

    adjusted_boxes = []
    for box in boxes:
        x, y, w, h = box

        xmin = x
        ymin = y
        xmax = x + w
        ymax = y + h

        ymin = max(0, np.floor(ymin + 0.5).astype('int32'))
        xmin = max(0, np.floor(xmin + 0.5).astype('int32'))
        ymax = min(height, np.floor(ymax + 0.5).astype('int32'))
        xmax = min(width, np.floor(xmax + 0.5).astype('int32'))
        adjusted_boxes.append([xmin,ymin,xmax,ymax])

    return np.array(adjusted_boxes,dtype=np.int32)

max_stars_repo_name: kimbfs/Proyecto

max_issues_repo_name: kimbfs/Proyecto

max_forks_repo_name: kimbfs/Proyecto

reason: import numpy,from scipy


text:
# Keywords: Python, nucleotide-based, nucleotide sequence, sequence-based mutation rate

import pyslim
import numpy as np

ts = pyslim.load("recipe_nucleotides.trees")
slim_gen = ts.metadata["SLiM"]["generation"]

M = np.zeros((4,4,4,4), dtype='int')
for mut in ts.mutations():
    pos = ts.site(mut.site).position
    # skip mutations at the end of the sequence
    if pos > 0 and pos < ts.sequence_length - 1:
        mut_list = mut.metadata["mutation_list"]
        k = np.argmax([u["slim_time"] for u in mut_list])
        derived_nuc = mut_list[k]["nucleotide"]
        left_nuc = ts.nucleotide_at(mut.node, pos - 1,
                time = slim_gen - mut_list[k]["slim_time"] - 1.0)
        right_nuc = ts.nucleotide_at(mut.node, pos + 1,
                time = slim_gen - mut_list[k]["slim_time"] - 1.0)
        if mut.parent == -1:
            acgt = ts.reference_sequence[int(mut.position)]
            parent_nuc = pyslim.NUCLEOTIDES.index(acgt)
        else:
            parent_mut = ts.mutation(mut.parent)
            assert(parent_mut.site == mut.site)
            parent_nuc = parent_mut.metadata["mutation_list"][0]["nucleotide"]
        M[left_nuc, parent_nuc, right_nuc, derived_nuc] += 1

print("{}\t{}\t{}".format('ancestral', 'derived', 'count'))
for j0, a0 in enumerate(pyslim.NUCLEOTIDES):
    for j1, a1 in enumerate(pyslim.NUCLEOTIDES):
        for j2, a2 in enumerate(pyslim.NUCLEOTIDES):
            for k, b in enumerate(pyslim.NUCLEOTIDES):
                print("{}{}{}\t{}{}{}\t{}".format(a0, a1, a2, a0, b, a2,
                        M[j0, j1, j2, k]))

max_stars_repo_path: tests/recipes/Recipe 18.13 - Tree-sequence recording and nucleotide-based models III.py

max_stars_repo_name: hyanwong/pyslim

max_issues_repo_name: hyanwong/pyslim

max_forks_repo_name: hyanwong/pyslim


text:
import streamlit as st
import pandas as pd
import numpy as np

from PIL import Image
#test_image = Image.open(r'C:\Users\njvdb\projects\fork\src\app\test.jpg')
#test_image = Image.open(r'https://github.com/dimitribenn/fork/blob/main/src/app/test.jpg')
#st.image(test_image, caption= "Kraftwerk im Sonnenuntergang")

st.title('Vorhersage der deutschen CO2-Emissionen')

"Autor: Dimitri Bennhäuser (https://github.com/dimitribenn)"

from scipy.stats import linregress

years = [2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020]
emissions_per_year = [10.3, 10.0, 10.1, 10.2, 9.7, 9.7, 9.7, 9.5, 9.1, 8.5, 7.7]
temp_per_year = [7.9, 9.6, 9.1, 8.7, 10.3, 9.9, 9.5, 9.6, 10.5, 10.2, 10.4]

regression_result = linregress(years, emissions_per_year)
scipy_slope = regression_result.slope
scipy_intercept = regression_result.intercept

regression_result_temp = linregress(years, temp_per_year)
scipy_slope_temp = regression_result_temp.slope
scipy_intercept_temp = regression_result_temp.intercept

def scipy_model(desired_year):
    model_result = scipy_slope * desired_year + scipy_intercept
    return model_result

def scipy_model_temp(desired_year):
    model_result_temp = scipy_slope_temp * desired_year + scipy_intercept_temp
    return model_result_temp

desired_year = st.number_input('Jahr', value=2022)

prediction = scipy_model(desired_year)
prediction_rounded = round(prediction, 2)

prediction_temp = scipy_model_temp(desired_year)
prediction_temp_rounded = round(prediction_temp, 2)

"Die Vorhersage der Emissionen für das ausgewählte Jahr ist:"

st.write(prediction_rounded)

"Tonnen pro Kopf pro Jahr gg"

"Die Vorhersage der durschnittlichen Temperatur für das ausgewählte Jahr ist:"
st.write(prediction_temp_rounded)

"Grad Celsius"

st.subheader("Über das Modell und die Daten")

"Der öffentliche Link zur App: https://share.streamlit.io/dimitribenn/fork/main/src/app/app.py"

"Das Modell ist ein lineares Regressionsmodell auf Grundlage von 2010 bis 2020. "
"Es steht ein Datenpunkt pro Jahr zur Verfügung."

"Die Daten stammen aus folgenden Quellen:"

"- Global Carbon Project. (2021). Supplemental data of Global Carbon Project 2021 (1.0) [Data set]. " \
"Global Carbon Project. https://doi.org/10.18160/gcp-2021. "
"- Andrew, Robbie M., & Peters, Glen P. (2021). The Global Carbon Project's fossil CO2 emissions dataset [Data set]. " \
"Zenodo. https://doi.org/10.5281/zenodo.5569235."
"- DWD. Entwicklung der Jahresmitteltemperatur in Deutschland in ausgewählten Jahren von 1960 bis 2021. [Data set]." \
    "https://de-statista-com.ezp.hs-duesseldorf.de/statistik/daten/studie/914891/umfrage/durchschnittstemperatur-in-deutschland/"

max_stars_repo_name: dimitribenn/fork

max_issues_repo_name: dimitribenn/fork

max_forks_repo_name: dimitribenn/fork


text:
#!/usr/bin/env python3

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import argparse

import numpy as np
import torch as th
from torch.distributions.multivariate_normal import MultivariateNormal
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from tensorboardX import SummaryWriter
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from tqdm import tqdm

import utils
from sparse_layer import (
    SparseLayer,
    SIGMA_BOOST,
    EPSILON)

MARGIN = 0.1

def identity(args):
    ns = [8, 16, 32, 64]
    a = [(1, 2), (2, 2), (2, 8), (2, 10)]
    writer = SummaryWriter(os.path.join("./logs", args.dir))

    for n, (a_local, a_global) in zip(ns, a):
        if not os.path.exists(f"./images/{args.dir}/{n}"):
            os.makedirs(f"./images/{args.dir}/{n}")

        cov = th.eye(n)
        if args.correlated:
            for i in range(n-1):
                cov[i, i+1] = 0.5
                cov[i+1, i] = 0.5
        normal = MultivariateNormal(th.zeros(n), cov)
        layer = SparseLayer(n, n, n, a_local, a_global, fix_values=True).to(device)
        optimizer = optim.Adam(layer.parameters(), args.lr)
        val = normal.sample(th.Size([args.batch_size])).to(device)

        running_loss = 0.
        for epoch in tqdm(range(args.epochs)):
            x = th.randn(args.batch_size, n, device=device)
            pred = layer(x)
            loss = F.mse_loss(pred, x)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
            if epoch % args.log_iter == args.log_iter - 1:
                with th.no_grad():
                    writer.add_scalar(f"train/{n}dims", running_loss / args.log_iter, epoch)
                    writer.add_scalar(f"val/{n}dims", F.mse_loss(layer(val), val).item(), epoch)
                    running_loss = 0.
                    means, sigmas, values = layer.hyper()
                    means, sigmas, values = means.unsqueeze(0), sigmas.unsqueeze(0), values.unsqueeze(0)

                    plt.figure(figsize=(7, 7))
                    plt.cla()
                    utils.plot(means, sigmas, values, shape=(n, n))
                    plt.xlim((-MARGIN*(n-1), (n-1) * (1.0+MARGIN)))
                    plt.ylim((-MARGIN*(n-1), (n-1) * (1.0+MARGIN)))

                    plt.savefig("./images/{}/{}/means{:06}.pdf".format(args.dir, n, epoch))
                    plt.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--lr", type=float, default=0.005)
    parser.add_argument("--epochs", type=int, default=10000)
    parser.add_argument("--batch_size", type=int, default=64)
    parser.add_argument("--log-iter", type=int, default=1000)
    parser.add_argument("--dir", default=utils.timestamp())
    parser.add_argument("--correlated", action="store_true", default=False)

    parser.add_argument("--local", type=int, default=4)
    parser.add_argument("--glob", type=int, default=4)
    parser.add_argument("--tau", type=float, default=0.1)
    parser.add_argument("--no-cuda", action="store_true", default=False)
    args = parser.parse_args()
    device = th.device("cpu" if (not th.cuda.is_available() or args.no_cuda) else "cuda")
    device = th.device("cpu")  # disables gpu support
    identity(args)

max_stars_repo_name: jiangdaniel/dl-papers

max_issues_repo_name: jiangdaniel/ml-implementations

max_forks_repo_name: jiangdaniel/ml-implementations


text:
import logging
import math
import numpy as np
import tensorflow as tf

logger = logging.getLogger(__name__)

class ActorCritic(object):
	"""
		Create the policy and value networks, which share input.

		The input is a tensor of shape (batch_size, num_rows, num_cols, 3),
		where the entries in the 3 channels are respectively set to:
		- 1 if occupied by stone of current color
		- 1 if occupied by stone of opposite color
		- 1 if unoccupied

		The policy output is a probability distribution across reasonable moves on the board,
		for the current color's best move.

		The value output is a single number representing the expected future discounted rewards.

		For now, model doesn't pass any its turns.
	"""

	def __init__(self, board_size):
		# hyperparams
		num_rows, num_cols = board_size
		gradient_clip = 50.
		learning_rate = 1e-4
		reg_const = 1e-4

		# forward pass placeholders
		input_tensor = tf.placeholder(
			tf.float32,
			shape=(None, num_rows, num_cols, 3),
			name='input',
		)
		reasonable_moves = tf.placeholder(
			tf.bool,
			shape=(None, num_rows, num_cols),
			name='reasonable_moves',
		)
		is_training = tf.placeholder(tf.bool, name='is_training')

		# L2 regularization
		regularizer = tf.contrib.layers.l2_regularizer(reg_const)

		# ops
		def create_conv_block(inputs, filters, layers, activation=True, batch_norm=True):
			curr = inputs
			for i in range(layers):
				curr = tf.layers.conv2d(
					inputs=curr,
					filters=filters,
					kernel_size=(3, 3),
					kernel_regularizer=regularizer,
					padding='same',
					use_bias=False if batch_norm else True,
					activation=None,
					name='conv{}'.format(i),
				)
				if batch_norm:
					curr = tf.layers.batch_normalization(
						inputs=curr,
						training=is_training,
						name='batch_norm{}'.format(i),
					)
				if activation:
					curr = tf.nn.relu(curr, name='relu{}'.format(i))
			return curr

		with tf.variable_scope('block1'):
			curr = create_conv_block(
				inputs=input_tensor,
				filters=32,
				layers=3,
			)

		with tf.variable_scope('block2'):
			curr = create_conv_block(
				inputs=curr,
				filters=64,
				layers=3,
			)

		with tf.variable_scope('block3'):
			shared_network_output = create_conv_block(
				inputs=curr,
				filters=128,
				layers=3,
			)

		# policy network output
		with tf.variable_scope('policy_subnetwork'):
			# no ReLU/BatchNorm before softmax
			curr = create_conv_block(
				inputs=shared_network_output,
				filters=1,
				layers=1,
				activation=False,
				batch_norm=False,
			)
			# remove last dimension of size 1
			logits = tf.reshape(
				curr,
				shape=(-1, num_rows, num_cols),
				name='squeeze',
			)

			# we have to compute our own softmax so that we can ignore illegal/bad moves
			# for stability, we must first subtract the maximum of the logits in each batch element

			# we mask the logits so that we only take max of the values of reasonable moves
			zeros = tf.zeros(tf.shape(reasonable_moves), dtype=tf.float32)
			infinity_mask = tf.where(
				reasonable_moves,
				zeros,
				math.inf + zeros, # addition is for broadcasting
			)
			logger.debug("infinity_mask shape: {}".format(infinity_mask.get_shape()))

			masked_logits = logits - infinity_mask
			logger.debug("masked_logits shape: {}".format(masked_logits.get_shape()))

			max_logits = tf.reduce_max(masked_logits, axis=(1, 2), keep_dims=True)
			logger.debug("max_logits shape: {}".format(max_logits.get_shape()))

			scaled_logits = masked_logits - max_logits
			logger.debug("scaled_logits shape: {}".format(scaled_logits))

			exp_logits = tf.exp(scaled_logits)
			logger.debug("exp_logits shape: {}".format(exp_logits.get_shape()))

			# we mask again to actually zero out the moves we don't consider
			masked_exp_logits = exp_logits * tf.to_float(reasonable_moves)
			logger.debug("masked_exp_logits shape: {}".format(masked_exp_logits.get_shape()))

			sum_exp_logits = tf.reduce_sum(masked_exp_logits, axis=(1, 2), keep_dims=True)
			logger.debug("sum_exp_logits shape: {}".format(sum_exp_logits.get_shape()))

			policy_output = masked_exp_logits / (sum_exp_logits + 1e-7) # add epsilon so don't divide by 0
			logger.debug("policy_output shape: {}".format(policy_output.get_shape()))

		# value network output
		with tf.variable_scope('value_subnetwork'):
			# no ReLU/BatchNorm before final output
			curr = create_conv_block(
				inputs=shared_network_output,
				filters=1,
				layers=1,
				activation=False,
				batch_norm=False,
			)
			curr = tf.layers.average_pooling2d(
				inputs=curr,
				strides=1,
				pool_size=curr.get_shape()[1:3],
			)
			value_output = tf.reshape(curr, shape=(-1,))

		# LOSS CALCULATION

		# pass in the chosen action for this state: tuple of (row_idx, col_idx)
		actions = tf.placeholder(tf.int32, shape=(None, 2), name='actions')
		# we pass in the advantages as well as rewards
		# because we don't want to compute gradient for the value network
		# when we use advantages in the policy loss
		advantages = tf.placeholder(tf.float32, shape=(None,), name='advantages')
		# discounted rewards
		rewards = tf.placeholder(tf.float32, shape=(None,), name='rewards')

		# loss
		with tf.variable_scope('loss'):
			# policy loss
			batch_indices = tf.range(tf.shape(actions)[0])[:, tf.newaxis]
			logger.debug("batch_indices shape: {}".format(batch_indices.get_shape()))

			action_indices = tf.concat([batch_indices, actions], axis=-1)
			logger.debug("indices shape: {}".format(action_indices.get_shape()))

			# we take the log of the softmax in a more numerically stable way
			# sum_logits is guaranteed to be at least 1
			# since we scaled our logits to have max = 0
			log_softmax = scaled_logits - tf.log(sum_exp_logits)
			logger.debug("log_softmax shape: {}".format(log_softmax.get_shape()))

			log_action_probs = tf.gather_nd(log_softmax, action_indices)
			logger.debug("log_action_probs shape: {}".format(log_action_probs.get_shape()))

			policy_loss = -tf.reduce_mean(log_action_probs * advantages)

			# value loss
			value_loss = tf.reduce_mean((rewards - value_output) ** 2)

			# reg loss
			reg_loss = tf.losses.get_regularization_loss()

			# total loss
			loss = policy_loss + value_loss + reg_loss

			# backprop
			optimizer = tf.train.AdamOptimizer(learning_rate)
			# optimizer = tf.train.MomentumOptimizer(
			# 	learning_rate=learning_rate,
			# 	momentum=0.6,
			# 	use_nesterov=True,
			# )
			grads_and_vars = optimizer.compute_gradients(loss)
			optimize = optimizer.apply_gradients(grads_and_vars)

		# keep pointers to relevant nodes
		self.input = input_tensor
		self.reasonable_moves = reasonable_moves
		self.is_training = is_training

		self.logits = logits
		self.infinity_mask = infinity_mask
		self.masked_logits = masked_logits
		self.max_logits = max_logits
		self.scaled_logits = scaled_logits
		self.exp_logits = exp_logits
		self.masked_exp_logits = masked_exp_logits
		self.sum_exp_logits = sum_exp_logits
		self.policy_output = policy_output
		self.value_output = value_output

		self.actions = actions
		self.advantages = advantages
		self.rewards = rewards

		self.action_indices = action_indices
		self.log_softmax = log_softmax
		self.log_action_probs = log_action_probs
		self.policy_loss = policy_loss
		self.value_loss = value_loss
		self.reg_loss = reg_loss
		self.loss = loss
		self.optimize = optimize

		self.init_op = tf.global_variables_initializer()

# test
if __name__ == '__main__':
	logger.setLevel(logging.DEBUG)
	model = ActorCritic((5, 10))

	with tf.Session() as session:
		session.run(model.init_op)

		actions = [(2, 8), (0, 1)]
		batch_shape = (2, 5, 10, 3)
		batch_input = np.reshape(np.arange(np.prod(batch_shape)), batch_shape)
		is_training = False
		reasonable_moves = np.ones((2, 5, 10))
		reasonable_moves[1, 3, 4] = 0

		advantages = [2, 5]
		rewards = [0.99 ** (batch_shape[0] - i) for i in range(batch_shape[0])]

		nodes = np.array([
			["infinity_mask", model.infinity_mask],
			["masked_logits", model.masked_logits],
			["max_logits", model.max_logits],
			["scaled_logits", model.scaled_logits],
			["exp_logits", model.exp_logits],
			["masked_exp_logits", model.masked_exp_logits],
			["sum_exp_logits", model.sum_exp_logits],
			["policy_output", model.policy_output],
			["value_output", model.value_output],
			["action_indices", model.action_indices],
			["log_softmax", model.log_softmax],
			["log_action_probs", model.log_action_probs],
			["policy_loss", model.policy_loss],
			["value_loss", model.value_loss],
			["reg_loss", model.reg_loss],
			["loss", model.loss],
			["optimize", model.optimize],
		])

		node_values = session.run(
			nodes[:, 1].tolist(),
			feed_dict={
				model.input: batch_input,
				model.is_training: is_training,
				model.actions: actions,
				model.reasonable_moves: reasonable_moves,
				model.advantages: advantages,
				model.rewards: rewards,
			}
		)

		for (node_name, node_value) in zip(nodes[:, 0], node_values):
			print("node: {}".format(node_name))
			print("{}\n".format(node_value))

		summary_writer = tf.summary.FileWriter('logs', graph=session.graph)

max_stars_repo_name: JamesonWeng/go-lite

max_issues_repo_name: JamesonWeng/go-lite

max_forks_repo_name: JamesonWeng/go-lite


text:
import numpy as np
import time

from integrators import BackwardsEuler, make_symmetric_random

class SimulatorEnv:
    """class for simulation environment
    Usage:
        env = SimulatorEnv(coefficient, init_state, target_state, time_limit, euler_limit, delta, eps_euler, eps_target, lambda_distance_reward)
        next_state, reward, done, info, distance_to_target = env(action)
    See `test_simulator` for concrete example.
    """

    def __init__(self, coefficient, init_state, target_state, time_limit, euler_limit, delta, eps_euler, eps_target,
                 lambda_distance_reward):
        self.num_genes = coefficient.shape[0]
        self.coefficient = coefficient  # np.array [num_genes, num_genes]
        self.target_state = target_state  # np.array [num_genes,]
        self.init_state = init_state  # np.array [num_genes,]
        self.time_limit = time_limit  # time limit for an episode, one episode corresponds to certain number of actions
        self.euler_limit = euler_limit  # the maximum delta_t can take for integration when calling backward euler
        self.delta = delta  # delta in integration method
        self.eps_euler = eps_euler  # epsilon used in backward Euler
        self.eps_target = eps_target  # epsilon used for deciding if  current state is close enough to  final state
        self.lambda_distance_reward = lambda_distance_reward  # lambda that controls the weight for the reward

        self.integrator = BackwardsEuler(ode_coefficients=self.coefficient,
                                         cutoff=self.euler_limit,
                                         delta=self.delta,
                                         epsilon=self.eps_euler)
        # initial state
        self.state = init_state
        self.accumulate_step = 0
        self.reset()

    def get_reward(self, next_state):
        """given next state, calculating the reward"""

        # distance to target state
        distance_to_target = ((self.target_state - next_state) ** 2).sum()

        # evaluate if the episode ends
        if distance_to_target < self.eps_target:
            done = True
            info = "reach the goal (error {})".format(distance_to_target)
        elif self.accumulate_step >= self.time_limit:
            done = True
            info = "reach the end of episode {}".format(self.accumulate_step)
        else:
            done = False
            info = "keep trying, step {} / {}".format(self.accumulate_step, self.time_limit)

        # calculate reward
        reward = - distance_to_target * self.lambda_distance_reward - 1
        return reward, done, info, distance_to_target

    def step(self, action):
        """take an action (perturb), output (next_state, reward, done, info)"""
        # use backward euler for integrate
        next_state = self.integrator.get_next(self.state, action)
        # calculate next state
        reward, done, info, distance_to_target = self.get_reward(next_state)
        # update the state
        self.state = next_state
        self.accumulate_step += 1
        return next_state, reward, done, info, distance_to_target

    def reset(self):
        """reset the env"""
        self.state = self.init_state
        self.accumulate_step = 0
        return self.state

def test_simulator():
    # define parameters
    num_genes = 100
    time_limit = 200
    euler_limit = 16000
    delta = 1e-2

    eps_euler = 1e-4
    eps_target = 1e-2
    lambda_distance_reward = 0.1  # reward = - distance_to_target * lambda_distance_reward - 1

    coefficient = make_symmetric_random(num_genes)
    init_state = np.random.rand(num_genes, 1)
    target_state = np.random.rand(num_genes, 1)

    # define env
    env = SimulatorEnv(coefficient, init_state, target_state, time_limit, euler_limit, delta, eps_euler, eps_target,
                       lambda_distance_reward)

    for i in range(time_limit):
        # generate random action, replace this with policy network in reinforcement learning
        action = np.random.rand(num_genes, 1)
        # put action into environment for evaluation
        start_time = time.time()
        next_state, reward, done, info, distance_to_target = env.step(
            action)  # env will update its current state after taking every step
        end_time = time.time()
        time_delta_step = end_time - start_time
        print("time: {}; reward: {}; done {}; distance_to_target {}".format(time_delta_step, reward, done,
                                                                            distance_to_target))
        print(info)

if __name__ == "__main__":
    # test_backward()
    test_simulator()