text:
# This contains "monkey patches" for the Cellpose Library and Detectron2
# to do the following:
#
# Cellpose: They hard coded inter = False into their SizeModel and so I
#           "monkey patched" it and received a very small increase in CV
#           and LB.
#
# Detectron2: Replaced "to_bitmasks" and "paste_masks_in_image" to get per
#             pixel predictions which also lead to a small increase in CV
#             and LB.
#

from detectron2.layers.mask_ops import _do_paste_mask
from cellpose import models, io, plot, utils
from types import SimpleNamespace
from tqdm import tqdm, trange
import numpy as np
import torch

# The following 2 functions allow for per pixel predictions in Detectron2
def to_bitmasks(self, boxes: torch.Tensor, height, width, threshold = 0.5):
        bitmasks = paste_masks_in_image(
            self.tensor,
            boxes,
            (height, width),
            threshold = threshold,
        )
        return SimpleNamespace(tensor = bitmasks)

@torch.jit.script_if_tracing
def paste_masks_in_image(masks, boxes, image_shape, threshold=0.5):
    """
    Copy pasted from detectron2.layers.mask_ops.paste_masks_in_image and deleted thresholding of the mask.
    Removed GPU ability for memory issues. Will slow down code unfortunately.

    """
    assert masks.shape[-1] == masks.shape[-2], "Only square mask predictions are supported"

    N = len(masks)

    if N == 0:
        return masks.new_empty((0,) + image_shape, dtype = torch.uint8)

    if not isinstance(boxes, torch.Tensor):
        boxes = boxes.tensor

    assert len(boxes) == N, boxes.shape

    img_h, img_w = image_shape
    num_chunks   = N
    device       = boxes.device

    chunks = torch.chunk(torch.arange(N, device = device), num_chunks)

    img_masks = torch.zeros(
        N, img_h, img_w, device = device, dtype = torch.float32
    )

    for inds in chunks:
        masks_chunk, spatial_inds = _do_paste_mask(
            masks[inds, None, :, :], boxes[inds], img_h, img_w, skip_empty = device.type == "cpu"
        )

        img_masks[(inds,) + spatial_inds] = masks_chunk

    return img_masks

# I switched this out because they have interp = False hard-coded.
# and interp = True gave a small increase in CV and LB.
def eval(self, x, channels=None, channel_axis=None,
         normalize=True, invert=False, augment=False, tile=True,
         batch_size=8, progress=None, interp=True, omni=False):

    assert x.squeeze().ndim < 3

    styles = self.cp.eval(x, channels=channels, channel_axis=channel_axis,
                          normalize=normalize, invert=invert, augment=augment,
                          tile=tile, batch_size=batch_size, net_avg=True,
                          compute_masks=False)[-1]

    diam_style = self._size_estimation(np.array(styles))
    diam_style = self.diam_mean if (diam_style==0 or np.isnan(diam_style)) else diam_style

    masks = self.cp.eval(x, channels=channels, channel_axis=channel_axis,
                         normalize=normalize, invert=invert, augment=augment,
                         tile=tile, batch_size=batch_size, net_avg=True,
                         rescale=(self.diam_mean / diam_style), diameter=None,
                         interp=True, omni=omni)[0]

    # allow backwards compatibility to older scale metric
    diam = utils.diameters(masks, omni = omni)[0]
    if hasattr(self, 'model_type') and (self.model_type=='nuclei' or self.model_type=='cyto') and not self.torch and not omni:
        diam_style /= (np.pi**0.5)/2
        diam = self.diam_mean / ((np.pi**0.5)/2) if (diam==0 or np.isnan(diam)) else diam
    else:
        diam = self.diam_mean if (diam==0 or np.isnan(diam)) else diam

    return diam, diam_style

max_stars_repo_name: ryanirl/data-analysis-projects

max_issues_repo_name: ryanirl/data-analysis-projects

max_forks_repo_name: ryanirl/data-analysis-projects


text:
#!/usr/bin/env python

"""
Author: Shashank Kotyan
Email:  shashankkotyan@gmail.com
"""

import os, sys, warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
warnings.filterwarnings('ignore')

import argparse, glob, pickle, time, GPUtil, numpy as np

from scipy.cluster.vq import whiten as normalise
from multiprocessing.managers import BaseManager

import build
from population import Population
from worker import Worker
from process import Process

class NeuroEvolution:

    def __init__(self, args):

        self.population_size = args.population_size
        self.population_dir  = args.population_dir
        self.num_mutations   = args.num_mutations
        self.number_of_child = args.num_mutated_child

        self.gen         = -1
        self.ind_counter = 0

        if not os.path.exists(self.population_dir): os.makedirs(self.population_dir)
        if not os.path.exists(args.log_dir):        os.makedirs(args.log_dir)

        BaseManager.register('Population', Population)
        manager = BaseManager()
        manager.start()

        self.population = manager.Population()
        self.population.set(args)

        self.worker = Worker(args, build.load_data(args.dataset))

    def parallel_run(self, type_create, args):

        start = False

        while not start:

            for deviceID in range(7):

                if deviceID in self.jobs:

                    if not self.jobs[deviceID].is_alive():

                        self.jobs[deviceID].close()
                        if self.jobs[deviceID].exception is not None: raise Exception(f"{self.jobs[deviceID].exception[0]}, {self.jobs[deviceID].exception[1]}")
                        finished_job = self.jobs.pop(deviceID, None)

            with open("num_workers.txt", "r") as f: num_workers = int(f.read())

            deviceIDs = GPUtil.getAvailable(order='memory', limit=7, maxLoad=1.1, maxMemory=0.5)

            alive = -1

            if len(deviceIDs) != 0:

                for deviceID in deviceIDs:

                    if deviceID not in self.jobs:

                        alive = deviceID
                        break

            if len(self.jobs) < num_workers and alive > -1:

                print(f"GPU {alive} running {self.ind_counter}")

                if type_create == 0: target=self.worker.create_parent
                else:                target=self.worker.create_child

                start = True

                args[0]          = alive
                job              = Process(target=target, args=tuple(args))
                self.jobs[alive] = job
                job.start()

            else:

                time.sleep(0.1)

    def run(self):

        found = False

        print(f"Searching for previous geneartions")
        population = sorted(glob.glob(f"{self.population_dir}/*/*/alive.txt"))

        if len(population) > 0:

            if len(population) == self.population_size or len(population) == self.population_size*(self.number_of_child+1):

                found = True

                for individual in population:

                    self.gen         = max(self.gen,         int(individual.split('/')[1]))
                    self.ind_counter = max(self.ind_counter, int(individual.split('/')[2]))

                if len(population) == self.population_size*(self.number_of_child+1): self.evolve_ras()

                print(f"Found Last Generation {self.gen} with last individual {self.ind_counter}")

            else: raise Exception(f"Corrupted Files, please delete the files in {self.population_dir}. Maybe the files were in between the evolution")

        else: found = False

        if found == False: self.create_initial_population()

        self.evolve()

    def create_initial_population(self):

        print(f"Did Not Found Any Last Generation\n")

        self.gen += 1

        generation_directory = f"{self.population_dir}/{self.gen}/"
        if not os.path.exists(generation_directory): os.makedirs(generation_directory)

        self.jobs = {}

        for _ in range(self.population_size):
            self.ind_counter   += 1

            individual = f"{self.population_dir}/{self.gen}/{self.ind_counter}"
            if not os.path.exists(individual): os.makedirs(individual)

            self.create_lineage(individual)
            self.parallel_run(0, [0, individual, self.population, self.store_individual, self.gen])

        for deviceID in self.jobs:
            self.jobs[deviceID].join()
            if self.jobs[deviceID].exception is not None: raise Exception(f"{self.jobs[deviceID].exception[0]}, {self.jobs[deviceID].exception[1]}")

        self.population.write_log(f"\n")
        self.population.save_populations(f"{self.population_dir}/{self.gen}")

    def evolve_ras(self):

        population = sorted(glob.glob(f"{self.population_dir}/*/*/alive.txt"))
        assert len(population) == self.population_size

        self.population.read_populations(f"{self.population_dir}/{self.gen}")

        self.gen += 1

        generation_directory = f"{self.population_dir}/{self.gen}/"
        if not os.path.exists(generation_directory): os.makedirs(generation_directory)

        cluster_population = {'individual':[], 'dna':[], 'metrics':[], 'spectrum': []}

        for i, individual in enumerate(population):

            individual = individual.split('/alive.txt')[0]
            dna, metrics = self.read_individual(individual)

            cluster_population['individual']  += [individual]
            cluster_population['dna']         += [dna]
            cluster_population['metrics']     += [metrics]
            cluster_population['spectrum']    += [dna['graph'].graph['spectrum']]

        child_individuals = []

        self.jobs = {}

        for parent_individual in population * self.number_of_child:

            self.ind_counter += 1

            parent_individual = parent_individual.split('/alive.txt')[0]
            parent_dna, parent_metrics = self.read_individual(parent_individual)

            child_individual = f"{self.population_dir}/{self.gen}/{self.ind_counter}"
            if not os.path.exists(child_individual): os.makedirs(child_individual)

            child_individuals += [child_individual]

            self.store_lineage(child_individual, parent_individual)
            self.parallel_run(1, [0, parent_dna, self.num_mutations, child_individual, self.population, self.store_individual, self.gen])

        for deviceID in self.jobs:

            self.jobs[deviceID].join()
            if self.jobs[deviceID].exception is not None: raise Exception(f"{self.jobs[deviceID].exception[0]}, {self.jobs[deviceID].exception[1]}")

        for child_individual in child_individuals:

            child_dna, child_metrics = self.read_individual(child_individual)
            child_spectrum           = child_dna['graph'].graph['spectrum']

            normalisation_spectrum = normalise(cluster_population['spectrum'] + [child_spectrum])
            distance               = [np.linalg.norm(x-normalisation_spectrum[-1]) for x in normalisation_spectrum[:-1]]
            closest_cluster_index  = distance.index(min(distance))

            if cluster_population['metrics'][closest_cluster_index]['fitness'] < child_metrics['fitness']:

                self.population.write_log(f"--> Worker changed  cluster {closest_cluster_index} head {cluster_population['individual'][closest_cluster_index].split('/')[2]} of fitness {cluster_population['metrics'][closest_cluster_index]['fitness']:.2f} to   {child_individual.split('/')[2]} of fitness {child_metrics['fitness']:.2f}\n")

                dead_individual_dir = cluster_population['individual'][closest_cluster_index]

                os.remove(f"{dead_individual_dir}/alive.txt")
                open(f"{dead_individual_dir}/dead_{self.gen}.txt", 'w').close()

                cluster_population['individual'][closest_cluster_index] = [child_individual]
                cluster_population['dna'][closest_cluster_index]        = [child_dna]
                cluster_population['metrics'][closest_cluster_index]    = [child_metrics]
                cluster_population['spectrum'] [closest_cluster_index]  = [child_spectrum]

            else:

                self.population.write_log(f"--> Worker retained cluster {closest_cluster_index} head {cluster_population['individual'][closest_cluster_index].split('/')[2]} of fitness {cluster_population['metrics'][closest_cluster_index]['fitness']:.2f} over {child_individual.split('/')[2]} of fitness {child_metrics['fitness']:.2f}\n")

                dead_individual_dir = child_individual

                os.remove(f"{dead_individual_dir}/alive.txt")
                open(f"{dead_individual_dir}/dead_{self.gen}.txt", 'w').close()

        self.population.write_log(f"\n")
        self.population.clean_populations()

        for pop in sorted(glob.glob(f"{self.population_dir}/*/*/alive.txt")):

            dna, metrics = self.read_individual(pop.split('/alive.txt')[0])
            self.population.update_populations(dna)

        self.population.save_populations(f"{self.population_dir}/{self.gen}")

    def evolve(self):
        while True:
            self.evolve_ras()

    def read_individual(self, individual):
        with open(f"{individual}/dna.pkl", 'rb')      as dna_file:      dna      = pickle.load(dna_file)
        with open(f"{individual}/metrics.pkl", 'rb')  as metrics_file:  metrics  = pickle.load(metrics_file)
        return dna, metrics

    def store_individual(self, individual, dna, metrics, mutations=None):
        with open(f"{individual}/dna.pkl", 'wb')      as dna_file:      pickle.dump(dna, dna_file, pickle.HIGHEST_PROTOCOL)
        with open(f"{individual}/metrics.pkl", 'wb')  as metrics_file:  pickle.dump(metrics, metrics_file, pickle.HIGHEST_PROTOCOL)
        if mutations is not None:
            with open(f"{individual}/mutations.pkl", 'wb') as mutations_file: pickle.dump(mutations, mutations_file, pickle.HIGHEST_PROTOCOL)

    def store_lineage(self, child_individual, parent_individual):
        with open(f"{parent_individual}/lineage.pkl", 'rb') as lineage_file: lineage = pickle.load(lineage_file)
        lineage += [parent_individual.split('/')[2]]
        with open(f"{child_individual}/lineage.pkl", 'wb')  as lineage_file:  pickle.dump(lineage, lineage_file, pickle.HIGHEST_PROTOCOL)

    def create_lineage(self, individual):
        with open(f"{individual}/lineage.pkl", 'wb')  as lineage_file:  pickle.dump([individual.split('/')[2]], lineage_file, pickle.HIGHEST_PROTOCOL)

def to_bool(arg_bool):

    if   arg_bool in ('True',  'true',  'T', 't', '1', 'Y', 'y'): return True
    elif arg_bool in ('False', 'false', 'F', 'f', '0', 'N', 'n'): return False
    else: raise argparse.ArgumentTypeError('Boolean value expected.')

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description=('Robust Architecture Search'))

    # Use for creating and mutating individual of population
    parser.add_argument("--use_cycles",             "-cy",   action="store_false",            help="if not to use cycles in populations")
    parser.add_argument("--use_adaptive_mutations", "-am",   action="store_true",             help="if use adaptive mutations")
    parser.add_argument("--use_random_params",      "-rp",   action="store_true",             help="if use random params")
    parser.add_argument("--use_non_squares",        "-sq",   action="store_true",             help="if use non-square kernels and strides")

    # Use for training an individual
    parser.add_argument("--use_augmentation",       "-au",   action="store_true",             help="if use augmented training")
    parser.add_argument("--use_limited_data",       "-ld",   action="store_true",             help="if use limited data for training")
    parser.add_argument("--dataset",                "-d",    default=2,            type=int,  help="Dataset to be used for training")
    parser.add_argument("--epochs",                 "-e",    default=50,           type=int,  help="Number of epochs to be used for a single individual")

    # Use for type of evolving generations
    parser.add_argument("--num_mutations",          "-m",    default=5,            type=int,  help="Number of mutations an individual undergoes")
    parser.add_argument("--num_mutated_child",      "-n",    default=2,            type=int,  help="Number of mutated individuals for a single parent")
    parser.add_argument("--population_size",        "-p",    default=25,           type=int,  help="Number of individuals in the population")

    # Use for changing directories
    parser.add_argument("--population_dir",         "-dir",  default="population", type=str,  help="Directory for storing all individuals")
    parser.add_argument("--log_dir",                "-log",  default="logs",       type=str,  help="Directory for logs")

    # Use for dry run
    parser.add_argument("--test",                   "-test", action="store_false",            help="Option for Dry Run to test, if the code runs. Please also check ./-log_dir-/exceptions.log too.")

    args = parser.parse_args()

    print(args)

    NeuroEvolution(args).run()

ext: py

max_stars_repo_name: shashankkotyan/RobustArchitectureSearch

max_issues_repo_name: shashankkotyan/RobustArchitectureSearch

max_forks_repo_name: shashankkotyan/RobustArchitectureSearch

reason: from scipy


text:
import argparse
import numpy as np

import chainer
from chainercv.utils import read_image
from chainercv.visualizations import vis_image

from chainercv.utils import apply_to_iterator
from chainercv.utils import ProgressHook

from lib.eval_projected_3d_bbox import eval_projected_3d_bbox_single
from lib.linemod_dataset import LinemodDataset
from lib.linemod_dataset import linemod_object_diameters
from lib.mesh import MeshPly
from lib.ssp import SSPYOLOv2
from lib.utils import get_linemod_intrinsics

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--gpu', type=int, default=-1)
    parser.add_argument('--pretrained-model', type=str)
    parser.add_argument('object')
    args = parser.parse_args()

    model = SSPYOLOv2()
    chainer.serializers.load_npz(args.pretrained_model, model)
    if args.gpu >= 0:
        chainer.cuda.get_device_from_id(args.gpu).use()
        model.to_gpu()

    test = LinemodDataset('.', obj_name=args.object, split='test')
    it = chainer.iterators.SerialIterator(
        test, batch_size=1, repeat=False, shuffle=False)

    in_values, out_values, rest_values = apply_to_iterator(
        model.predict, it, hook=ProgressHook(len(test)))
    del in_values
    points, labels, scores = out_values
    gt_points, gt_labels = rest_values

    intrinsics = get_linemod_intrinsics()
    mesh = MeshPly('LINEMOD/{}/{}.ply'.format(args.object, args.object))
    vertex      = np.c_[
        np.array(mesh.vertices),
        np.ones((len(mesh.vertices), 1))]
    result = eval_projected_3d_bbox_single(
        points, scores,
        gt_points, vertex, intrinsics,
        diam=linemod_object_diameters[args.object])
    print('Acc using 5 px 2D projection = {0:.2f}'.format(result['proj_acc']))
    print('Acc using 10% threshold: 3D transformation = {0:.2f}'.format(
        result['point3d_acc']))
    print('Acc using 5 cm 5 degree metric = {0:.2f}'.format(
        result['trans_rot_acc']))
    print('Mean projected 2D pixel error = {0:.4f}'.format(
        result['mean_proj_error']))
    print('Mean 2D pixel error = {0:.4f}'.format(
        result['mean_point2d_error']))

max_stars_repo_name: take-cheeze/models

max_issues_repo_name: take-cheeze/models

max_forks_repo_name: take-cheeze/models

reason: import numpy


text:
import torch
import numpy as np
from copy import deepcopy
from elegantrl.agents.AgentBase import AgentBase
from elegantrl.agents.net import ActorBiConv, CriticBiConv, ShareBiConv

class AgentStep1AC(AgentBase):
    def __init__(self):
        AgentBase.__init__(self)
        self.ClassAct = ActorBiConv
        self.ClassCri = CriticBiConv
        self.if_use_cri_target = False
        self.if_use_act_target = False
        self.explore_noise = 2 ** -8
        self.obj_critic = (-np.log(0.5)) ** 0.5  # for reliable_lambda

    def init(self, net_dim=256, state_dim=8, action_dim=2, reward_scale=1.0, gamma=0.99,
             learning_rate=1e-4, if_per_or_gae=False, env_num=1, gpu_id=0):
        AgentBase.init(self, net_dim=net_dim, state_dim=state_dim, action_dim=action_dim,
                       reward_scale=reward_scale, gamma=gamma,
                       learning_rate=learning_rate, if_per_or_gae=if_per_or_gae,
                       env_num=env_num, gpu_id=gpu_id, )
        if if_per_or_gae:  # if_use_per
            self.criterion = torch.nn.MSELoss(reduction='none')
            self.get_obj_critic = self.get_obj_critic_per
        else:
            self.criterion = torch.nn.MSELoss(reduction='mean')
            self.get_obj_critic = self.get_obj_critic_raw
        self.get_obj_critic = self.get_obj_critic_raw

    def select_actions(self, state: torch.Tensor) -> torch.Tensor:
        action = self.act.get_action(state.to(self.device), self.explore_noise)
        return action.detach().cpu()

    def update_net(self, buffer, batch_size, repeat_times, soft_update_tau) -> (float, float):
        buffer.update_now_len()

        obj_actor = None
        update_a = 0
        for update_c in range(1, int(buffer.now_len / batch_size * repeat_times)):
            '''objective of critic (loss function of critic)'''
            obj_critic, state = self.get_obj_critic(buffer, batch_size)
            self.obj_critic = 0.99 * self.obj_critic + 0.01 * obj_critic.item()  # for reliable_lambda
            self.optim_update(self.cri_optim, obj_critic)
            if self.if_use_cri_target:
                self.soft_update(self.cri_target, self.cri, soft_update_tau)

            '''objective of actor using reliable_lambda and TTUR (Two Time-scales Update Rule)'''
            reliable_lambda = np.exp(-self.obj_critic ** 2)  # for reliable_lambda
            if_update_a = update_a / update_c < 1 / (2 - reliable_lambda)
            if if_update_a:  # auto TTUR
                update_a += 1

            obj_actor = -self.cri(state, self.act(state)).mean()  # policy gradient
            self.optim_update(self.act_optim, obj_actor)
            if self.if_use_act_target:
                self.soft_update(self.act_target, self.act, soft_update_tau)

        return self.obj_critic, obj_actor.item()

    def get_obj_critic_raw(self, buffer, batch_size):
        with torch.no_grad():
            # reward, mask, action, state, next_s = buffer.sample_batch(batch_size)
            q_label, action, state = buffer.sample_batch_one_step(batch_size)

        q_value = self.cri(state, action)
        obj_critic = self.criterion(q_value, q_label)
        return obj_critic, state

    def get_obj_critic_per(self, buffer, batch_size):
        with torch.no_grad():
            # reward, mask, action, state, next_s, is_weights = buffer.sample_batch(batch_size)
            q_label, action, state, is_weights = buffer.sample_batch_one_step(batch_size)

        q_value = self.cri(state, action)
        td_error = self.criterion(q_value, q_label)  # or td_error = (q_value - q_label).abs()
        obj_critic = (td_error * is_weights).mean()

        buffer.td_error_update(td_error.detach())
        return obj_critic, q_value

class AgentShareStep1AC(AgentBase):
    def __init__(self):
        AgentBase.__init__(self)
        self.ClassAct = ShareBiConv
        self.ClassCri = self.ClassAct
        self.if_use_cri_target = True
        self.if_use_act_target = True
        self.obj_critic = (-np.log(0.5)) ** 0.5  # for reliable_lambda

    def init(self, net_dim=256, state_dim=8, action_dim=2, reward_scale=1.0, gamma=0.99,
             learning_rate=1e-4, if_per_or_gae=False, env_num=1, gpu_id=0):
        AgentBase.init(self, net_dim=net_dim, state_dim=state_dim, action_dim=action_dim,
                       reward_scale=reward_scale, gamma=gamma,
                       learning_rate=learning_rate, if_per_or_gae=if_per_or_gae,
                       env_num=env_num, gpu_id=gpu_id, )
        self.act = self.cri = self.ClassAct(net_dim, state_dim, action_dim).to(self.device)
        if self.if_use_act_target:
            self.act_target = self.cri_target = deepcopy(self.act)
        else:
            self.act_target = self.cri_target = self.act

        self.cri_optim = torch.optim.Adam(
            [{'params': self.act.enc_s.parameters(), 'lr': learning_rate * 1.25},
             {'params': self.act.enc_a.parameters(), },
             {'params': self.act.mid_n.parameters(), 'lr': learning_rate * 1.25},
             {'params': self.act.dec_a.parameters(), },
             {'params': self.act.dec_q.parameters(), },
             ], lr=learning_rate)
        self.act_optim = self.cri_optim

        if if_per_or_gae:  # if_use_per
            self.criterion = torch.nn.MSELoss(reduction='none')
            self.get_obj_critic = self.get_obj_critic_per
        else:
            self.criterion = torch.nn.MSELoss(reduction='mean')
            self.get_obj_critic = self.get_obj_critic_raw

    def select_actions(self, state: torch.Tensor) -> torch.Tensor:
        action = self.act.get_action(state.to(self.device), self.explore_noise)
        return action.detach().cpu()

    def update_net(self, buffer, batch_size, repeat_times, soft_update_tau) -> (float, float):
        buffer.update_now_len()

        obj_critic = None
        obj_actor = None
        update_a = 0
        for update_c in range(1, int(buffer.now_len / batch_size * repeat_times)):
            '''objective of critic'''
            obj_critic, state = self.get_obj_critic(buffer, batch_size)
            self.obj_critic = 0.995 * self.obj_critic + 0.005 * obj_critic.item()  # for reliable_lambda
            reliable_lambda = np.exp(-self.obj_critic ** 2)  # for reliable_lambda

            '''objective of actor using reliable_lambda and TTUR (Two Time-scales Update Rule)'''
            if_update_a = update_a / update_c < 1 / (2 - reliable_lambda)
            if if_update_a:  # auto TTUR
                update_a += 1

                action_pg = self.act(state)  # policy gradient
                obj_actor = -self.act_target.critic(state, action_pg).mean()

                obj_united = obj_critic + obj_actor * reliable_lambda
            else:
                obj_united = obj_critic

            self.optim_update(self.cri_optim, obj_united)
            if self.if_use_act_target:
                self.soft_update(self.act_target, self.act, soft_update_tau)

        return obj_critic.item(), obj_actor.item()

    def get_obj_critic_raw(self, buffer, batch_size):
        with torch.no_grad():
            # reward, mask, action, state, next_s = buffer.sample_batch(batch_size)
            q_label, action, state = buffer.sample_batch_one_step(batch_size)

        q_value = self.act.critic(state, action)
        obj_critic = self.criterion(q_value, q_label)
        return obj_critic, state

    def get_obj_critic_per(self, buffer, batch_size):
        with torch.no_grad():
            # reward, mask, action, state, next_s, is_weights = buffer.sample_batch(batch_size)
            q_label, action, state, is_weights = buffer.sample_batch_one_step(batch_size)

        q_value = self.act.critic(state, action)
        td_error = self.criterion(q_value, q_label)  # or td_error = (q_value - q_label).abs()
        obj_critic = (td_error * is_weights).mean()

        buffer.td_error_update(td_error.detach())
        return obj_critic, q_value


text:
"""
This script evaluates the size of the LWW linkable ring signatures
implemented in koppercoin.crypto.lww_signature
"""
from koppercoin.crypto.lww_signature import *
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('ggplot')
import jsonpickle
from pympler import asizeof
import gc

# The ringsizes which will be tested
ringsizes = range(1, 21)

memsizes = pd.DataFrame(np.zeros(len(ringsizes)), index=ringsizes)

for ringsize in ringsizes:
    print("Running LWW-Signature on Ringsize " + str(ringsize) + " from " + str(list(ringsizes)))

    # generate key of other participants in the anonymity set
    public_keys = [keygen()[1] for j in range(ringsize)]
    m = "some message"
    (sec, pub) = keygen()
    public_keys.append(pub)

    # generate the signature
    sig = ringsign(public_keys, sec, m)

    # Save its size
    memsizes[0][ringsize] = asizeof.asizeof(sig)

    print("Size of memsizes object is " + str(asizeof.asizeof(memsizes)) + " Bytes")
    print("Number of gc-tracked objects: " + str(len(gc.get_objects())))

# Save the data in complicated JSON-format
with open('memsizes_LWWsig.json', 'w') as f:
    json_obj = jsonpickle.encode(memsizes)
    f.write(json_obj)

print("Running postprocessing steps")

# Save the data in handy .csv
memsizes.to_csv('memsizes_LWWsig.csv')

# Set up the plot
plt.figure()
plt.rc('text', usetex=True)
plt.rc('font', family='serif')

# plot it
memsizes.plot(style='bo', legend=None)

plt.xlabel('Size of the Ring')
plt.ylabel('Memory Size in Bytes')
plt.title('Memory Measurements for LWW-signatures')
#plt.legend(loc='upper left')
plt.savefig('memsizes_LWWsig.png')

max_stars_repo_name: vs-uulm/PriCloud

max_issues_repo_name: vs-uulm/PriCloud

max_forks_repo_name: vs-uulm/PriCloud


text:
import os
import cv2
import torch
import tempfile
import numpy as np
import face_alignment

def transformation_from_points(points1, points2):
    points1 = points1.astype(np.float64)
    points2 = points2.astype(np.float64)

    c1 = np.mean(points1, axis=0)
    c2 = np.mean(points2, axis=0)
    points1 -= c1
    points2 -= c2
    s1 = np.std(points1)
    s2 = np.std(points2)
    points1 /= s1
    points2 /= s2

    U, S, Vt = np.linalg.svd(points1.T * points2)
    R = (U * Vt).T

    return np.vstack([np.hstack(((s2 / s1) * R,
                        c2.T - (s2 / s1) * R * c1.T)),
                        np.matrix([0., 0., 1.])])

def get_position(size, padding=0.25):
    x = [0.000213256, 0.0752622, 0.18113, 0.29077, 0.393397, 0.586856, 0.689483, 0.799124,
                    0.904991, 0.98004, 0.490127, 0.490127, 0.490127, 0.490127, 0.36688, 0.426036,
                    0.490127, 0.554217, 0.613373, 0.121737, 0.187122, 0.265825, 0.334606, 0.260918,
                    0.182743, 0.645647, 0.714428, 0.793132, 0.858516, 0.79751, 0.719335, 0.254149,
                    0.340985, 0.428858, 0.490127, 0.551395, 0.639268, 0.726104, 0.642159, 0.556721,
                    0.490127, 0.423532, 0.338094, 0.290379, 0.428096, 0.490127, 0.552157, 0.689874,
                    0.553364, 0.490127, 0.42689]

    y = [0.106454, 0.038915, 0.0187482, 0.0344891, 0.0773906, 0.0773906, 0.0344891,
                    0.0187482, 0.038915, 0.106454, 0.203352, 0.307009, 0.409805, 0.515625, 0.587326,
                    0.609345, 0.628106, 0.609345, 0.587326, 0.216423, 0.178758, 0.179852, 0.231733,
                    0.245099, 0.244077, 0.231733, 0.179852, 0.178758, 0.216423, 0.244077, 0.245099,
                    0.780233, 0.745405, 0.727388, 0.742578, 0.727388, 0.745405, 0.780233, 0.864805,
                    0.902192, 0.909281, 0.902192, 0.864805, 0.784792, 0.778746, 0.785343, 0.778746,
                    0.784792, 0.824182, 0.831803, 0.824182]

    x, y = np.array(x), np.array(y)

    x = (x + padding) / (2 * padding + 1)
    y = (y + padding) / (2 * padding + 1)
    x = x * size
    y = y * size

    return np.array(list(zip(x, y)))

def load_video(file):
    p = tempfile.mkdtemp()
    cmd = 'ffmpeg -i "{}" -qscale:v 2 -r 29.97 "{}/%d.jpg"'.format(file, p)
    os.system(cmd)

    files = os.listdir(p)
    files = sorted(files, key=lambda x: int(os.path.splitext(x)[0]))

    array = [cv2.imread(os.path.join(p, file)) for file in files]
    array = list(filter(lambda im: not im is None, array))

    fa = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=False, device='cuda')
    points = [fa.get_landmarks(I) for I in array]
    print(points)

    front256 = get_position(256)
    video = []

    for point, scene in zip(points, array):
        if(point is not None):
            shape = np.array(point[0])
            shape = shape[17:]
            M = transformation_from_points(np.matrix(shape), np.matrix(front256))

            img = cv2.warpAffine(scene, M[:2], (256, 256))
            (x, y) = front256[-20:].mean(0).astype(np.int32)
            w = 160//2
            img = img[y-w//2:y+w//2,x-w:x+w,...]
            img = cv2.resize(img, (128, 64))
            video.append(img)

    video = np.stack(video, axis=0).astype(np.float32)
    video = torch.FloatTensor(video.transpose(3, 0, 1, 2)) / 255.0

    return video, p


text:
"""
This tutorial shows you how to save the world and load it again with all
its wrappers as well.
"""
from causal_world.utils.config_utils import load_world
from causal_world.task_generators.task import generate_task
from causal_world.envs.causalworld import CausalWorld
from causal_world.intervention_actors import VisualInterventionActorPolicy
from causal_world.curriculum import Curriculum
from causal_world.wrappers.curriculum_wrappers import CurriculumWrapper
from causal_world.wrappers import DeltaActionEnvWrapper
import numpy as np

def example():
    #initialize env
    task_gen = generate_task(task_generator_id='pushing')
    env = CausalWorld(task_gen, skip_frame=1, enable_visualization=True)
    env = DeltaActionEnvWrapper(env)
    env = CurriculumWrapper(
        env,
        intervention_actors=[VisualInterventionActorPolicy()],
        actives=[(0, 20, 1, 0)])

    for reset_idx in range(10):
        obs = env.reset()
        for time in range(15):
            obs, reward, done, info = env.step(action=np.zeros(9,))
    env.save_world('./')
    env.close()
    #now load it again

    env = load_world(tracker_relative_path='./', enable_visualization=True)
    for reset_idx in range(10):
        obs = env.reset()
        for time in range(15):
            obs, reward, done, info = env.step(action=np.zeros(9,))

if __name__ == '__main__':
    example()

max_stars_repo_name: michaelfeil/CausalWorld

max_issues_repo_name: michaelfeil/CausalWorld

max_forks_repo_name: michaelfeil/CausalWorld


text:
##############################################################################
##                     Written by Damilola Payne                            ##
##    Creating a recurrent neural network (RNN) in python with only Numpy   ##
##             Three layer RNN to handle sequential data                    ##
##############################################################################

# Firstly how do Recurrent Neural Nets (RNN) work?
# To understand that you need to understand how information is structured.
# When we you sing a song you not only remember the words but the relationship
# each word has to the previous word.
# If someone was to ask you to sing the song backwards although you remember
# all the words the sequence in which they occur has changed so you cannot sing the song.
# We call this type of information sequential data.
# Regular neural networks are feed-forward meaning that they expect fiXed
# sized inputs and outputs, where the inputs have no relation to the outputs.
# The lack of a relationship means that information only moves in one direction (feed-forward)
# In order to model this type of information we need a neural network which
# can understand sequences, this is why we use RNN's.
# RNN's at every hidden layer not only incorporates the information from the previous
# layer but also the information from the previous time-step recursively.

# So lets write it!
# The only two things we need is 'Numpy' to do our maths and 'Copy' to copy data
# This RNN is going to predict binary sums

import copy
import numpy as np
np.random.seed(0)

# First we need a sigmoid function

def sigmoid(x):
    '''converts values into a probability'''
    output = 1 / (1 + np.exp(-x))
    return output

def tanh(x):
    '''adding tanh activation funtion for comparison'''
     output = np.sinh(x)/np.cosh(x)
     return output

# Now we need to get the derivative of the sigmoid

def sigmoid_output_to_derivative(output):
    '''calculates the gradient of our sigmoid which is used to find our error'''
    '''via a method known as gradient descent'''
    return output * (1 - output)

def tanh_output_to_derivative(output):
    '''calculates the gradient of our tanh which is used to find our error'''
    return output * (1 - output)

# training dataset generation
interger_to_binary = {}  # This is a lookup table that changes integers to binaries
binary_dimensions = 8

largest_number = pow(2, binary_dimensions)
binary = np.unpackbits(
    np.array([range(largest_number)], dtype=np.uint8).T, axis=1)
for i in range(largest_number):
    interger_to_binary[i] = binary[i]

#define input variables
alpha = 0.1
input_dimensions = 2
hidden_dimensions = 16
output_dimensions = 1

# Initialise RNN weights to adjust values
# Synapse 0 connects of the input layer to the hidden layer
# so it has two rows and 16 columns.
synapse_0 = 2 * np.random.random((input_dimensions, hidden_dimensions)) - 1

# Synapse 1 connects the hidden layer to the output layer
# so it has 16 rows and one column.
synapse_1 = 2 * np.random.random((hidden_dimensions, output_dimensions)) - 1

# Synapse h is where the magic of the RNN happens
# it connects the hidden layer in the previous time-step and next time-step
# to the hidden layer in the current time-step
# So it has 16 rows and 16 columns
synapse_h = 2 * np.random.random((hidden_dimensions, hidden_dimensions)) - 1

synapse_0_update = np.zeros_like(synapse_0)
synapse_1_update = np.zeros_like(synapse_1)
synapse_h_update = np.zeros_like(synapse_h)

# Training logic
for j in range(10000):

    # generate a simple addition problem (a + b = c)
    a_int = np.random.randint(largest_number / 2)  # int version
    a = interger_to_binary[a_int]  # binary encoding

    b_int = np.random.randint(largest_number / 2)  # int version
    b = interger_to_binary[b_int]  # binary encoding

    # True answer
    c_int = a_int + b_int
    c = interger_to_binary[c_int]

    # Where we will store the binary encoded guess
    d = np.zeros_like(c)

    overallError = 0

    layer_2_deltas = list()
    layer_1_values = list()
    layer_1_values.append(np.zeros(hidden_dimensions))

    # Moving along the positions in the binary encoding
    for position in range(binary_dimensions):

        # Generate input and output
        X = np.array([[a[binary_dimensions - position - 1], b[binary_dimensions - position - 1]]])
        y = np.array([[c[binary_dimensions - position - 1]]]).T

        # Hidden layer 1
        # is passed through the sigmoid functions as a addition of the layers
        layer_1 = sigmoid(np.dot(X, synapse_0) + np.dot(layer_1_values[-1], synapse_h))

        # Output layer
        # Passes the output from our layers through the sigmoid function
        # to give a prediction
        layer_2 = sigmoid(np.dot(layer_1, synapse_1))

        # Calculate the amount you are away from the true value (the error)
        layer_2_error = y - layer_2
        layer_2_deltas.append((layer_2_error) * sigmoid_output_to_derivative(layer_2))
        overallError += np.abs(layer_2_error[0])  # abs is the absolute value (square root of sum)

        # Decode estimate so we can print it out
        d[binary_dimensions - position - 1] = np.round(layer_2[0][0])

        # Store hidden layer so it can be used it in the next time-step
        layer_1_values.append(copy.deepcopy(layer_1))

    future_layer_1_delta = np.zeros(hidden_dimensions)

    # This is the back propagation (see BPN from scratch for more info)
    for position in range(binary_dimensions):

        X = np.array([[a[position], b[position]]])
        layer_1 = layer_1_values[-position - 1]
        prev_layer_1 = layer_1_values[-position - 2]

        # Error at output layer
        layer_2_delta = layer_2_deltas[-position - 1]
        # error at hidden layer
        layer_1_delta = (future_layer_1_delta.dot(synapse_h.T) + layer_2_delta.dot(synapse_1.T)) * sigmoid_output_to_derivative(layer_1)

        # This information is used to update the weight functions
        synapse_1_update += np.atleast_2d(layer_1).T.dot(layer_2_delta)
        synapse_h_update += np.atleast_2d(prev_layer_1).T.dot(layer_1_delta)
        synapse_0_update += X.T.dot(layer_1_delta)

        future_layer_1_delta = layer_1_delta

    synapse_0 += synapse_0_update * alpha
    synapse_1 += synapse_1_update * alpha
    synapse_h += synapse_h_update * alpha

    synapse_0_update *= 0
    synapse_1_update *= 0
    synapse_h_update *= 0

    # Print out of progress
    if(j % 1000 == 0):
        print("Model Error:" + str(overallError))
        print("Prediction Value:" + str(d))
        print("True Value:" + str(c))
        out = 0
        for index, x in enumerate(reversed(d)):
            out += x * pow(2, index)
        print(str(a_int) + " + " + str(b_int) + " = " + str(out))
        print("------------")


text:
# Convenience module for loading in RCP emissions datasets
#
# Usage:
#
# import rcp85
# rcp85.Emissions.co2

import numpy as np
import os
emissions_filename = os.path.join(
    os.path.dirname(__file__), 'RCP85_EMISSIONS.csv')
forcing_filename = os.path.join(
    os.path.dirname(__file__), 'RCP85_MIDYEAR_RADFORCING.csv')
aviNOx_filename = os.path.join(
    os.path.dirname(__file__), 'aviNOx_fraction.csv')
fossilCH4_filename = os.path.join(
    os.path.dirname(__file__), 'fossilCH4_fraction.csv')

aviNOx_frac = np.loadtxt(aviNOx_filename, skiprows=5, usecols=4, delimiter=',')
fossilCH4_frac = np.loadtxt(fossilCH4_filename, skiprows=5, usecols=4,
    delimiter=',')

class Emissions:
    emissions = np.loadtxt(emissions_filename, skiprows=37, delimiter=',')
    year      = emissions[:,0]
    co2_fossil= emissions[:,1]
    co2_land  = emissions[:,2]
    co2       = np.sum(emissions[:,1:3],axis=1)
    ch4       = emissions[:,3]
    n2o       = emissions[:,4]
    sox       = emissions[:,5]
    co        = emissions[:,6]
    nmvoc     = emissions[:,7]
    nox       = emissions[:,8]
    bc        = emissions[:,9]
    oc        = emissions[:,10]
    nh3       = emissions[:,11]
    cf4       = emissions[:,12]
    c2f6      = emissions[:,13]
    c6f14     = emissions[:,14]
    hfc23     = emissions[:,15]
    hfc32     = emissions[:,16]
    hfc43_10  = emissions[:,17]
    hfc125    = emissions[:,18]
    hfc134a   = emissions[:,19]
    hfc143a   = emissions[:,20]
    hfc227ea  = emissions[:,21]
    hfc245fa  = emissions[:,22]
    sf6       = emissions[:,23]
    cfc11     = emissions[:,24]
    cfc12     = emissions[:,25]
    cfc113    = emissions[:,26]
    cfc114    = emissions[:,27]
    cfc115    = emissions[:,28]
    carb_tet  = emissions[:,29]
    mcf       = emissions[:,30]
    hcfc22    = emissions[:,31]
    hcfc141b  = emissions[:,32]
    hcfc142b  = emissions[:,33]
    halon1211 = emissions[:,34]
    halon1202 = emissions[:,35]
    halon1301 = emissions[:,36]
    halon2402 = emissions[:,37]
    ch3br     = emissions[:,38]
    ch3cl     = emissions[:,39]

class Forcing:
    forcing   = np.loadtxt(forcing_filename, skiprows=59, delimiter=',')
    year      = forcing[:,0]
    total     = forcing[:,1]
    volcanic  = forcing[:,2]
    solar     = forcing[:,3]
    ghg       = forcing[:,5]
    co2       = forcing[:,8]
    ch4       = forcing[:,9]
    n2o       = forcing[:,10]
    fgas      = forcing[:,11]
    halo      = forcing[:,12]
    aero      = forcing[:,41]
    cloud     = forcing[:,48]
    strato3   = forcing[:,49]
    tropo3    = forcing[:,50]
    stwv      = forcing[:,51]
    dust      = forcing[:,47]
    landuse   = forcing[:,52]
    bcsnow    = forcing[:,53]

max_stars_repo_name: OMS-NetZero/FAIR-pro

max_issues_repo_name: OMS-NetZero/FAIR-pro

max_forks_repo_name: OMS-NetZero/FAIR-pro


text:
"""
Copyright 2013 Steven Diamond, Eric Chu

This file is part of CVXPY.

CVXPY is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

CVXPY is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with CVXPY.  If not, see <http://www.gnu.org/licenses/>.
"""

from cvxpy import *
import cvxpy.atoms.elementwise.log as cvxlog
from cvxpy.tests.base_test import BaseTest
import math
import numpy as np
import sys
if sys.version_info >= (3, 0):
    from functools import reduce

class TestSCS(BaseTest):
    """ Unit tests for SCS. """
    def setUp(self):
        self.x = Variable(2, name='x')
        self.y = Variable(2, name='y')

        self.A = Variable(2, 2, name='A')
        self.B = Variable(2, 2, name='B')
        self.C = Variable(3, 2, name='C')

    # Overriden method to assume lower accuracy.
    def assertItemsAlmostEqual(self, a, b, places=2):
        super(TestSCS, self).assertItemsAlmostEqual(a, b, places=places)

    # Overriden method to assume lower accuracy.
    def assertAlmostEqual(self, a, b, places=2):
        super(TestSCS, self).assertAlmostEqual(a, b, places=places)

    def test_log_problem(self):
        # Log in objective.
        obj = Maximize(sum_entries(log(self.x)))
        constr = [self.x <= [1, math.e]]
        p = Problem(obj, constr)
        result = p.solve(solver=SCS)
        self.assertAlmostEqual(result, 1)
        self.assertItemsAlmostEqual(self.x.value, [1, math.e])

        # Log in constraint.
        obj = Minimize(sum_entries(self.x))
        constr = [log(self.x) >= 0, self.x <= [1, 1]]
        p = Problem(obj, constr)
        result = p.solve(solver=SCS)
        self.assertAlmostEqual(result, 2)
        self.assertItemsAlmostEqual(self.x.value, [1, 1])

        # Index into log.
        obj = Maximize(log(self.x)[1])
        constr = [self.x <= [1, math.e]]
        p = Problem(obj, constr)
        result = p.solve(solver=SCS)
        self.assertAlmostEqual(result, 1)

    def test_sigma_max(self):
        """Test sigma_max.
        """
        const = Constant([[1, 2, 3], [4, 5, 6]])
        constr = [self.C == const]
        prob = Problem(Minimize(norm(self.C, 2)), constr)
        result = prob.solve(solver=SCS)
        self.assertAlmostEqual(result, norm(const, 2).value)
        self.assertItemsAlmostEqual(self.C.value, const.value)

    def test_sdp_var(self):
        """Test sdp var.
        """
        const = Constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
        X = Semidef(3)
        prob = Problem(Minimize(0), [X == const])
        prob.solve(verbose=True, solver=SCS)
        self.assertEqual(prob.status, INFEASIBLE)

    def test_entr(self):
        """Test the entr atom.
        """
        self.assertEqual(entr(0).value, 0)
        assert np.isneginf(entr(-1).value)

    def test_kl_div(self):
        """Test a problem with kl_div.
        """
        import numpy as np
        import cvxpy as cp

        kK = 50
        kSeed = 10

        prng = np.random.RandomState(kSeed)
        #Generate a random reference distribution
        npSPriors = prng.uniform(0.0, 1.0, (kK, 1))
        npSPriors = npSPriors/sum(npSPriors)

        #Reference distribution
        p_refProb = cp.Parameter(kK, 1, sign='positive')
        #Distribution to be estimated
        v_prob = cp.Variable(kK, 1)
        objkl = 0.0
        for k in range(kK):
            objkl += cp.kl_div(v_prob[k, 0], p_refProb[k, 0])

        constrs = [sum([v_prob[k, 0] for k in range(kK)]) == 1]
        klprob = cp.Problem(cp.Minimize(objkl), constrs)
        p_refProb.value = npSPriors
        result = klprob.solve(solver=SCS, verbose=True)
        self.assertItemsAlmostEqual(v_prob.value, npSPriors)

    def test_entr(self):
        """Test a problem with entr.
        """
        for n in [5, 10, 25]:
            print(n)
            x = Variable(n)
            obj = Maximize(sum_entries(entr(x)))
            p = Problem(obj, [sum_entries(x) == 1])
            p.solve(solver=SCS, verbose=True)
            self.assertItemsAlmostEqual(x.value, n*[1./n])

    def test_exp(self):
        """Test a problem with exp.
        """
        for n in [5, 10, 25]:
            print(n)
            x = Variable(n)
            obj = Minimize(sum_entries(exp(x)))
            p = Problem(obj, [sum_entries(x) == 1])
            p.solve(solver=SCS, verbose=True)
            self.assertItemsAlmostEqual(x.value, n*[1./n])

    def test_log(self):
        """Test a problem with log.
        """
        for n in [5, 10, 25]:
            print(n)
            x = Variable(n)
            obj = Maximize(sum_entries(log(x)))
            p = Problem(obj, [sum_entries(x) == 1])
            p.solve(solver=SCS, verbose=True)
            self.assertItemsAlmostEqual(x.value, n*[1./n])

    def test_consistency(self):
        """Test case for non-deterministic behavior in cvxopt.
        """
        import cvxpy

        xs = [0, 1, 2, 3]
        ys = [51, 60, 70, 75]

        eta1 = cvxpy.Variable()
        eta2 = cvxpy.Variable()
        eta3 = cvxpy.Variable()
        theta1s = [eta1 + eta3*x for x in xs]
        lin_parts = [theta1 * y + eta2 * y**2 for (theta1, y) in zip(theta1s, ys)]
        g_parts = [-cvxpy.quad_over_lin(theta1, -4*eta2) + 0.5 * cvxpy.log(-2 * eta2)
                   for theta1 in theta1s]
        objective = reduce(lambda x, y: x+y, lin_parts + g_parts)
        problem = cvxpy.Problem(cvxpy.Maximize(objective))
        problem.solve(verbose=True, solver=cvxpy.SCS)
        assert problem.status in [cvxpy.OPTIMAL_INACCURATE, cvxpy.OPTIMAL]
        return [eta1.value, eta2.value, eta3.value]

    def test_warm_start(self):
        """Test warm starting.
        """
        x = Variable(10)
        obj = Minimize(sum_entries(exp(x)))
        prob = Problem(obj, [sum_entries(x) == 1])
        result = prob.solve(solver=SCS)
        assert prob.solve(solver=SCS, verbose=True) == result
        # TODO Probably a bad check. Ought to be the same.
        assert prob.solve(solver=SCS, warm_start=True, verbose=True) != result

    # def test_kl_div(self):
    #     """Test the kl_div atom.
    #     """
    #     self.assertEqual(kl_div(0, 0).value, 0)
    #     self.assertEqual(kl_div(1, 0).value, np.inf)
    #     self.assertEqual(kl_div(0, 1).value, np.inf)
    #     self.assertEqual(kl_div(-1, -1).value, np.inf)

max_stars_repo_name: quantopian/cvxpy

max_issues_repo_name: quantopian/cvxpy

max_forks_repo_name: quantopian/cvxpy

reason: import numpy,import cvxpy,from cvxpy


text:
import json
import random
import filetype
import os, librosa
import numpy as np
import soundfile as sf
from neural_loop_combiner.config import settings

def save_np(file_path, file):
    existed = os.path.exists(file_path)
    if not existed:
        np.save(file_path, file)
    return existed

def load_np(file_path, cache):
    return np.load(file_path, allow_pickle=True) if cache and os.path.exists(file_path) else None

def save_audio(audio_path, audio, sr):
    existed = os.path.exists(audio_path)
    sf.write(audio_path, audio, sr)
    return existed

def check_exist(output_path):
    existed = os.path.exists(output_path)
    if not existed:
        os.makedirs(output_path)
    return existed

def check_files_exist(files):
    exists = [os.path.exists(file) for file in files]
    return sum(exists) == len(files)

def str_replace(string):
    return string.replace('/', '_').replace(' ', '_').replace('__', '_').replace('__', '_')

def get_save_dir(out_dir, cat_dirs):
    for cat_dir in cat_dirs:
        out_dir = os.path.join(out_dir, cat_dir)
        check_exist(out_dir)
    return out_dir

def get_file_name(file_path):
    media_type = filetype.guess(file_path).extension
    file_name  = os.path.split(file_path)[-1].split(f'.{media_type}')[0]
    return file_name, media_type

def remove_prefix_dir(path, first_dir):
    return os.path.relpath(path, first_dir)

def log_message(message, log_info=[]):
    if len(log_info) <= 0:
        print(f'{message} ...')
    else:
        junction = '/'
        print(f'[{junction.join([str(log) for log in log_info])}] {message}...')

def discrete_matrix(matrix, thre):
    return matrix > thre

def pair_iteration(lst1, lst2):
    return [list(lst) for lst in np.unique([np.array(sorted([x, y])) for x in lst1 for y in lst2 if x != y], axis=0)]

def pair_table_creation(template):
    pair_template = [v for v in template if sum(v) >= 2]
    if len(pair_template) != 0:
        unique_bar = np.unique(pair_template, axis=0)
        pair_bar   = [sorted(y[0]) for y in (map(lambda x: np.where(x == True), unique_bar))]
        pair_table = np.unique([np.array(y) for x in list(map(lambda bar: pair_iteration(bar, bar), pair_bar)) for y in x], axis=0)
        pair_table = [list(x) for x in pair_table]
    else:
        pair_table = []
    return pair_table

def data_shuffle(array):
    random.shuffle(array)
    return array

def data_exclude(array, excl_array):
    return [element for element in array if element not in excl_array]

def data_include(array, incd_array):
    return [element for element in array if element in incd_array]

def data_deduplicate(array):
    deduplicate_array = []
    for element in array:
        if element not in deduplicate_array:
            deduplicate_array.append(element)
    return deduplicate_array

def save_json(data, file_path):
    with open(file_path, 'w') as outfile:
        json.dump(data, outfile)

def load_json(file_path):
    with open(file_path, 'r') as file:
        data = json.load(file)
    return data

max_stars_repo_name: mir-aidj/neural-Loop-combiner

max_issues_repo_name: mir-aidj/neural-Loop-combiner

max_forks_repo_name: mir-aidj/neural-Loop-combiner


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Oct  1 15:24:04 2020

@author: agnitm
"""

import numpy as np # Numerical Python
from spacepy.pybats import rim # To read in SWMF files
import ampere_read as amprd
import datetime as dt

def calc_I_swmf(fname):
    """
        See string for calc_I in SpacePy's pybats.rim
    """

    # Read the IE data file
    data = rim.Iono(fname)
    data['time'] = data.attrs['time']

    # Compute dLon and dLat
    data.dlon = data['n_psi'  ][0,3] - data['n_psi'  ][0,2]
    data.dlat = data['n_theta'][3,0] - data['n_theta'][2,0]

    # Calculate some physically meaningful values/units
    units = 1E-6*1E-6  # micro amps to amps, amps to MegaAmps
    R = (6371.0+110.0)*1000.0 # Radius of Earth + iono altitude
    dTheta = np.pi*data.dlat/180.
    dPhi   = np.pi*data.dlon/180.

    # -----NORTHERN HEMISPHERE-----
    # Get relevant values:
    colat     = data['n_theta']*np.pi/180.
    integrand = data['n_jr']*np.sin(colat)*dTheta*dPhi
    # Get locations of "up" and "down"
    loc_up = data['n_jr']>0
    loc_do = data['n_jr']<0
    data['n_I']     = units*R**2 * np.sum(integrand)
    data['n_Iup']   = units*R**2 * np.sum(integrand[loc_up])
    data['n_Idown'] = units*R**2 * np.sum(integrand[loc_do])
    data['n_Itotal']= units*R**2 * np.sum(np.abs(integrand)) * 0.5

    # -----SOUTHERN HEMISPHERE-----
    # Get relevant values:
    colat     = data['s_theta']*np.pi/180.
    integrand = data['s_jr']*np.sin(colat)*dTheta*dPhi
    # Get locations of "up" and "down"
    loc_up = data['s_jr']>0
    loc_do = data['s_jr']<0
    data['s_I']     = units*R**2 * np.sum(integrand)
    data['s_Iup']   = units*R**2 * np.sum(integrand[loc_up])
    data['s_Idown'] = units*R**2 * np.sum(integrand[loc_do])
    data['s_Itotal']= units*R**2 * np.sum(np.abs(integrand)) * 0.5

    return data['n_Itotal'], data['s_Itotal']

def calc_I_ampere(fname, time):
    """
        See string for calc_I in SpacePy's pybats.rim
    """

    # Read the IE data file
    data = amprd.ampere_read(fname, time)
    data['time'] = time

    # Compute dLon and dLat
    dlon = data['MLT'  ][0,3] - data['MLT'  ][0,2]
    dlat = data['Lat'][3,0] - data['Lat'][2,0]

    # Calculate some physically meaningful values/units
    units = 1E-6*1E-6  # micro amps to amps, amps to MegaAmps
    R = (6371.0+110.0)*1000.0 # Radius of Earth + iono altitude
    dTheta = np.pi*dlat/180.
    dPhi   = np.pi*dlon/180.

    # -----NORTHERN HEMISPHERE-----
    # Get relevant values:
    colat     = data['Lat']*np.pi/180.
    integrand = data['Jr']*np.sin(colat)*dTheta*dPhi
    # Get locations of "up" and "down"
    loc_up = data['Jr']>0
    loc_do = data['Jr']<0
    data['I']     = units*R**2 * np.sum(integrand)
    data['Iup']   = units*R**2 * np.sum(integrand[loc_up])
    data['Idown'] = units*R**2 * np.sum(integrand[loc_do])

    print(data['Iup'])

    return data

# # AMPERE time start - This should be fixed in future renditions of this code!
# tstart = dt.datetime(2011, 9, 26, 10, 0, 0) # Convert the numbers into a datetime
# t = tstart.timetuple() # Stripped the numbers into a timetuple

# northfile = (('./AMPERE/Sept2011_Event_CUSIA/' +
#               '{0:4d}{1:0=2d}{2:0=2d}.{3:0=2d}00.86400.120.north.grd.ncdf')
#              .format(t[0], t[1], t[2], t[3]))

# southfile = (('./AMPERE/Sept2011_Event_CUSIA/' +
#               '{0:4d}{1:0=2d}{2:0=2d}.{3:0=2d}00.86400.120.south.grd.ncdf')
#              .format(t[0], t[1], t[2], t[3]))

# time = dt.datetime(2011, 9, 27, 0, 30, 0)
# data = calc_I_ampere(northfile, time)


text:
from app import app
from flask import request, render_template, url_for
import os
import cv2
import numpy as np
from PIL import Image
import random
import string
import pytesseract
#download tesserect
pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
# Adding path to config
app.config['INITIAL_FILE_UPLOADS'] = 'app/static/uploads'
language='eng'
@app.route("/", methods=["GET", "POST"])
def index():

	# Execute if request is get
	if request.method == "GET":
		file_name =  'images/default.jpg'
		return render_template("index1.html", full_filename = full_filename)

	# Execute if reuqest is post
	if request.method == "POST":

		id_image = request.files['image_upload']
		imagename = id_image.filename
		image = Image.open(image_upload)

		# Converting image to array
		image_arr = np.array(image.convert('RGB'))
		# Converting image to grayscale
		gray_img_arr = cv2.cvtColor(image_arr, cv2.COLOR_BGR2GRAY)
		#Converting image back to rbg
		image = Image.fromarray(gray_img_arr)

		# Printing lowercase
		letters = string.ascii_lowercase
		# Generating unique image name for dynamic image display
		name = ''.join(random.choice(letters) for i in range(10)) + '.png'
		file_name =  'uploads/' + name

		# Extracting text from image
		custom_config = '-l '+language+' --oem 3 --psm 6'
		text = pytesseract.image_to_string(image,config=custom_config)

		# Remove symbol if any
		characters_to_remove = "!()@—*“>+-/,'|£#%$&^_~"
		new_string = text
		for character in characters_to_remove:
			new_string = new_string.replace(character, "")

		# Converting string into list to dislay extracted text in seperate line
		new_string = new_string.split("\n")

		# Saving image to display in html
		img = Image.fromarray(image_arr, 'RGB')
		img.save(os.path.join(app.config['INITIAL_FILE_UPLOADS'], name))
		# Returning template, filename, extracted text
		return render_template('index1.html', file_name = file_name, text = new_string)

@app.route('/about/', methods=["GET"])
def about():
	if request.method == "GET":

		return render_template('About.html')

# Main function
if __name__ == '__main__':
    app.run(debug=True)


text:
import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

iris = load_iris()
X, y = iris.data, iris.target

print("Dataset size: %d  number of features: %d  number of classes: %d"
      % (X.shape[0], X.shape[1], len(np.unique(y))))

X_train, X_test, y_train, y_test = train_test_split(X, y)

plt.scatter(X_train[:, 0], X_train[:, 1], c=y_train)
plt.xlabel(iris.feature_names[0])
plt.ylabel(iris.feature_names[1])

plt.figure()
plt.scatter(X_train[:, 2], X_train[:, 3], c=y_train)
plt.xlabel(iris.feature_names[2])
plt.ylabel(iris.feature_names[3])

import sklearn.datasets
import os
import pandas as pd
iris_path = os.path.join(sklearn.datasets.__path__[0], 'data', 'iris.csv')
iris_df =  pd.read_csv(iris_path, header=None)
display(iris_df.head())

iris_df = pd.read_csv(iris_path, skiprows=1, header=None)
display(iris_df.head())

features = iris_df.iloc[:, :4]
target = iris_df.iloc[:, 4]


text:
# ======== IMPORTS ========

import os
from io import BytesIO
import numpy as np
from functools import partial
import PIL.Image
import tensorflow as tf

# =========================

# ======== SETUP ========

"""
The GoogLeNet architecture (InceptionV5) is used here which has been pretrained on multiple for several weeks on the
ImageNet dataset.
"""

# Model location
model_fn = "tensorflow_inception_graph.pb"

# Create an interactive session and base to load the graph into
graph = tf.Graph()
sess = tf.InteractiveSession(graph=graph)

# Read the graph in
with tf.gfile.FastGFile(model_fn, "rb") as f:
    graph_def = tf.GraphDef()
    graph_def.ParseFromString(f.read())

# Define the input Tensor
t_input = tf.placeholder(np.float32, name="input")
imagenet_mean = 117.0
t_preprocessed = tf.expand_dims(t_input - imagenet_mean, 0)
tf.import_graph_def(graph_def, {"input":t_preprocessed})

# =======================

# ======== HELPER FUNCTIONS ========

def showarray(a, fmt="jpeg"):
    a = np.uint8(np.clip(a, 0, 1) * 255)
    f = BytesIO()
    img = PIL.Image.fromarray(a) #.save(f, fmt)
    img.show()

def wait():
    input("Press enter to continue...")

def visstd(a, s=0.1):
    """
    Normalise the image range for visualisation.
    :param a: the array to normalise
    :param s: ?
    :return: the normalised image
    """
    return (a - a.mean()) / max(a.std(), 1e-4)*s + 0.5

def T(layer):
    """
    Convenience function for getting a layer's output tensor
    :param layer: the layer to get the tensor
    :return: the tensor
    """
    return graph.get_tensor_by_name("import/%s:0" % layer)

def tffunc(*argtypes):
    """
    Helper function that transforms the TF-graph generating function into a regular one - used to resize the image with
    Tensorflow in combination with the "resize" function below.
    :param argtypes: multiple parameters.
    :return: a normal function
    """
    placeholders = list(map(tf.placeholder, argtypes))

    def wrap(f):
        out = f(*placeholders)

        def wrapper(*args, **kw):
            return out.eval(dict(zip(placeholders, args)), session=kw.get("session"))

        return wrapper

    return wrap

def resize(img, size):
    """
    Resizes and image using Tensorflow. Works in tandem with tffunc, above.
    :param img: the image to resize.
    :param size: the size to change the image to.
    :return: the resized image.
    """
    # Adds an extra dimension to the image at index 1, for example we already have img=[height, width, channels], then
    # by using "expand_dims" we turn this into a batch of 1 images: [1, height, width, channels].
    img = tf.expand_dims(img, 0)

    return tf.image.resize_bilinear(img, size)[0, :, :, :]

# Wrap the TF-based resize function to make it into a normally callable function
resize = tffunc(np.float32, np.int32)(resize)

def calc_grad_tiled(img, t_grad, tile_size=512):
    """
    Computes the value of tensor t_grad over the image in a tiled way. Random shifts are applied to the image to blur
    tile boundaries over multiple iterations.
    :param img: the image to modify.
    :param t_grad: the gradient to compute, as a TensorFlow operation.
    :param tile_size: the size of each image tile.
    :return: the randomly shifted image.
    """
    # Image metrics
    size = tile_size
    height, width = img.shape[:2]

    # Random shift coordinates
    sx, sy = np.random.randint(size, size=2)

    # Shift the image
    img_shift = np.roll(np.roll(img, sx, 1), sy, 0)

    # The gradient of the image
    grad = np.zeros_like(img)

    # Calculate the gradients for the image tiles. These funky for loop conditions are for if the image is larger than
    # the size of each tile. If it's smaller than the tile, we can compute it all in one. If it's larger than the tile,
    # then we will have to do multiple iterations to discover the gradient for the whole image.
    for y in range(0, max(height-size//2, size), size):
        for x in range(0, max(width-size//2, size), size):

            sub = img_shift[y:y+size, x:x+size]
            g = sess.run(t_grad, {t_input: sub})
            grad[y:y+size, x:x+size] = g

    return np.roll(np.roll(grad, -sx, 1), -sy, 0)

def render_multiscale(t_obj, img0, iter_n=10, step=1.0, octave_n=3, octave_scale=1.4):
    """
    Renders the image at different sizes.
    :param t_obj: the objective to render.
    :param img0: the image to alter.
    :param iter_n: the number of iterations of changes to apply.
    :param step: the step size for each image alteration.
    :param octave_n: the number of different octaves to scale over.
    :param octave_scale: scale up the octaves.
    """
    # The optimisation objective
    t_score = tf.reduce_mean(t_obj)

    # The gradient
    t_grad = tf.gradients(t_score, t_input)[0]

    img = img0.copy()

    for octave in range(octave_n):

        if octave > 0:

            hw = np.float32(img.shape[:2]) * octave_scale
            img = resize(img, np.int32(hw))

        for i in range(iter_n):

            # Calculate the image gradient for different tiles of the image
            g = calc_grad_tiled(img, t_grad)

            # Normalise the gradient, so the same step size should work for different layers and networks
            g /= g.std() + 1e-8

            # Modify the image
            img += g*step

            print(".", end=" ")

        showarray(visstd(img))

# ==================================

# ======== MAIN CODE ========

"""
We try to generate images that maximize the sum of activations of a a particular channel of a particular convolutional
layer of the neural network. InceptionV5 contains many convolutional layers, each of which outputs tens to hundreds of
feature channels. This allows many different patterns to be explored.
"""

# Create a list of all the layers in the network
layers = [op.name for op in graph.get_operations() if op.type=="Conv2D" and "import/" in op.name]
feature_nums = [int(graph.get_tensor_by_name(name+":0").get_shape()[-1]) for name in layers]

"""
MULTISCALE_DREAMING applies gradient ascent on multiple scales within the network. Details formed on the smaller scale
will be upscaled and augmented with additional details on the next scale. To make a higher resolution image, we split
the image into smaller tiles and compute each tile gradient independently. By applying random shifts to the image before
each iteration, we help to avoid tile seams and improve the overall image quality. We need to do this in a tiled way
otherwise the computer is liable to run out of memory.
"""

# Pick an internal layer to enhance. We use outputs before applying the ReLU nonlinearity to have non-zero gradients
# for features with negative initial activations
layer = "mixed4d_3x3_bottleneck_pre_relu"

# Pick a random feature channel to visualise - there are 144 in that layer
channel = 32

# Make an image of random noise
img_noise = np.random.uniform(size=(224, 224, 3)) + 100.0

# Read an image - applying simple dreaming to it doesn't really do anything just overlays the same pattern as random
# noise but very vaguely
image = PIL.Image.open("mountain.jpg")

# The objective to visualise
objective = T(layer)[:, :, :, channel]

# Render the image
render_multiscale(objective, img0=img_noise)

# ===========================


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (c) 2020 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import abstractmethod
from collections import Counter
from lpot.utils.utility import LazyImport, singleton
from ..utils import logger
from sklearn.metrics import accuracy_score
import numpy as np

torch_ignite = LazyImport('ignite')
torch = LazyImport('torch')
tf = LazyImport('tensorflow')
mx = LazyImport('mxnet')

@singleton
class TensorflowMetrics(object):
    def __init__(self):
        self.metrics = {}
        self.metrics.update(TENSORFLOW_METRICS)

@singleton
class PyTorchMetrics(object):
    def __init__(self):
        self.metrics = {
            "topk": WrapPyTorchMetric(
                torch_ignite.metrics.TopKCategoricalAccuracy),
            "Accuracy": WrapPyTorchMetric(
                torch_ignite.metrics.Accuracy),
            "Loss": WrapPyTorchMetric(
                torch_ignite.metrics.Loss),
            "MAE": WrapPyTorchMetric(
                torch_ignite.metrics.MeanAbsoluteError),
            "RMSE": WrapPyTorchMetric(
                torch_ignite.metrics.RootMeanSquaredError),
            "MSE": WrapPyTorchMetric(
                torch_ignite.metrics.MeanSquaredError),
        }
        self.metrics.update(PYTORCH_METRICS)

@singleton
class MXNetMetrics(object):
    def __init__(self):
        self.metrics = {
            "Accuracy": WrapMXNetMetric(mx.metric.Accuracy),
            "MAE": WrapMXNetMetric(mx.metric.MAE),
            "MSE": WrapMXNetMetric(mx.metric.MSE),
            "Loss": WrapMXNetMetric(mx.metric.Loss),
        }
        self.metrics.update(MXNET_METRICS)

@singleton
class ONNXRTQLMetrics(object):
    def __init__(self):
        self.metrics = {}
        self.metrics.update(ONNXRT_QL_METRICS)

@singleton
class ONNXRTITMetrics(object):
    def __init__(self):
        self.metrics = {}
        self.metrics.update(ONNXRT_IT_METRICS)

framework_metrics = {"tensorflow": TensorflowMetrics,
                     "mxnet": MXNetMetrics,
                     "pytorch": PyTorchMetrics,
                     "pytorch_ipex": PyTorchMetrics,
                     "onnxrt_qlinearops": ONNXRTQLMetrics,
                     "onnxrt_integerops": ONNXRTITMetrics}

# user/model specific metrics will be registered here
TENSORFLOW_METRICS = {}
MXNET_METRICS = {}
PYTORCH_METRICS = {}
ONNXRT_QL_METRICS = {}
ONNXRT_IT_METRICS = {}

registry_metrics = {"tensorflow": TENSORFLOW_METRICS,
                    "mxnet": MXNET_METRICS,
                    "pytorch": PYTORCH_METRICS,
                    "pytorch_ipex": PYTORCH_METRICS,
                    "onnxrt_qlinearops": ONNXRT_QL_METRICS,
                    "onnxrt_integerops": ONNXRT_IT_METRICS}

class METRICS(object):
    def __init__(self, framework):
        assert framework in ("tensorflow", "pytorch", "pytorch_ipex", "onnxrt_qlinearops",
                             "onnxrt_integerops", "mxnet"), \
                             "framework support tensorflow pytorch mxnet onnxrt"
        self.metrics = framework_metrics[framework]().metrics

    def __getitem__(self, metric_type):
        assert metric_type in self.metrics.keys(), "only support metrics in {}".\
            format(self.metrics.keys())

        return self.metrics[metric_type]

    def register(self, name, metric_cls):
        assert name not in self.metrics.keys(), 'registered metric name already exists.'
        self.metrics.update({name: metric_cls})

def metric_registry(metric_type, framework):
    """The class decorator used to register all Metric subclasses.
       cross framework metric is supported by add param as framework='tensorflow, \
                       pytorch, mxnet, onnxrt'

    Args:
        cls (class): The class of register.

    Returns:
        cls: The class of register.
    """
    def decorator_metric(cls):
        for single_framework in [fwk.strip() for fwk in framework.split(',')]:
            assert single_framework in [
                "tensorflow",
                "mxnet",
                "onnxrt_qlinearops",
                "onnxrt_integerops",
                "pytorch",
                "pytorch_ipex"], "The framework support tensorflow mxnet pytorch onnxrt"

            if metric_type in registry_metrics[single_framework].keys():
                raise ValueError('Cannot have two metrics with the same name')
            registry_metrics[single_framework][metric_type] = cls
        return cls
    return decorator_metric

class Metric(object):
    def __init__(self, metric, single_output=False):
        self._metric_cls = metric
        self._single_output = single_output

    def __call__(self, *args, **kwargs):
        self._metric = self._metric_cls(*args, **kwargs)
        return self

    @abstractmethod
    def update(self, preds, labels=None, sample_weight=None):
        raise NotImplementedError

    @abstractmethod
    def reset(self):
        raise NotImplementedError

    @abstractmethod
    def result(self):
        raise NotImplementedError

    @property
    def metric(self):
        return self._metric

class WrapPyTorchMetric(Metric):

    def update(self, preds, labels=None, sample_weight=None):
        if self._single_output:
            output = torch.as_tensor(preds)
        else:
            output = (torch.as_tensor(preds), torch.as_tensor(labels))
        self._metric.update(output)

    def reset(self):
        self._metric.reset()

    def result(self):
        return self._metric.compute()

class WrapMXNetMetric(Metric):

    def update(self, preds, labels=None, sample_weight=None):
        preds = mx.nd.array(preds)
        labels = mx.nd.array(labels)
        self._metric.update(labels=labels, preds=preds)

    def reset(self):
        self._metric.reset()

    def result(self):
        acc_name, acc = self._metric.get()
        return acc

class WrapONNXRTMetric(Metric):

    def update(self, preds, labels=None, sample_weight=None):
        preds = np.array(preds)
        labels = np.array(labels)
        self._metric.update(labels=labels, preds=preds)

    def reset(self):
        self._metric.reset()

    def result(self):
        acc_name, acc = self._metric.get()
        return acc

def _topk_shape_validate(preds, labels):
    # preds shape can be Nxclass_num or class_num(N=1 by default)
    # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax
    if isinstance(preds, int):
        preds = [preds]
    preds = np.array(preds)

    # consider labels just int value 1x1
    if isinstance(labels, int):
        labels = [labels]
    # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot)
    # only support 2 dimension one-shot labels
    # or 1 dimension one-hot class_num will confuse with N
    labels = np.array(labels)

    if len(preds.shape) == 1:
        N = 1
        class_num = preds.shape[0]
        preds = preds.reshape([-1, class_num])
    elif len(preds.shape) >= 2:
        N = preds.shape[0]
        preds = preds.reshape([N, -1])
        class_num = preds.shape[1]

    label_N = labels.shape[0]
    assert label_N == N, 'labels batch size should same with preds'
    labels = labels.reshape([N, -1])
    # one-hot labels will have 2 dimension not equal 1
    if labels.shape[1] != 1:
        labels = labels.argsort()[..., -1:]
    return preds, labels

def _shape_validate(preds, labels):
    if isinstance(preds, int):
        preds = [preds]
    preds = np.array(preds)
    if isinstance(labels, int):
        labels = [labels]
    labels = np.array(labels)
    assert preds.shape == labels.shape, 'shape of labels {} does not match \
                    shape of predictions {}'.format(labels.shape, preds.shape)
    return preds, labels

@metric_registry('topk', 'mxnet')
class MxnetTopK(Metric):
    """The class of calculating topk metric, which usually is used in classification.

    Args:
        topk (dict): The dict of topk for configuration.

    """

    def __init__(self, k=1):
        self.k = k
        self.num_correct = 0
        self.num_sample = 0

    def update(self, preds, labels, sample_weight=None):

        preds, labels = _topk_shape_validate(preds, labels)
        preds = preds.argsort()[..., -self.k:]
        if self.k == 1:
            correct = accuracy_score(preds, labels, normalize=False)
            self.num_correct += correct

        else:
            for p, l in zip(preds, labels):
                # get top-k labels with np.argpartition
                # p = np.argpartition(p, -self.k)[-self.k:]
                l = l.astype('int32')
                if l in p:
                    self.num_correct += 1

        self.num_sample += len(labels)

    def reset(self):
        self.num_correct = 0
        self.num_sample = 0

    def result(self):
        if self.num_sample == 0:
            logger.warning("sample num is 0 can't calculate topk")
            return 0
        else:
            return self.num_correct / self.num_sample

@metric_registry('F1', 'tensorflow, pytorch, mxnet, onnxrt_qlinearops, onnxrt_integerops')
class F1(Metric):
    def __init__(self):
        self._score_list = []

    def update(self, preds, labels):
        from .f1 import f1_score
        result = f1_score(preds, labels)
        self._score_list.append(result)

    def reset(self):
        self._score_list = []

    def result(self):
        return np.array(self._score_list).mean()

def _accuracy_shape_check(preds, labels):
    if isinstance(preds, int):
        preds = [preds]
    preds = np.array(preds)
    if isinstance(labels, int):
        labels = [labels]
    labels = np.array(labels)
    if len(labels.shape) != len(preds.shape) and len(labels.shape)+1 != len(preds.shape):
        raise ValueError(
            'labels must have shape of (batch_size, ..) and preds must have'
            'shape of (batch_size, num_classes, ...) or (batch_size, ..),'
            'but given {} and {}.'.format(labels.shape, preds.shape))
    return preds, labels

def _accuracy_type_check(preds, labels):
   if len(preds.shape) == len(labels.shape)+1:
       num_classes = preds.shape[1]
       if num_classes == 1:
           update_type = 'binary'
       else:
           update_type = 'multiclass'
   elif len(preds.shape) == len(labels.shape):
       if len(preds.shape) == 1 or preds.shape[1] ==1:
           update_type = 'binary'
       else:
           update_type = 'multilabel'
   return update_type

@metric_registry('Accuracy', 'tensorflow, onnxrt_qlinearops, onnxrt_integerops')
class Accuracy(Metric):
    def __init__(self):
        self.pred_list = []
        self.label_list = []
        self.sample = 0

    def update(self, preds, labels, sample_weight=None):
        preds, labels = _accuracy_shape_check(preds, labels)
        update_type = _accuracy_type_check(preds, labels)
        if update_type == 'binary':
            self.pred_list.extend(preds)
            self.label_list.extend(labels)
            self.sample += labels.shape[0]
        elif update_type == 'multiclass':
            self.pred_list.extend(np.argmax(preds, axis=1).astype('int32'))
            self.label_list.extend(labels)
            self.sample += labels.shape[0]
        elif update_type == 'multilabel':
            #(N, C, ...) -> (N*..., C)
            num_label = preds.shape[1]
            last_dim = len(preds.shape)
            if last_dim-1 != 1:
                trans_list = [0]
                trans_list.extend(list(range(2, len(preds.shape))))
                trans_list.extend([1])
                preds = preds.transpose(trans_list).reshape(-1, num_label)
                labels = labels.transpose(trans_list).reshape(-1, num_label)
            self.sample += preds.shape[0]*preds.shape[1]
            self.pred_list.append(preds)
            self.label_list.append(labels)

    def reset(self):
        self.pred_list = []
        self.label_list = []
        self.sample = 0

    def result(self):
        correct_num = np.sum(
            np.array(self.pred_list) == np.array(self.label_list))
        return correct_num / self.sample

@metric_registry('Loss', 'tensorflow, onnxrt_qlinearops, onnxrt_integerops')
class Loss(Metric):
    def __init__(self):
        self.sample = 0
        self.sum = 0

    def update(self, preds, labels, sample_weight=None):
        preds, labels = _shape_validate(preds, labels)
        self.sample += labels.shape[0]
        self.sum += sum(preds)

    def reset(self):
        self.sample = 0
        self.sum = 0

    def result(self):
        return self.sum / self.sample

@metric_registry('MAE', 'tensorflow, onnxrt_qlinearops, onnxrt_integerops')
class MAE(Metric):
    def __init__(self):
        self.label_list = []
        self.pred_list = []

    def update(self, preds, labels, sample_weight=None):
        preds, labels = _shape_validate(preds, labels)
        self.label_list.extend(labels)
        self.pred_list.extend(preds)

    def reset(self):
        self.label_list = []
        self.pred_list = []

    def result(self):
        ae = [abs(a-b) for (a,b) in zip(self.label_list, self.pred_list)]
        return np.mean(ae)

@metric_registry('RMSE', 'tensorflow, mxnet, onnxrt_qlinearops, onnxrt_integerops')
class RMSE(Metric):
    def __init__(self):
        self.mse = MSE()

    def update(self, preds, labels, sample_weight=None):
        self.mse.update(preds, labels, sample_weight)

    def reset(self):
        self.mse.reset()

    def result(self):
        return np.sqrt(self.mse.result())

@metric_registry('MSE', 'tensorflow, onnxrt_qlinearops, onnxrt_integerops')
class MSE(Metric):
    def __init__(self):
        self.label_list = []
        self.pred_list = []

    def update(self, preds, labels, sample_weight=None):
        preds, labels = _shape_validate(preds, labels)
        self.pred_list.extend(preds)
        self.label_list.extend(labels)

    def reset(self):
        self.label_list = []
        self.pred_list = []

    def result(self):
        square = [(a-b)**2.0 for (a,b) in zip(self.label_list, self.pred_list)]
        return np.mean(square)

@metric_registry('topk', 'tensorflow')
class TensorflowTopK(Metric):
    """The class of calculating topk metric, which usually is used in classification.

    Args:
        topk (dict): The dict of topk for configuration.

    """

    def __init__(self, k=1):
        self.k = k
        self.num_correct = 0
        self.num_sample = 0

    def update(self, preds, labels, sample_weight=None):

        preds, labels = _topk_shape_validate(preds, labels)

        labels = labels.reshape([len(labels)])
        with tf.Graph().as_default() as acc_graph:
          topk = tf.nn.in_top_k(predictions=tf.constant(preds, dtype=tf.float32),
                                targets=tf.constant(labels, dtype=tf.int32), k=self.k)
          fp32_topk = tf.cast(topk, tf.float32)
          correct_tensor = tf.reduce_sum(input_tensor=fp32_topk)

          with tf.compat.v1.Session() as acc_sess:
            correct  = acc_sess.run(correct_tensor)

        self.num_sample += len(labels)
        self.num_correct += correct

    def reset(self):
        self.num_correct = 0
        self.num_sample = 0

    def result(self):
        if self.num_sample == 0:
            logger.warning("sample num is 0 can't calculate topk")
            return 0
        else:
            return self.num_correct / self.num_sample

@metric_registry('topk', 'onnxrt_qlinearops, onnxrt_integerops')
class ONNXRTTopK(Metric):
    """The class of calculating topk metric, which usually is used in classification.

    Args:
        topk (dict): The dict of topk for configuration.

    """

    def __init__(self, k=1):
        self.k = k
        self.num_correct = 0
        self.num_sample = 0

    def update(self, preds, labels, sample_weight=None):

        preds, labels = _topk_shape_validate(preds, labels)
        preds = preds.argsort()[..., -self.k:]
        if self.k == 1:
            correct = accuracy_score(preds, labels, normalize=False)
            self.num_correct += correct

        else:
            for p, l in zip(preds, labels):
                # get top-k labels with np.argpartition
                # p = np.argpartition(p, -self.k)[-self.k:]
                l = l.astype('int32')
                if l in p:
                    self.num_correct += 1

        self.num_sample += len(labels)

    def reset(self):
        self.num_correct = 0
        self.num_sample = 0

    def result(self):
        if self.num_sample == 0:
            logger.warning("sample num is 0 can't calculate topk")
            return 0
        else:
            return self.num_correct / self.num_sample

@metric_registry('COCOmAP', 'tensorflow')
class TensorflowCOCOMAP(Metric):
    """The class of calculating mAP metric

    """
    def __init__(self, anno_path=None):
        from lpot.metric.coco_label_map import category_map
        if anno_path:
            import os
            import json
            assert os.path.exists(anno_path), 'Annotation path does not exists!'
            label_map = {}
            with open(anno_path) as fin:
                annotations = json.load(fin)
            for cnt, cat in enumerate(annotations["categories"]):
                label_map[cat["name"]] = cnt + 1
            self.category_map_reverse = {k:v for k,v in label_map.items()}
        else:
            # label: index
            self.category_map_reverse = {v: k for k, v in category_map.items()}
        self.image_ids = []
        self.ground_truth_list = []
        self.detection_list = []
        self.annotation_id = 1
        self.category_map = category_map
        self.category_id_set = set(
            [cat for cat in self.category_map]) #index

    def update(self, predicts, labels, sample_weight=None):
        from lpot.metric.coco_tools import ExportSingleImageGroundtruthToCoco,\
            ExportSingleImageDetectionBoxesToCoco
        bbox, str_label,int_label, image_id = labels
        detection = {}
        if len(predicts) == 3:
            detection['boxes'] = np.asarray(predicts[0][0])
            detection['scores'] = np.asarray(predicts[1][0])
            detection['classes'] = np.asarray(predicts[2][0])
        elif len(predicts) == 4:
            num = int(predicts[0][0])
            detection['boxes'] = np.asarray(predicts[1][0])[0:num]
            detection['scores'] = np.asarray(predicts[2][0])[0:num]
            detection['classes'] = np.asarray(predicts[3][0])[0:num]
        else:
            raise ValueError("Unsupported prediction format!")

        ground_truth = {}
        ground_truth['boxes'] = np.asarray(bbox[0])
        if len(int_label[0]) == 0:
            # convert string label to int index
            str_label = [
                x if type(x) == 'str' else x.decode('utf-8')
                for x in str_label[0]
            ]
            ground_truth['classes'] = np.asarray(
                [self.category_map_reverse[x] for x in str_label])
        elif len(str_label[0]) == 0:
            ground_truth['classes'] = np.asarray(
                [x for x in int_label[0]])
        image_id = image_id[0] if type(
            image_id[0]) == 'str' else image_id[0].decode('utf-8')

        if image_id in self.image_ids:
            return
        self.image_ids.append(image_id)
        self.ground_truth_list.extend(
            ExportSingleImageGroundtruthToCoco(
                image_id=image_id,
                next_annotation_id=self.annotation_id,
                category_id_set=self.category_id_set,
                groundtruth_boxes=ground_truth['boxes'],
                groundtruth_classes=ground_truth['classes']))
        self.annotation_id += ground_truth['boxes'].shape[0]

        self.detection_list.extend(
            ExportSingleImageDetectionBoxesToCoco(
                image_id=image_id,
                category_id_set=self.category_id_set,
                detection_boxes=detection['boxes'],
                detection_scores=detection['scores'],
                detection_classes=detection['classes']))

    def reset(self):
        self.image_ids = []
        self.ground_truth_list = []
        self.detection_list = []
        self.annotation_id = 1

    def result(self):
        from lpot.metric.coco_tools import COCOWrapper, COCOEvalWrapper
        if len(self.ground_truth_list) == 0:
            logger.warning("sample num is 0 can't calculate mAP")
            return 0
        else:
            groundtruth_dict = {
                'annotations':
                self.ground_truth_list,
                'images': [{
                    'id': image_id
                } for image_id in self.image_ids],
                'categories': [{
                    'id': k,
                    'name': v
                } for k, v in self.category_map.items()]
            }
            coco_wrapped_groundtruth = COCOWrapper(groundtruth_dict)
            coco_wrapped_detections = coco_wrapped_groundtruth.LoadAnnotations(
                self.detection_list)
            box_evaluator = COCOEvalWrapper(coco_wrapped_groundtruth,
                                                 coco_wrapped_detections,
                                                 agnostic_mode=False)
            box_metrics, box_per_category_ap = box_evaluator.ComputeMetrics(
                include_metrics_per_category=False, all_metrics_per_category=False)
            box_metrics.update(box_per_category_ap)
            box_metrics = {
                'DetectionBoxes_' + key: value
                for key, value in iter(box_metrics.items())
            }

            return box_metrics['DetectionBoxes_Precision/mAP']

max_stars_repo_name: daisyden/lpot

max_issues_repo_name: daisyden/lpot

max_forks_repo_name: daisyden/lpot


text:
# %%
# =========================================================================== #
#                                MONITOR                                      #
# =========================================================================== #
# =========================================================================== #
# Project: ML Studio                                                          #
# Version: 0.1.14                                                             #
# File: \monitor.py                                                           #
# Python Version: 3.8.0                                                       #
# ---------------                                                             #
# Author: John James                                                          #
# Company: Decision Scients                                                   #
# Email: jjames@decisionscients.com                                           #
# ---------------                                                             #
# Create Date: Tuesday November 5th 2019, 8:47:45 pm                          #
# Last Modified: Saturday November 30th 2019, 10:37:20 am                     #
# Modified By: John James (jjames@decisionscients.com)                        #
# ---------------                                                             #
# License: Modified BSD                                                       #
# Copyright (c) 2019 Decision Scients                                         #
# =========================================================================== #

"""Module containing callbacks used to monitor and report training performance."""
import datetime
import numpy as np
import types
from ml_studio.supervised_learning.training.callbacks import Callback

# --------------------------------------------------------------------------- #
#                             HISTORY CLASS                                   #
# --------------------------------------------------------------------------- #
class History(Callback):
    """Records history and metrics for training by epoch."""
    def on_train_begin(self, logs=None):
        """Sets instance variables at the beginning of training.

        Parameters
        ----------
        logs : Dict
            Dictionary containing the X and y data
        """
        self.total_epochs = 0
        self.total_batches = 0
        self.start = datetime.datetime.now()
        self.epoch_log = {}
        self.batch_log = {}

    def on_train_end(self, logs=None):
        """Sets instance variables at end of training.

        Parameters
        ----------
        logs : Dict
            Not used
        """
        self.end = datetime.datetime.now()
        self.duration = (self.end-self.start).total_seconds()

    def on_batch_end(self, batch, logs=None):
        """Updates data and statistics relevant to the training batch.

        Parameters
        ----------
        batch : int
            The current training batch

        logs : dict
            Dictionary containing batch statistics, such as batch size, current
            weights and training cost.
        """
        self.total_batches = batch
        for k,v in logs.items():
            self.batch_log.setdefault(k,[]).append(v)

    def on_epoch_end(self, epoch, logs=None):
        """Updates data and statistics relevant to the training epoch.

        Parameters
        ----------
        epoch : int
            The current training epoch

        logs : dict
            Dictionary containing data and statistics for the current epoch,
            such as weights, costs, and optional validation set statistics
            beginning with 'val_'.
        """
        logs = logs or {}
        self.total_epochs = epoch
        for k,v in logs.items():
            self.epoch_log.setdefault(k,[]).append(v)

# --------------------------------------------------------------------------- #
#                            PROGRESS CLASS                                   #
# --------------------------------------------------------------------------- #
class Progress(Callback):
    """Class that reports progress at designated points during training."""

    def on_epoch_end(self, epoch, logs=None):
        """Reports progress at the end of each epoch.

        Parameters
        ----------
        epoch : int
            The current training epoch

        logs : Dict
            Statistics obtained at end of epoch
        """
        if self.model.verbose and (epoch % self.model.checkpoint == 0):
            items_to_report = ('epoch', 'train', 'val')
            logs = {k:v for k,v in logs.items() if k.startswith(items_to_report)}
            progress = "".join(str(key) + ': ' + str(round(value,4)) + ' ' \
                for key, value in logs.items())
            print(progress)

# --------------------------------------------------------------------------- #
#                                SUMMARY                                      #
# --------------------------------------------------------------------------- #

center = 25

def summary(history):
    """Summarizes statistics for model.

    Parameters
    ----------
    history : history object
        history object containing data and statistics from training.
    """
    metric = history.params.get('metric', "")

    print("\nOptimization Summary")
    print("                     Name: " + history.params.get('name'))
    print("                    Start: " + str(history.start))
    print("                      End: " + str(history.end))
    print("                 Duration: " + str(history.duration) + " seconds.")
    print("                   Epochs: " + str(history.total_epochs))
    print("                  Batches: " + str(history.total_batches))
    print("\n")
    print("Performance Summary")
    print("      Final Training Loss: " +
          str(round(history.epoch_log.get('train_cost')[-1],4)))
    if history.epoch_log.get('train_score'):
        print("     Final Training Score: " +\
             str(round(history.epoch_log.get('train_score')[-1],4))
              + " " + history.params.get('metric'))
    if history.epoch_log.get('val_cost'):
        print("    Final Validation Loss: " +
              str(round(history.epoch_log.get('val_cost')[-1],4)))
    if history.epoch_log.get('val_score'):
        print("   Final Validation Score: " + \
            str(round(history.epoch_log.get('val_score')[-1],4))
                + " " + metric)
    if history.epoch_log.get('theta')[-1].shape[0] < 10:
        print("          Final Intercept: " + str(history.epoch_log.get('theta')[-1][0]))
        print("       Final Coefficients:\t " +\
            str(history.epoch_log.get('theta')[-1][1:]).replace('\n','\n\t\t\t'))
    print("\nModel Parameters")

    for p, v in history.params.items():
        label_length = len(p)
        spaces = center - label_length
        if isinstance(v, (str, bool, int, list, np.ndarray, types.FunctionType, float)) \
                or v is None:
            p = " " * spaces + p + ": "
            # If v is a function type, it is the lambda function that
            # initializes the regularizer to zeros if the parameter
            # is None. If this is the case, we'll print "None" for
            # this parameter.
            if isinstance(v, types.FunctionType):
                v = ""
            print(p + str(v))
        else:
            _recur(p, v)

    print("\nData")
    print("        Num. Observations: " +\
             str(history.model.X.shape[0]))
    print("            Num. Features: " +\
             str(history.model.X.shape[1]-1))

def _recur(callable_type, callable_object):
    callable_name = callable_object.name
    spaces = center - len(callable_type)
    callable_type = " " * spaces + callable_type + ":"
    print(callable_type, callable_name)
    config = callable_object.get_params()
    if len(config) > 0:
        for k, v in config.items():
            if isinstance(v, (str, bool, int, list, np.ndarray, types.FunctionType, float)) \
                    or v is None:
                spaces = center - len(k)
                k = " " * spaces + k + ": "
                # If v is a function type, it is the lambda function that
                # initializes the regularizer to zeros if the parameter
                # is None. If this is the case, we'll print "None" for
                # this parameter.
                if isinstance(v, types.FunctionType):
                    v = None
                print(k + str(v))
            else:
                _recur(k, v)

max_stars_repo_name: john-james-ai/ml-studio

max_issues_repo_name: john-james-ai/ml-studio

max_forks_repo_name: john-james-ai/ml-studio