text:
import math
import torch
import torch.nn.functional as F
from scipy.special import erf

class Mechanism:
    def __init__(self, eps, input_range, **kwargs):
        self.eps = eps
        self.alpha, self.beta = input_range

    def __call__(self, x):
        raise NotImplementedError

class Gaussian(Mechanism):
    def __init__(self, *args, delta=1e-7, **kwargs):
        super().__init__(*args, **kwargs)
        self.delta = delta
        self.sigma = None
        self.sensitivity = None

    def __call__(self, x):
        len_interval = self.beta - self.alpha
        if torch.is_tensor(len_interval) and len(len_interval) > 1:
            self.sensitivity = torch.norm(len_interval, p=2)
        else:
            d = x.size(1)
            self.sensitivity = len_interval * math.sqrt(d)

        self.sigma = self.calibrate_gaussian_mechanism()
        out = torch.normal(mean=x, std=self.sigma)
        # out = torch.clip(out, min=self.alpha, max=self.beta)
        return out

    def calibrate_gaussian_mechanism(self):
        return self.sensitivity * math.sqrt(2 * math.log(1.25 / self.delta)) / self.eps

class AnalyticGaussian(Gaussian):
    def calibrate_gaussian_mechanism(self, tol=1.e-12):
        """ Calibrate a Gaussian perturbation for differential privacy
        using the analytic Gaussian mechanism of [Balle and Wang, ICML'18]
        Arguments:
        tol : error tolerance for binary search (tol > 0)
        Output:
        sigma : standard deviation of Gaussian noise needed to achieve (epsilon,delta)-DP under global sensitivity GS
        """
        delta_thr = self._case_a(0.0)
        if self.delta == delta_thr:
            alpha = 1.0
        else:
            if self.delta > delta_thr:
                predicate_stop_DT = lambda s: self._case_a(s) >= self.delta
                function_s_to_delta = lambda s: self._case_a(s)
                predicate_left_BS = lambda s: function_s_to_delta(s) > self.delta
                function_s_to_alpha = lambda s: math.sqrt(1.0 + s / 2.0) - math.sqrt(s / 2.0)
            else:
                predicate_stop_DT = lambda s: self._case_b(s) <= self.delta
                function_s_to_delta = lambda s: self._case_b(s)
                predicate_left_BS = lambda s: function_s_to_delta(s) < self.delta
                function_s_to_alpha = lambda s: math.sqrt(1.0 + s / 2.0) + math.sqrt(s / 2.0)
            predicate_stop_BS = lambda s: abs(function_s_to_delta(s) - self.delta) <= tol
            s_inf, s_sup = self._doubling_trick(predicate_stop_DT, 0.0, 1.0)
            s_final = self._binary_search(predicate_stop_BS, predicate_left_BS, s_inf, s_sup)
            alpha = function_s_to_alpha(s_final)
        sigma = alpha * self.sensitivity / math.sqrt(2.0 * self.eps)
        return sigma

    @staticmethod
    def _phi(t):
        return 0.5 * (1.0 + erf(t / math.sqrt(2.0)))

    def _case_a(self, s):
        return self._phi(math.sqrt(self.eps * s)) - math.exp(self.eps) * self._phi(-math.sqrt(self.eps * (s + 2.0)))

    def _case_b(self, s):
        return self._phi(-math.sqrt(self.eps * s)) - math.exp(self.eps) * self._phi(-math.sqrt(self.eps * (s + 2.0)))

    @staticmethod
    def _doubling_trick(predicate_stop, s_inf, s_sup):
        while not predicate_stop(s_sup):
            s_inf = s_sup
            s_sup = 2.0 * s_inf
        return s_inf, s_sup

    @staticmethod
    def _binary_search(predicate_stop, predicate_left, s_inf, s_sup):
        s_mid = s_inf + (s_sup - s_inf) / 2.0
        while not predicate_stop(s_mid):
            if predicate_left(s_mid):
                s_sup = s_mid
            else:
                s_inf = s_mid
            s_mid = s_inf + (s_sup - s_inf) / 2.0
        return s_mid

class Piecewise(Mechanism):
    def __call__(self, x):
        # normalize x between -1,1
        t = (x - self.alpha) / (self.beta - self.alpha)
        t = 2 * t - 1

        # piecewise mechanism's variables
        P = (math.exp(self.eps) - math.exp(self.eps / 2)) / (2 * math.exp(self.eps / 2) + 2)
        C = (math.exp(self.eps / 2) + 1) / (math.exp(self.eps / 2) - 1)
        L = t * (C + 1) / 2 - (C - 1) / 2
        R = L + C - 1

        # thresholds for random sampling
        threshold_left = P * (L + C) / math.exp(self.eps)
        threshold_right = threshold_left + P * (R - L)

        # masks for piecewise random sampling
        x = torch.rand_like(t)
        mask_left = x < threshold_left
        mask_middle = (threshold_left < x) & (x < threshold_right)
        mask_right = threshold_right < x

        # random sampling
        t = mask_left * (torch.rand_like(t) * (L + C) - C)
        t += mask_middle * (torch.rand_like(t) * (R - L) + L)
        t += mask_right * (torch.rand_like(t) * (C - R) + R)

        # unbias data
        x_prime = (self.beta - self.alpha) * (t + 1) / 2 + self.alpha
        return x_prime

class MultiDimPiecewise(Piecewise):
    def __call__(self, x):
        n, d = x.size()
        k = int(max(1, min(d, math.floor(self.eps / 2.5))))
        sample = torch.rand_like(x).topk(k, dim=1).indices
        mask = torch.zeros_like(x, dtype=torch.bool)
        mask.scatter_(1, sample, True)
        self.eps /= k
        y = super().__call__(x)
        z = mask * y * d / k
        return z

class RandomizedResponseTopK:
    def __init__(self, eps, k):
        """
        Args:
            eps: privacy budget parameter epsilon
            k: limits choices to top-k items having the highest probabilities
        """
        self.eps = eps
        self.k = k

    def __call__(self, y, p):
        """
        Perturbs input vector y with prior probabilities indicated by p
        Args:
            y: tensor with shape (N,1) with one private label per row
            p: tensor with shape (N,D) with one row per label incdicating the prior distribution
                over D possible outcomes

        Returns:
            perturbed labels
        """
        # add a small random noise to break ties randomly
        p += torch.randn_like(p) / 1e4

        # select top-k items
        kth_max = p.sort(dim=1, descending=True).values.gather(1, self.k - 1)
        included = p >= kth_max

        # apply randomized response on top-k items
        p_incorrect = 1.0 / (math.exp(self.eps) + self.k - 1)
        pr = torch.ones_like(p) * p_incorrect
        y = y.unsqueeze(dim=1) if len(y.size()) == 1 else y
        pr.scatter_(1, y, math.exp(self.eps) * p_incorrect)
        pr.mul_(included)
        pr.mul_(1.0 / pr.sum(dim=1).unsqueeze(1))
        return torch.multinomial(pr, num_samples=1).squeeze()

class RandomizedResponseWithPrior:
    def __init__(self, eps):
        """
        Args:
            eps: privacy budget parameter epsilon
        """
        self.eps = eps

    def __call__(self, y, p):
        """
        Perturbs input vector y with prior probabilities indicated by p
        Args:
            y: tensor with shape (N,1) with one private label per row
            p: tensor with shape (N,D) with one row per label incdicating the prior distribution
                over D possible outcomes

        Returns:
            perturbed labels
        """

        # add a small random noise to break ties randomly
        p_noisy = p + (torch.randn_like(p) / 1e4)

        # select best value of k for RRTop-K
        n, d = p.size()
        k = torch.tensor([range(d)] * n, dtype=float, device=y.device) + 1
        p_noisy = p_noisy.sort(dim=1, descending=True).values
        w = math.exp(self.eps) / (math.exp(self.eps) + k - 1)
        w.mul_(p_noisy.cumsum(dim=1))
        k = w.argmax(dim=1, keepdim=True) + 1
        return RandomizedResponseTopK(eps=self.eps, k=k)(y, p)

class OptimizedUnaryEncoding:
    def __init__(self, eps, d):
        self.d = d
        self.p = 0.5
        self.q = 1 / (math.exp(eps) + 1)

    def __call__(self, y):
        pr = y * self.p + (1 - y) * self.q
        return torch.bernoulli(pr)

    def estimate(self, b):
        n = b.size(0)
        return (b.sum(dim=0) / n - self.q) / (self.p - self.q)

max_stars_repo_name: sisaman/label-dp

max_issues_repo_name: sisaman/label-dp

max_forks_repo_name: sisaman/label-dp

reason: from scipy


text:
import argparse
import math
import numpy as np
import cv2
import yaml
from models.sr.blind_sr import BlindSR
import options.options as option
from data.common import random_anisotropic_gaussian_kernel, downsample, conv
from utils import util
from utils.util import img2tensor
import os
import imageio
from data import common
from torchvision.utils import save_image

# import skimage.metrics
os.environ["CUDA_VISIBLE_DEVICES"] = "3"

def calc_psnr(img1, img2):
    # img1 and img2 have range [0, 255]

    img1 = img1.astype(np.float64)
    img2 = img2.astype(np.float64)
    mse = np.mean((img1 - img2)**2)
    if mse == 0:
        return float('inf')
    return 20 * math.log10(255.0 / math.sqrt(mse))

def calc_ssim(img1, img2):

    C1 = (0.01 * 255)**2
    C2 = (0.03 * 255)**2

    img1 = img1.astype(np.float64)
    img2 = img2.astype(np.float64)
    kernel = cv2.getGaussianKernel(11, 1.5)
    window = np.outer(kernel, kernel.transpose())

    mu1 = cv2.filter2D(img1, -1, window)[5:-5, 5:-5]  # valid
    mu2 = cv2.filter2D(img2, -1, window)[5:-5, 5:-5]
    mu1_sq = mu1**2
    mu2_sq = mu2**2
    mu1_mu2 = mu1 * mu2
    sigma1_sq = cv2.filter2D(img1**2, -1, window)[5:-5, 5:-5] - mu1_sq
    sigma2_sq = cv2.filter2D(img2**2, -1, window)[5:-5, 5:-5] - mu2_sq
    sigma12 = cv2.filter2D(img1 * img2, -1, window)[5:-5, 5:-5] - mu1_mu2

    ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) *
                                                            (sigma1_sq + sigma2_sq + C2))
    return ssim_map.mean()

def main():
    parser = argparse.ArgumentParser(description='Train Super Resolution Models')
    #	parser.add_argument('-opt', type=str, required=True, help='Path to options JSON file.')
    #	opt = option.parse(parser.parse_args().opt)
    opt = option.parse('options/test/test_IDK.json')

    args = parser.parse_args()
    # Initializing mode
    folder_path = '../SRbenchmark/HR_x4'
    model = BlindSR(opt)
    for img in os.listdir(folder_path):
        image_path = os.path.join(folder_path, img)
        hr = common.read_img(image_path, 'img')
        k = common.random_anisotropic_gaussian_kernel(19)
        hr_tensor = common.np2Tensor([hr], 255)[0]
        # print('hr.max(): {}, {}'.format(hr.max(), hr.mean()))

        input = hr_tensor.unsqueeze(0).permute(1, 0, 2, 3)
        kernel = k.unsqueeze(0) # 1,1,19,19
        save_image(kernel, './k_GT.png',nrow=1,  normalize=True)

        hr_blur = common.conv(input, kernel, padding=19//2)
        hr_blur = hr_blur.permute(1, 0, 2, 3).float() #1,3,H, W
        # print(hr_blur.mean(), hr_blur.max(), hr_blur.min())
        img_blur = util.Tensor2np([hr_blur.squeeze(0)], 255)[0]
        cv2.imwrite('./img_blur.png', cv2.cvtColor(img_blur, cv2.COLOR_BGR2RGB))
        lr_tensor = common.downsample(hr_blur).float()
        lr_img = util.Tensor2np([lr_tensor.squeeze(0)], 255)[0]
        cv2.imwrite('./img_lr.png', cv2.cvtColor(lr_img, cv2.COLOR_BGR2RGB))
        hr_blur = hr_blur.squeeze(0)
        res = model.SR_step(lr_tensor.cuda())
        # res = model.SR_fak(lr_tensor.cuda(), kernel.cuda(), hr_blur.unsqueeze(0).cuda())
        psnr, ssim = calc_psnr(hr, res),\
                                    calc_ssim(hr, res)
        print(psnr, ssim)
        cv2.imwrite(img, res)

main()

ext: py

reason: import numpy


text:
import dash_html_components as html
import numpy as np
import sys

from coralsim.Simulation import Simulation
import shapely.geometry as sg
from pyproj import Proj

# generate table
def generate_table(dataframe, max_rows=10):
    return html.Table([
        html.Thead(
            html.Tr([html.Th(col) for col in dataframe.columns])
        ),
        html.Tbody([
            html.Tr([
                html.Td(dataframe.iloc[i][col]) for col in dataframe.columns
            ]) for i in range(min(len(dataframe), max_rows))
        ])
    ])

# estimate the rough density based on the sites in the selected region
def estimate_density_based_on_selected_sites(selected_data, data):
    if not selected_data:
        return 0

    points = selected_data['points']
    points_ind = [x['pointIndex'] for x in points]

    # if no points within range
    if not points:
        mapbox = selected_data['range']['mapbox']
        center = [(mapbox[0][0] + mapbox[1][0]) / 2, (mapbox[0][1] + mapbox[1][1]) / 2]

        # if the center is too far from historical sites, just return a default value
        if center[0] > max(data['Longitude']) or center[0] < min(data['Longitude']) or center[1] > max(data['Latitude']) or center[1] < min(data['Latitude']):
            return 0.03  # a default value

        # otherwise, use 10 nearest sites to estimate the prop cover
        x = data['Longitude']
        y = data['Latitude']
        res = {}
        for ind, pair in enumerate(zip(x,y)):
            res[ind] = (center[0] - pair[0])**2 + (center[1] - pair[1])**2

        print(res)
        # first 10 ind
        points_ind = [k for k,v in sorted(res.items(), key=lambda item: item[1])[:10]]

        print({k:v for k,v in sorted(res.items(), key=lambda item: item[1])[:10]})

    # calculate the total number of corals in all the selected coral sites
    total_corals = data['count'].iloc[points_ind].sum()

    # considering each coral site has the same area, calculate the rough density estimate of the coral
    single_site_area = 1600
    total_area = single_site_area * len(points_ind)
    prop_cover = total_corals / total_area
    return prop_cover

# generate simulation setting
def generate_setting(dropdown_select_process, prop_cover,
                     disease_prevalence, n_toolkits, fun_lambda, parent_prop,
                     parent_range, strauss_beta, strauss_gamma, strauss_R, selected_data,
                     toolkit_length,
                     toolkit_width,
                     line_intercept_ratio,
                     coral_size,
                     coral_size_sd
                     ):
    # specify some constants
    # coral_size=np.mean(np.array([109, 72, 42, 50])) / 100 ** 2
    # coral_size_sd=0.001
    # toolkit_length = 25
    # toolkit_width = 6
    #line_intercept_ratio = 1 / 6

    # project the lat and lon of the boundary of the selected region to meters
    if 'range' in selected_data.keys():
        temp = selected_data['range']['mapbox']
        data = []

        # Florida projection, record location
        p = Proj('epsg:2337', preserve_units=False)
        data.append(p(temp[0][0], temp[0][1]))
        data.append(p(temp[1][0], temp[1][1]))
        center = ((data[0][0] + data[1][0]) / 2, (data[1][0] + data[1][1]) / 2)
        left_down = (min(data[0][0], data[1][0]), min(data[0][1], data[1][1]))
        left_up = (min(data[0][0], data[1][0]), max(data[0][1], data[1][1]))
        right_down = (max(data[0][0], data[1][0]), min(data[0][1], data[1][1]))
        right_up = (max(data[0][0], data[1][0]), max(data[0][1], data[1][1]))
        boundary = [left_down, left_up, right_up, right_down]
        area = sg.Polygon(boundary)

    #     if area.area < 10000 or area.area > 40000:
    #
    #         # if the area is to small, raise error
    #         if area.area < 10000:
    #             print("The selected region is too small!")
    #             print("the square meters are the selected area is %d" % area.area)
    #             print("Automatically select a region containing your selected region")
    #
    #         else:
    #             print("The selected region is too large!")
    #             print("the square meters are the selected area is %d" % area.area)
    #             print("Automatically select a region within your selected region")
    #
    #         left_down = [(center[0] - 50), (center[1] - 50)]
    #         left_up = [(center[0] - 50), (center[1] + 50)]
    #         right_down = [(center[0] + 50), (center[1] - 50)]
    #         right_up = [(center[0] + 50), (center[1] + 50)]
    #         boundary = [left_down, left_up, right_up, right_down]
    #         area = sg.Polygon(boundary)
    #
    #
    #     #print("area bounds=", area.bounds)
    #
    #
    # else:
    #     # use a constant area
    #     area = sg.Polygon([(0, 0), (0, 100), (100, 100), (100, 0)])

    # core simulation function
    if dropdown_select_process == 1:
        # setting
        setting = Simulation(area, dropdown_select_process, disease_prevalence, coral_size, coral_size_sd, n_toolkits,
                             toolkit_length, toolkit_width, line_intercept_ratio, prop_cover)

    elif dropdown_select_process == 2:
        # deal with the input lambda_fun
        f = eval('lambda x,y: ' + fun_lambda)
        setting = Simulation(area, dropdown_select_process, disease_prevalence, coral_size, coral_size_sd, n_toolkits,
                             toolkit_length, toolkit_width, line_intercept_ratio, fun_lambda=f)

    elif dropdown_select_process == 3:
        setting = Simulation(area, dropdown_select_process, disease_prevalence, coral_size, coral_size_sd, n_toolkits,
                             toolkit_length, toolkit_width, line_intercept_ratio, prop_cover=prop_cover,
                             parent_prop=parent_prop,
                             parent_range=parent_range)

    elif dropdown_select_process == 4:
        setting = Simulation(area, dropdown_select_process, disease_prevalence, coral_size, coral_size_sd, n_toolkits,
                             toolkit_length, toolkit_width, line_intercept_ratio, strauss_beta=strauss_beta,
                             strauss_gamma=strauss_gamma, strauss_R=strauss_R)

    return setting, area, p

max_stars_repo_name: KEHUIYAO/coral-sampling-tool

max_issues_repo_name: KEHUIYAO/coral-sampling-tool

max_forks_repo_name: KEHUIYAO/coral-sampling-tool


text:
# Beidou B3I code construction
#
# Copyright 2018 Peter Monta

import numpy as np

chip_rate = 10230000
code_length = 10230

secondary_code = np.array([0,0,0,0,0,1,0,0,1,1,0,1,0,1,0,0,1,1,1,0])
secondary_code = 1.0 - 2.0*secondary_code

b3i_g2_initial = {
   1: "1010111111111",   2: "1111000101011",   3: "1011110001010",   4: "1111111111011",
   5: "1100100011111",   6: "1001001100100",   7: "1111111010010",   8: "1110111111101",
   9: "1010000000010",  10: "0010000011011",  11: "1110101110000",  12: "0010110011110",
  13: "0110010010101",  14: "0111000100110",  15: "1000110001001",  16: "1110001111100",
  17: "0010011000101",  18: "0000011101100",  19: "1000101010111",  20: "0001011011110",
  21: "0010000101101",  22: "0010110001010",  23: "0001011001111",  24: "0011001100010",
  25: "0011101001000",  26: "0100100101001",  27: "1011011010011",  28: "1010111100010",
  29: "0001011110101",  30: "0111111111111",  31: "0110110001111",  32: "1010110001001",
  33: "1001010101011",  34: "1100110100101",  35: "1101001011101",  36: "1111101110100",
  37: "0010101100111",  38: "1110100010000",  39: "1101110010000",  40: "1101011001110",
  41: "1000000110100",  42: "0101111011001",  43: "0110110111100",  44: "1101001110001",
  45: "0011100100010",  46: "0101011000101",  47: "1001111100110",  48: "1111101001000",
  49: "0000101001001",  50: "1000010101100",  51: "1111001001100",  52: "0100110001111",
  53: "0000000011000",  54: "1000000000100",  55: "0011010100110",  56: "1011001000110",
  57: "0111001111000",  58: "0010111001010",  59: "1100111110110",  60: "1001001000101",
  61: "0111000100000",  62: "0011001000010",  63: "0010001001110",
}

def str2list(s):
  x = []
  for c in s:
    if c=='0':
      x.append(0)
    else:
      x.append(1)
  return x

def b3i_g1_shift(x):
  if x==[1,1,1,1,1,1,1,1,1,1,1,0,0]:
    return [1,1,1,1,1,1,1,1,1,1,1,1,1]
  else:
    return [x[0]^x[2]^x[3]^x[12]] + x[0:12]

def b3i_g2_shift(x):
  return [x[0]^x[4]^x[5]^x[6]^x[8]^x[9]^x[11]^x[12]] + x[0:12]

def b3i(prn):
  n = code_length
  g1 = [1,1,1,1,1,1,1,1,1,1,1,1,1]
  g2 = str2list(b3i_g2_initial[prn])
  b3i = np.zeros(n)
  for i in range(n):
    b3i[i] = g1[12] ^ g2[12]
    g1 = b3i_g1_shift(g1)
    g2 = b3i_g2_shift(g2)
  return b3i

codes = {}

def b3i_code(prn):
  if prn not in codes:
    codes[prn] = b3i(prn)
  return codes[prn]

def code(prn,chips,frac,incr,n):
  c = b3i_code(prn)
  idx = (chips%code_length) + frac + incr*np.arange(n)
  idx = np.floor(idx).astype('int')
  idx = np.mod(idx,code_length)
  x = c[idx]
  return 1.0 - 2.0*x

try:
  from numba import jit
except:
  def jit(**kwargs):
    return lambda x: x

@jit(nopython=True)
def correlate(x,prn,chips,frac,incr,c):
  n = len(x)
  p = 0.0j
  cp = (chips+frac)%code_length
  for i in range(n):
    p += x[i]*(1.0-2.0*c[int(cp)])
    cp = (cp+incr)%code_length
  return p

# test

if __name__=='__main__':
  print(b3i_code(1)[0:20])
  print(b3i_code(2)[0:20])

max_stars_repo_name: wumouyan/GNSS-SDR-Python

max_issues_repo_name: wumouyan/GNSS-SDR-Python

max_forks_repo_name: wumouyan/GNSS-SDR-Python

reason: import numpy,from numba


text:
import PIL
import PIL.Image
import numpy as np
import pandas as pd

import sonaion_heatmaps.masks as masks
import sonaion_heatmaps.addition_kernel as addition_kernel
import scipy
import scipy.ndimage
import matplotlib.pyplot as plt
import cv2
from numba import cuda
from tqdm.notebook import tqdm

def __get_mask_function(dimension, **kwargs):
    mask_function = None

    if 'use_rectangle' in kwargs:
        # Check if kwargs['use_rectangle'] is a boolean value
        if not isinstance(kwargs['use_rectangle'], bool):
            raise TypeError('kwargs[\'use_rectangle\'] must be a boolean value')
        # Check if kwargs has key 'rectangle_width' or 'rectangle_height'
        if not ('rectangle_width' in kwargs or 'rectangle_height' in kwargs):
            raise KeyError('kwargs must have key \'rectangle_width\' or \'rectangle_height\'')

        rectangle_width = kwargs['rectangle_width']
        rectangle_height = kwargs['rectangle_height']

        # Check if rectangle_width is a positive integer
        if not isinstance(rectangle_width, int):
            raise TypeError('rectangle_width must be a positive integer')
        if rectangle_width <= 0:
            raise ValueError('rectangle_width must be a positive integer')
        # Check if rectangle_height is a positive integer
        if not isinstance(rectangle_height, int):
            raise TypeError('rectangle_height must be a positive integer')
        if rectangle_height <= 0:
            raise ValueError('rectangle_height must be a positive integer')

        def tmp_mask_function(x, y, delta_t):
            return masks.get_rectangular_mask((x, y), rectangle_width, rectangle_height, dimension, delta_t)

        mask_function = tmp_mask_function

    elif 'use_circle' in kwargs:
        # Check if kwargs['use_circle'] is a boolean value
        if not isinstance(kwargs['use_circle'], bool):
            raise TypeError('kwargs[\'use_circle\'] must be a boolean value')
        # Check if kwargs has key 'circle_radius'
        if not 'circle_radius' in kwargs:
            raise KeyError('kwargs must have key \'circle_radius\'')

        circle_radius = kwargs['circle_radius']

        # Check if circle_radius is a positive integer
        if not isinstance(circle_radius, int):
            raise TypeError('circle_radius must be a positive integer')
        if circle_radius <= 0:
            raise ValueError('circle_radius must be a positive integer')

        def tmp_mask_function(x, y, delta_t):
            return masks.get_circular_mask((x, y), circle_radius, dimension, delta_t)

        mask_function = tmp_mask_function

    elif 'use_eclipse' in kwargs:
        # Check if kwargs['use_eclipse'] is a boolean value
        if not isinstance(kwargs['use_eclipse'], bool):
            raise TypeError('kwargs[\'use_eclipse\'] must be a boolean value')
        # Check if kwargs has key 'eclipse_radius'
        if not 'eclipse_radius' in kwargs:
            raise KeyError('kwargs must have key \'eclipse_radius\'')

        eclipse_width = kwargs['eclipse_width']
        eclipse_height = kwargs['eclipse_height']

        # Check if eclipse_width is a positive integer
        if not isinstance(eclipse_width, int):
            raise TypeError('eclipse_width must be a positive integer')
        if eclipse_width <= 0:
            raise ValueError('eclipse_width must be a positive integer')
        # Check if eclipse_height is a positive integer
        if not isinstance(eclipse_height, int):
            raise TypeError('eclipse_height must be a positive integer')
        if eclipse_height <= 0:
            raise ValueError('eclipse_height must be a positive integer')

        def tmp_mask_function(x, y, delta_t):
            return masks.get_elliptical_mask((x, y), eclipse_width, eclipse_height, dimension, delta_t)

        mask_function = tmp_mask_function

    else:
        raise KeyError('kwargs must have key \'use_rectangle\', \'use_circle\', or \'use_eclipse\'')

    return mask_function

def __get_mask_function_cuda(dimension, **kwargs):
    mask_function = None

    if 'use_rectangle' in kwargs:
        # Check if kwargs['use_rectangle'] is a boolean value
        if not isinstance(kwargs['use_rectangle'], bool):
            raise TypeError('kwargs[\'use_rectangle\'] must be a boolean value')
        # Check if kwargs has key 'rectangle_width' or 'rectangle_height'
        if not ('rectangle_width' in kwargs or 'rectangle_height' in kwargs):
            raise KeyError('kwargs must have key \'rectangle_width\' or \'rectangle_height\'')

        rectangle_width = kwargs['rectangle_width']
        rectangle_height = kwargs['rectangle_height']

        # Check if rectangle_width is a positive integer
        if not isinstance(rectangle_width, int):
            raise TypeError('rectangle_width must be a positive integer')
        if rectangle_width <= 0:
            raise ValueError('rectangle_width must be a positive integer')
        # Check if rectangle_height is a positive integer
        if not isinstance(rectangle_height, int):
            raise TypeError('rectangle_height must be a positive integer')
        if rectangle_height <= 0:
            raise ValueError('rectangle_height must be a positive integer')

        def tmp_mask_function(heatmap, x, y, delta_t, blockspergrid, threadsperblock):
            addition_kernel.rectangular_addition_kernel[blockspergrid, threadsperblock](heatmap, x, y, rectangle_width, rectangle_height, delta_t, dimension[1], dimension[0])

        mask_function = addition_kernel.circular_addition_kernel

    elif 'use_circle' in kwargs:
        # Check if kwargs['use_circle'] is a boolean value
        if not isinstance(kwargs['use_circle'], bool):
            raise TypeError('kwargs[\'use_circle\'] must be a boolean value')
        # Check if kwargs has key 'circle_radius'
        if not 'circle_radius' in kwargs:
            raise KeyError('kwargs must have key \'circle_radius\'')

        circle_radius = kwargs['circle_radius']

        # Check if circle_radius is a positive integer
        if not isinstance(circle_radius, int):
            raise TypeError('circle_radius must be a positive integer')
        if circle_radius <= 0:
            raise ValueError('circle_radius must be a positive integer')

        def tmp_mask_function(heatmap, x, y, delta_t, blockspergrid, threadsperblock):
            addition_kernel.circular_addition_kernel[blockspergrid, threadsperblock](heatmap, x, y, circle_radius, delta_t, dimension[1], dimension[0])

        mask_function = tmp_mask_function

    elif 'use_eclipse' in kwargs:
        # Check if kwargs['use_eclipse'] is a boolean value
        if not isinstance(kwargs['use_eclipse'], bool):
            raise TypeError('kwargs[\'use_eclipse\'] must be a boolean value')
        # Check if kwargs has key 'eclipse_radius'
        if not 'eclipse_radius' in kwargs:
            raise KeyError('kwargs must have key \'eclipse_radius\'')

        eclipse_width = kwargs['eclipse_width']
        eclipse_height = kwargs['eclipse_height']

        # Check if eclipse_width is a positive integer
        if not isinstance(eclipse_width, int):
            raise TypeError('eclipse_width must be a positive integer')
        if eclipse_width <= 0:
            raise ValueError('eclipse_width must be a positive integer')
        # Check if eclipse_height is a positive integer
        if not isinstance(eclipse_height, int):
            raise TypeError('eclipse_height must be a positive integer')
        if eclipse_height <= 0:
            raise ValueError('eclipse_height must be a positive integer')

        def tmp_mask_function(heatmap, x, y, delta_t, blockspergrid, threadsperblock):
            addition_kernel.elliptical_addition_kernel[blockspergrid, threadsperblock](heatmap, x, y, eclipse_width, eclipse_height, delta_t, dimension[1], dimension[0])

        mask_function = tmp_mask_function

    else:
        raise KeyError('kwargs must have key \'use_rectangle\', \'use_circle\', or \'use_eclipse\'')

    return mask_function

def heatmask(coordinate_array, validity_array, time_array, image, **kwargs):
    """

    :param coordinate_array:    An Array with coordinates in image coordinates
    :param validity_array:      An Array with validity of each coordinate
    :param time_array:          An Array with time of each coordinate
    :param image:               An Image object as PIL object
    :param kwargs:              Keyword arguments
    :kwargs use_rectangle:      If True, draw rectangle around each coordinate
    :kwargs rectangle_width:    Width of rectangle
    :kwargs rectangle_height:   Height of rectangle

    :kwargs use_circle:         If True, draw circle around each coordinate
    :kwargs circle_radius:      Radius of circle

    :kwargs use_eclipse:        If True, draw line between each coordinate
    :kwargs eclipse_width:      Width of eclipse
    :kwargs eclipse_height:     Height of eclipse

    :kwargs blurr:              If True blurr the image using a gaussian filter
    :kwargs blurr_sigma:        Sigma of gaussian filter

    :kwargs use_cuda:           If True, use cuda to accelerate computation
    :kwargs threads_per_block:  Number of threads per block (tuple of 2 integers)

    :return:                    A heatmap image as PIL object
    """
    # Check if image is a PIL object
    if not isinstance(image, PIL.Image.Image):
        raise TypeError('image must be a PIL object')
    # Check if coordinate_array is a numpy array
    if not isinstance(coordinate_array, np.ndarray):
        raise TypeError('coordinate_array must be a numpy array')
    # Check if validity_array is a numpy array
    if not isinstance(validity_array, np.ndarray):
        raise TypeError('validity_array must be a numpy array')
    # Check if time_array is a numpy array
    if not isinstance(time_array, np.ndarray):
        raise TypeError('time_array must be a numpy array')
    # Check if coordinate_array and validity_array have same length
    if len(coordinate_array) != len(validity_array):
        raise ValueError('coordinate_array and validity_array must have same length')
    # Check if coordinate_array and time_array have same length
    if len(coordinate_array) != len(time_array):
        raise ValueError('coordinate_array and time_array must have same length')
    # Check if coordinate_array is a numpy array of shape (n,2)
    if coordinate_array.shape != (len(coordinate_array), 2):
        raise ValueError('coordinate_array must be a numpy array of shape (n,2)')
    # Check if validity_array is a numpy array of shape (n,)
    if validity_array.shape != (len(validity_array),):
        raise ValueError('validity_array must be a numpy array of shape (n,)')
    # Check if time_array is a numpy array of shape (n,)
    if time_array.shape != (len(time_array),):
        raise ValueError('time_array must be a numpy array of shape (n,)')
    # Check if validity_array is a numpy array of boolean values
    if not np.issubdtype(validity_array.dtype, np.bool_):
        raise ValueError('validity_array must be a numpy array of boolean values')
    # Check if time_array is a numpy array of float values
    if not np.issubdtype(time_array.dtype, np.float_):
        raise ValueError('time_array must be a numpy array of float values')
    # Check if image is a PIL object
    if not isinstance(image, PIL.Image.Image):
        raise TypeError('image must be a PIL object')

    # Check if kwargs is a dictionary
    if not isinstance(kwargs, dict):
        raise TypeError('kwargs must be a dictionary')

    threads_per_block = None
    blockspergrid = None
    mask_function = None
    if 'use_cuda' in kwargs:
        if 'threads_per_block' not in kwargs:
            raise ValueError('threads_per_block must be specified if use_cuda is set')
        if not isinstance(kwargs['threads_per_block'], tuple):
            raise TypeError('threads_per_block must be a tuple')
        if len(kwargs['threads_per_block']) != 2:
            raise ValueError('threads_per_block must be a tuple of 2 integers')
        if not isinstance(kwargs['threads_per_block'][0], int):
            raise TypeError('threads_per_block[0] must be an integer')
        if not isinstance(kwargs['threads_per_block'][1], int):
            raise TypeError('threads_per_block[1] must be an integer')
        threads_per_block = kwargs['threads_per_block']
        width, height = image.size
        blockspergrid = (int(np.ceil(height / threads_per_block[1])), int(np.ceil(width / threads_per_block[0])))
        mask_function = __get_mask_function_cuda(image.size, **kwargs)
    else:
        mask_function = __get_mask_function(image.size, **kwargs)

    # Create a new numpy array with same shape as image
    new_image_array = np.zeros((image.size[1], image.size[0]), dtype=np.float32)
    if 'use_cuda' in kwargs:
        new_image_array = cuda.to_device(new_image_array)

    # Iterate over all coordinates
    for (x, y), valid, time in zip(coordinate_array, validity_array, time_array):
        # Check if coordinate is valid
        if valid:
            if 'use_cuda' in kwargs:
                mask_function(new_image_array, x, y, time, blockspergrid, threads_per_block)
            else:
                current_heat = mask_function(x, y, float(time))
                new_image_array += current_heat

    if 'use_cuda' in kwargs:
        new_image_array = new_image_array.copy_to_host()

    # Check if blurr is in kwargs
    if 'blurr' in kwargs:
        # Check if blurr is a boolean
        if not isinstance(kwargs['blurr'], bool):
            raise TypeError('blurr must be a boolean')
        # Check if blurr is True
        if kwargs['blurr']:
            # Check if blurr_sigma is in kwargs
            if 'blurr_sigma' in kwargs:
                # Check if blurr_sigma is a float
                if not isinstance(kwargs['blurr_sigma'], float):
                    raise TypeError('blurr_sigma must be a float')
                # Check if blurr_sigma is positive
                if kwargs['blurr_sigma'] <= 0:
                    raise ValueError('blurr_sigma must be positive')
                # Blurr the image
                new_image_array = scipy.ndimage.gaussian_filter(new_image_array, kwargs['blurr_sigma'])

    return new_image_array

def heatmap_image(coordinate_array, validity_array, time_array, image, **kwargs):
    """

    :param coordinate_array:    An Array with coordinates in image coordinates
    :param validity_array:      An Array with validity of each coordinate
    :param time_array:          An Array with time of each coordinate
    :param image:               An Image object as PIL object
    :param kwargs:              Keyword arguments
    :kwargs use_rectangle:      If True, draw rectangle around each coordinate
    :kwargs rectangle_width:    Width of rectangle
    :kwargs rectangle_height:   Height of rectangle

    :kwargs use_circle:         If True, draw circle around each coordinate
    :kwargs circle_radius:      Radius of circle

    :kwargs use_eclipse:        If True, draw line between each coordinate
    :kwargs eclipse_width:      Width of eclipse
    :kwargs eclipse_height:     Height of eclipse

    :kwargs blurr:              If True blurr the image using a gaussian filter
    :kwargs blurr_sigma:        Sigma of gaussian filter

    :kwargs key_out_threshold:  Threshold for key out in percent of max heat

    :kwargs image_opacity:      Opacity of image in the heatmap

    :kwargs use_cuda:           If True, use cuda to accelerate computation
    :kwargs threads_per_block:  Number of threads per block (tuple of 2 integers)

    :return:                    A Tuple with:
                                    - A numpy array which contains the heatmap image
                                    - A numpy array which contains the heatmask normalized to 1.0
                                    - The maximum heat value before the normalization
    """

    cmap = plt.cm.get_cmap('jet')
    current_heatmask = heatmask(coordinate_array, validity_array, time_array, image, **kwargs)
    max_value = np.max(current_heatmask)
    current_heatmask = current_heatmask / max_value

    # Check if key_out_threshold is in kwargs
    key_out_threshold = 0.0
    if 'key_out_threshold' in kwargs:
        # Check if key_out_threshold is a float
        if not isinstance(kwargs['key_out_threshold'], float):
            raise TypeError('key_out_threshold must be a float')
        # Check if key_out_threshold is between 0 and 1
        if kwargs['key_out_threshold'] < 0 or kwargs['key_out_threshold'] > 1:
            raise ValueError('key_out_threshold must be between 0 and 1')
        key_out_threshold = kwargs['key_out_threshold']

    key_out_mask = current_heatmask >= key_out_threshold
    current_heatmask = current_heatmask * 255
    current_heatmask = current_heatmask.astype(np.uint8)
    current_heatmask = cmap(current_heatmask)
    current_heatmask = current_heatmask * 255
    current_heatmask = current_heatmask.astype(np.uint8)

    # Check if image_opacity is in kwargs
    image_opacity = 0.5
    if 'image_opacity' in kwargs:
        # Check if image_opacity is a float
        if not isinstance(kwargs['image_opacity'], float):
            raise TypeError('image_opacity must be a float')
        # Check if image_opacity is between 0 and 1
        if kwargs['image_opacity'] < 0 or kwargs['image_opacity'] > 1:
            raise ValueError('image_opacity must be between 0 and 1')
        image_opacity = kwargs['image_opacity']

    hmap = np.array(image)
    hmap[key_out_mask] = image_opacity * hmap[key_out_mask] + (1.0 - image_opacity) * current_heatmask[key_out_mask]
    return hmap, current_heatmask, max_value

def heatmap_video(coordinate_array, validity_array, time_array, video_capture, **kwargs):
    """
    :param coordinate_array:        An Array with coordinates in image coordinates
    :param validity_array:          An Array with validity of each coordinate
    :param time_array:              An Array with time of each coordinate in ms
    :param video_capture:           An Image object as PIL object
    :param kwargs:                  Keyword arguments
    :kwargs use_rectangle:          If True, draw rectangle around each coordinate
    :kwargs rectangle_width:        Width of rectangle
    :kwargs rectangle_height:       Height of rectangle

    :kwargs use_circle:             If True, draw circle around each coordinate
    :kwargs circle_radius:          Radius of circle

    :kwargs use_eclipse:            If True, draw line between each coordinate
    :kwargs eclipse_width:          Width of eclipse
    :kwargs eclipse_height:         Height of eclipse

    :kwargs blurr:                  If True blurr the image using a gaussian filter
    :kwargs blurr_sigma:            Sigma of gaussian filter

    :kwargs key_out_threshold:      Threshold for key out in percent of max heat

    :kwargs image_opacity:          Opacity of image in the heatmap

    :kwargs time_range:             Time range into the past for the heatmap in ms
    :kwargs fps:                    Frames per second of the video
    :kwargs fourcc:                 Fourcc of the video
    :kwargs file_name:              Name of the output file
    :kwargs tqdm:                   If True, show a tqdm progress bar
    :kwargs skip_frames:            Number of frames to skip between each frame

    :kwargs use_cuda:               If True, use cuda to accelerate computation
    :kwargs threads_per_block:      Number of threads per block (tuple of 2 integers)

    :return:                        True if successful, False if not
    """

    # Check if coordinate_array is a numpy array
    if not isinstance(coordinate_array, np.ndarray):
        raise TypeError("coordinate_array must be a numpy array")
    # Check if validity_array is a numpy array
    if not isinstance(validity_array, np.ndarray):
        raise TypeError("validity_array must be a numpy array")
    # Check if time_array is a numpy array
    if not isinstance(time_array, np.ndarray):
        raise TypeError("time_array must be a numpy array")
    # Check if coordinate_array and validity_array have the same length
    if len(coordinate_array) != len(validity_array):
        raise ValueError("coordinate_array and validity_array must have the same length")
    # Check if coordinate_array and time_array have the same length
    if len(coordinate_array) != len(time_array):
        raise ValueError("coordinate_array and time_array must have the same length")
    # Check if coordinate_array is a numpy array with shape (n, 2)
    if coordinate_array.shape != (len(coordinate_array), 2):
        raise ValueError("coordinate_array must be a numpy array with shape (n, 2)")
    # Check if validity_array is a numpy array with shape (n,)
    if validity_array.shape != (len(validity_array),):
        raise ValueError("validity_array must be a numpy array with shape (n,)")
    # Check if time_array is a numpy array with shape (n,)
    if time_array.shape != (len(time_array),):
        raise ValueError("time_array must be a numpy array with shape (n,)")
    # Check if validity_array is a numpy array of boolean values
    if not np.issubdtype(validity_array.dtype, np.bool_):
        raise ValueError("validity_array must be a numpy array of boolean values")
    # Check if time_array is a numpy array of float values
    if not np.issubdtype(time_array.dtype, np.float):
        raise ValueError("time_array must be a numpy array of float values")
    # Check if video_capture is a opencv object
    if not isinstance(video_capture, cv2.VideoCapture):
        raise TypeError("video_capture must be a opencv VideoCapture object")

    # Check if kwargs is a dictionary
    if not isinstance(kwargs, dict):
        raise TypeError("kwargs must be a dictionary")

    # Check if key_out_threshold is in kwargs
    key_out_threshold = 0.0
    if 'key_out_threshold' in kwargs:
        # Check if key_out_threshold is a float
        if not isinstance(kwargs['key_out_threshold'], float):
            raise TypeError('key_out_threshold must be a float')
        # Check if key_out_threshold is between 0 and 1
        if kwargs['key_out_threshold'] < 0 or kwargs['key_out_threshold'] > 1:
            raise ValueError('key_out_threshold must be between 0 and 1')
        key_out_threshold = kwargs['key_out_threshold']

    # Check if image_opacity is in kwargs
    image_opacity = 0.5
    if 'image_opacity' in kwargs:
        # Check if image_opacity is a float
        if not isinstance(kwargs['image_opacity'], float):
            raise TypeError('image_opacity must be a float')
        # Check if image_opacity is between 0 and 1
        if kwargs['image_opacity'] < 0 or kwargs['image_opacity'] > 1:
            raise ValueError('image_opacity must be between 0 and 1')
        image_opacity = kwargs['image_opacity']

    dimension = int(video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)), (video_capture.get(cv2.CAP_PROP_FRAME_WIDTH))
    dt = (time_array[1] - time_array[0]) / 1000.0
    df = pd.DataFrame({
        "x": coordinate_array[:, 0],
        "y": coordinate_array[:, 1],
        "validity": validity_array,
        "time": time_array})

    # Create a new numpy array with same shape as image
    destin_fps = kwargs["fps"] if "fps" in kwargs else video_capture.get(cv2.CAP_PROP_FPS)
    skip_frames = kwargs["skip_frames"] if "skip_frames" in kwargs else int(1.0 / (destin_fps * dt))
    file_name = kwargs["file_name"] if "file_name" in kwargs else "./heatmap.mp4"
    fourcc = kwargs["fourcc"] if "fourcc" in kwargs else cv2.VideoWriter_fourcc(*"mp4v")
    use_tqdm = kwargs["tqdm"] if "tqdm" in kwargs else False
    time_range = kwargs["time_range"] if "time_range" in kwargs else time_array[-1]
    # source_fps =  video_capture.get(cv2.CAP_PROP_FPS)
    # frame_count = video_capture.get(cv2.CAP_PROP_FRAME_COUNT)

    cmap = plt.cm.get_cmap(kwargs["cmap"] if "cmap" in kwargs else "jet")
    monitor_size = (int(dimension[1]), int(dimension[0]))
    video_output = cv2.VideoWriter(file_name, fourcc, destin_fps, monitor_size)

    for idx in (tqdm(range(len(df)), total=len(df)) if use_tqdm else df.iterrows()):
        if idx % skip_frames != 0:
            continue
        if not video_output.isOpened():
            raise ValueError("Video output could not be opened")
        current_time_stamp = df.iloc[idx]["time"]
        start_time = max(0, current_time_stamp - time_range)
        df_tmp = df[(df["time"] >= start_time) & (df["time"] <= current_time_stamp)]
        x_values = df_tmp["x"].values
        y_values = df_tmp["y"].values
        current_coordinates = np.stack((x_values, y_values), axis=1)
        current_validity = df_tmp["validity"].values
        current_time = df_tmp["time"].values

        video_capture.set(cv2.CAP_PROP_POS_MSEC, current_time_stamp / 1000.0)
        ret, np_frame = video_capture.read()
        if not ret:
            raise ValueError("Video could not be read")

        frame = PIL.Image.fromarray(np_frame)
        current_heatmask = heatmask(current_coordinates, current_validity, current_time, frame, **kwargs)

        max_value = np.max(current_heatmask)
        if max_value == 0.0:
            max_value = 1.0
        current_heatmask = current_heatmask / max_value

        # Check if key_out_threshold is in kwargs
        key_out_mask = current_heatmask >= key_out_threshold
        current_heatmask = current_heatmask * 255
        current_heatmask = current_heatmask.astype(np.uint8)
        current_heatmask = cmap(current_heatmask)
        current_heatmask = current_heatmask * 255
        current_heatmask = current_heatmask.astype(np.uint8)
        current_heatmask = cv2.cvtColor(current_heatmask, cv2.COLOR_RGB2BGR)

        hmap = np_frame
        current_heatmask = current_heatmask[:, :, :3]
        hmap[key_out_mask] = image_opacity * hmap[key_out_mask] + (1.0 - image_opacity) * current_heatmask[key_out_mask]
        cv2.imshow("Video", hmap)
        if cv2.waitKey(1) == ord("q"):
            break
        video_output.write(hmap)

    video_output.release()
    cv2.destroyAllWindows()
    return True

max_stars_repo_name: sonaion-eye-tracking/Heatmaps

max_issues_repo_name: sonaion-eye-tracking/Heatmaps

max_forks_repo_name: sonaion-eye-tracking/Heatmaps

reason: import numpy,import scipy,from numba


text:
# authors: Maor Marcus & Roy Mazuz
import xlsxwriter
import numpy as np
from qiskit import (execute, Aer)
from qiskit import QuantumCircuit

FILENAME = 'rotator.xlsx'  # the file we write our data to
N, RES = 100, 100  # the number of the mean calculations we want and the resolution we want for the rotator's angle
M = 10  # the number of the measurements we want to do in each mean calculation

def angle_error(p):
    """ gets the probability of getting 0, returns the biased random 0 or 1 """
    simulator = Aer.get_backend('qasm_simulator')
    qc = QuantumCircuit(1, 1)
    initial_state = [np.sqrt(p), np.sqrt(1 - p)]
    qc.initialize(initial_state, 0)
    qc.measure_all()
    # qc.draw(output='mpl', filename='polarization rotator angle.png')  # saving an image of the circuit
    job = execute(qc, simulator, shots=1)
    result = job.result()
    counts = result.get_counts(qc)
    return int(max(counts, key=counts.get)[0])

def mean_angle_error(p, n):
    """ calculates the mean of activating angle_error n times """
    sum = 0
    for k in range(n):
        sum += angle_error(p)
    return sum / n

workbook = xlsxwriter.Workbook(FILENAME)
worksheet = workbook.add_worksheet()
for x in range(RES + 1):
    print(x)
    worksheet.write(0, x, 0.5 * x / RES)
    worksheet.write(0, x + RES, 0.5 + x / RES)
    for y in range(N):
        worksheet.write(y + 1, x, mean_angle_error(0.5 * x / RES, M))  # increase error of angle in each iteration
        worksheet.write(y + 1, x + RES, mean_angle_error(0.5 + 0.5 * x / RES, M))  # now the other way around
workbook.close()

max_stars_repo_path: polarization rotator angle.py

max_issues_repo_path: polarization rotator angle.py

max_forks_repo_path: polarization rotator angle.py


text:
import numpy as np
import pandas as pd
import hashlib
from pathlib import Path
from biom import parse_table
from biom import Table as BiomTable
from omicexperiment.util import parse_fasta, parse_fastq

def load_biom(biom_filepath):
    with open(biom_filepath) as f:
        t = parse_table(f)
    return t

def is_biomtable_object(obj):
    return isinstance(obj, BiomTable)

def biomtable_to_dataframe(biom_table_object):
  _bt = biom_table_object
  data = _bt.matrix_data.todense()
  out = pd.SparseDataFrame(data, index=_bt.ids('observation'),
                           columns=_bt.ids('sample'))
  return out.to_dense()

def biomtable_to_sparsedataframe(biom_table_object):
  _bt = biom_table_object
  m = _bt.matrix_data
  data = [pd.SparseSeries(m[i].toarray().ravel()) for i in np.arange(m.shape[0])]
  out = pd.SparseDataFrame(data, index=_bt.ids('observation'),
                           columns=_bt.ids('sample'))
  return out

def load_biom_as_dataframe(biom_filepath):
    t = load_biom(biom_filepath)
    return biomtable_to_dataframe(t)

def load_fasta(fasta_filepath, calculate_sha1=False):

    descs = []
    seqs = []
    for desc, seq in parse_fasta(fasta_filepath):
        descs.append(desc)
        seqs.append(seq)

    fasta_df = pd.DataFrame({'description': descs, 'sequence': seqs}, columns=['description', 'sequence'])

    del descs
    del seqs

    if calculate_sha1:
        fasta_df['sha1'] = fasta_df['sequence'].apply(lambda x: hashlib.sha1(x.encode('utf-8')).hexdigest())

    return fasta_df

def load_fastq(fastq_filepath, calculate_sha1=False):

    descs = []
    seqs = []
    quals = []

    for desc, seq, qual in parse_fastq(fastq_filepath):
        descs.append(desc)
        seqs.append(seq)
        quals.append(qual)

    fastq_df = pd.DataFrame({'description': descs, 'sequence': seqs, 'qual': quals}, columns=['description', 'sequence', 'qual'])

    del descs
    del seqs
    del quals

    if calculate_sha1:
        fastq_df['sha1'] = fastq_df['sequence'].apply(lambda x: hashlib.sha1(x.encode('utf-8')).hexdigest())

    return fastq_df

def load_fasta_counts(fasta_filepath, sample_name=None):
    fasta_df = load_fasta(fasta_filepath, calculate_sha1=True)
    if sample_name is None:
        counts = fasta_df['sha1'].value_counts().to_frame(name='count')
    else:
        counts = fasta_df['sha1'].value_counts().to_frame(name=sample_name)

    fasta_df.drop('description', axis=1, inplace=True)

    fasta_df.set_index('sha1', inplace=True)

    joined_df = counts.join(fasta_df)
    joined_df.index.name = 'sha1'

    return joined_df.drop_duplicates()

def counts_table_from_fasta_files(fasta_filepaths, sample_names=None):

    if sample_names is None:
        sample_names = [None for i in range(len(fasta_filepaths))]

    concated_df = None
    seq_sha1_df = None

    for fasta, sample_name in zip(fasta_filepaths, sample_names):
        fasta_df = load_fasta_counts(fasta, fasta)
        seq_sha1_df = pd.concat([seq_sha1_df, fasta_df['sequence']])
        fasta_df.drop('sequence', axis=1, inplace=True)
        concated_df = pd.concat([concated_df, fasta_df])
        del fasta_df

    concated_df = concated_df.fillna(0).groupby(level=0).sum()
    concated_df.set_index(seq_sha1_df.drop_duplicates(), append=True, inplace=True)

    return concated_df

def load_fasta_descriptions(fasta_filepath):
    import hashlib
    from skbio.parse.sequences import parse_fasta

    descs = []
    for desc, seq in parse_fasta(fasta_filepath):
        descs.append(desc)

    fasta_df = pd.DataFrame({'description': descs})

    del descs

    return fasta_df

def fasta_df_to_counts_table(fasta_df, desc_to_sampleid_func, index='sha1'):
    fasta_df['sample'] = fasta_df['description'].apply(desc_to_sampleid_func)

    if index == 'sha1' \
    and 'sha1' not in fasta_df.columns:
        fasta_df['sha1'] = fasta_df['sequence'].apply(lambda x: hashlib.sha1(x.encode('utf-8')).hexdigest())

    pivoted = fasta_df.pivot_table(index=index, columns='sample', aggfunc='count', fill_value=0)

    fasta_df.drop(['sample', 'sha1'], axis=1, inplace=True)

    pivoted.columns = pivoted.columns.droplevel()

    #sha1_seqs = pivoted.index.to_series().apply(lambda x: hashlib.sha1(x).hexdigest())
    #sha1_seqs.name = 'sha1'
    #pivoted.set_index(sha1_seqs, append=True, inplace=True)

    return pivoted

def load_uc_file(uc_filepath):
    columns = ['Type', 'Cluster', 'Size', 'Id', 'Strand', 'Qlo', 'Tlo', 'Alignment', 'Query', 'Target']
    df = pd.read_csv(uc_filepath, names=columns, header=None, sep="\t")
    df.rename(columns={'Query': 'observation', 'Cluster': 'cluster'}, inplace=True)

    #check for size annotations and take them away
    sizes_in_labels = df['observation'].apply(lambda x: ';size=' in x).any()

    if sizes_in_labels:
        df['observation'] = df['observation'].apply(lambda x: x.split(';size=')[0])

    df = df[df['Type'] != 'C']
    seeds = df[df['Type'] == 'S']
    df_joined = pd.merge(df, seeds, on='cluster', suffixes=('', '_seed'), left_index=True)
    df_joined.rename(columns={'observation_seed': 'seed'}, inplace=True)

    df_joined.set_index('observation', drop=False, inplace=True)

    return df_joined[['observation','cluster', 'seed']]

def load_swarm_otus_file(swarm_otus_filepath):
    columns = ['amplicon_a', 'amplicon_b', 'differences', 'cluster', 'steps']
    swarms_df = pd.read_csv(swarm_otus_filepath, \
                            names=columns, \
                            sep="\t")
    duplicate_amplicons = swarms_df.drop_duplicates('amplicon_a')
    duplicate_amplicons['amplicon_b'] = duplicate_amplicons['amplicon_a']
    concat_df = pd.concat([swarms_df, duplicate_amplicons]).drop_duplicates('amplicon_b')
    concat_df.rename(columns={'amplicon_b': 'observation'}, inplace=True)
    concat_df.set_index('observation', drop=False, inplace=True)
    return concat_df[['observation', 'cluster']].sort_values('cluster')

def load_qiime_otu_assignment_file(otu_assignment_filepath):
    with open(otu_assignment_filepath) as f:
        lines = f.readlines()

    observation_list = []
    otu_list = []

    for l in lines:
        splt = l.split()
        otu_name = splt[0].strip()
        observations = splt[1:]
        for obs in observations:
            obs = obs.strip()
            observation_list.append(obs)
            otu_list.append(otu_name)

    observation_to_otu_dict = OrderedDict(observation=observation_list, otu=otu_list)

    del observation_list
    del otu_list

    df = pd.DataFrame(observation_to_otu_dict, index=observation_list)
    df.set_index('observation', drop=False, inplace=True)

    return df

def load_qiime_otu_map_file(otu_map_file):
    obs_cluster_tuples = []

    with open(otu_map_file) as f:
        for l in f.readlines():
            reads = [r.strip() for r in l.split("\t")]
            seed = reads[0]
            rest = reads[1:]

            for obs in rest:
                obs_cluster_tuples.append((obs, seed))

    return pd.DataFrame.from_records(obs_cluster_tuples, columns=['observation', 'cluster'])

def load_qiime_taxonomy_assignment_file(tax_assignments_file):
    tax_file_df = pd.read_csv(tax_assignments_file, sep="\t", header=None, names=['otu','tax','evalue','tax_id'])

    if tax_file_df['otu'][0] == "#OTU ID":
        tax_file_df = pd.read_csv(tax_assignments_file, sep="\t", header=0, names=['otu','tax','evalue','tax_id'])

    return tax_file_df

def load_dataframe(input_file_or_obj, first_col_in_file_as_index=True):
    if isinstance(input_file_or_obj, (str, Path)):
        #assume file path
        fp = Path(input_file_or_obj)
        assert(fp.exists())
        if fp.suffix == '.biom':
            df = load_biom_as_dataframe(str(fp))
            return df
        elif fp.suffix == '.csv':
            index_col = 0 if first_col_in_file_as_index else None
            df = pd.read_csv(str(fp), index_col=index_col)
            return df
        elif fp.suffix == '.tsv':
            index_col = 0 if first_col_in_file_as_index else None
            df = pd.read_csv(str(fp), sep='\t', index_col=index_col)
            return df

    elif isinstance(input_file_or_obj, pd.DataFrame):
        return input_file_or_obj

    elif isinstance(input_file_or_obj, BiomTable):
        return biomtable_to_dataframe(input_file_or_obj)

max_stars_repo_name: bassio/omicexperiment

max_issues_repo_name: bassio/omicexperiment

max_forks_repo_name: bassio/omicexperiment


text:
import tensorflow as tf
import numpy as np
import argparse
from eventnet import *
import pickle
import time
import os
import networkx as nx
from sklearn import preprocessing

import event2vec

def main():
    ## Parameters setting
    #           beta    rep_dim     epochs      batch_size      learning_rate
    # Movielens: 30        64        2000           128             0.01
    # DBLP     : 30        64        200            128             0.01
    # Douban   : 30        64        200            128             0.01
    # IMDB     : 2         64        200            128             0.01
    # Yelp     : 30        64        2000           128             0.01

    parser = argparse.ArgumentParser()
    parser.add_argument('--beta', default=30)
    parser.add_argument('--representation_dim', default=64)
    parser.add_argument('--epochs', default=200)
    parser.add_argument('--batch_size', default=128)
    parser.add_argument('--learning_rate', default=0.01)

    parser.add_argument('--graph_file', default='dblp/dblp.txt')
    # parser.add_argument('--graph_file', default='douban/douban.txt')
    # parser.add_argument('--graph_file', default='imdb/imdb.txt')
    # parser.add_argument('--graph_file', default='yelp/yelp.txt')

    parser.add_argument('--event_id_idx', default=[0]) # for dblp | douban | yelp
    # parser.add_argument('--event_id_idx', default=[1]) # for imdb

    parser.add_argument('--node_types', default=['a', 'p', 'c', 't']) # for dblp
    # parser.add_argument('--node_types', default=['m', 'a', 'd', 'u']) # douban
    # parser.add_argument('--node_types', default=['a', 'm', 'u', 'd']) # imdb
    # parser.add_argument('--node_types', default=['b', 'u', 'l', 'c']) # yelp

    parser.add_argument('--output_file', default='output/dblp/dblp_e2v_embeddings.txt')
    # parser.add_argument('--output_file', default='output/douban/douban_e2v_embeddings.txt')
    # parser.add_argument('--output_file', default='output/imdb/imdb_e2v_embeddings.txt')
    # parser.add_argument('--output_file', default='output/yelp/yelp_e2v_embeddings.txt')

    args = parser.parse_args()
    train(args)

def train(args):
    data = LoadData()
    data.load_data(filename=args.graph_file, event_id_idx=args.event_id_idx)

    nodes_num = data.nodes_num
    event_num = data.event_num
    node_ind = data.node_ind
    event_dict_int = data.event_dict_int

    node_types = args.node_types

    del data

    event = EventNet(event_dict=event_dict_int,
                    nodes_num=nodes_num,
                    event_num=event_num,
                    node_types=node_types)

    node_deg = event.node_deg
    inc_mat = event.inc_mat

    print('number of events: {}'.format(event_num))
    print(nodes_num)
    model = event2vec.EVENT2VEC(nodes_num=nodes_num,
                                node_types=node_types,
                                nodes_ind=node_ind,
                                beta=args.beta,
                                rep_size=args.representation_dim,
                                epochs=args.epochs,
                                batch_size=args.batch_size,
                                learning_rate=args.learning_rate)

    for node_type in node_types:
        inc_mat[node_type] = inc_mat[node_type].T

    t1 = time.time()
    model.train(inc_mat)
    t2 = time.time()
    print('training time: %s'%(t2-t1))

    model.save_embeddings(args.output_file, inc_mat, node_deg)

if __name__ == '__main__':
    main()

reason: import numpy,import networkx


text:
# Copyright 2018-2021 Xanadu Quantum Technologies Inc.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Benchmarks for a circuit training application.
"""
from numpy.random import random

import pennylane as qml
import tensorflow as tf
import torch
from pennylane import numpy as pnp
from .default_settings import _core_defaults

def benchmark_optimization(hyperparams={}, n_steps=20, num_repeats=1):
    """Trains a quantum circuit for n_steps steps with a gradient descent optimizer.

    Args:
    hyperparams (dict): hyperparameters to configure this benchmark

            * 'n_wires': Number of wires to use. Will be ignored if custom device and template are provided.

            * 'n_layers': Number of layers in the default template. Will be ignored if custom params are provided.

            * 'diff_method': name of differentiation method

            * 'device': device on which the circuit is run, or valid device name

            * 'interface': name of the interface to use

            * 'template': Template to use. The template must take the trainable parameters as its only argument.

            * 'params': Numpy array of trainable parameters that is fed into the template.

            * 'measurement': measurement function like `qml.expval(qml.PauliZ(0)))`

    n_steps (int): number of optimization steps
    num_repeats (int): How often the same circuit is evaluated in a for loop. Default is 1.
    """

    device, diff_method, interface, params, template, measurement = _core_defaults(hyperparams)

    @qml.qnode(device, interface=interface, diff_method=diff_method)
    def circuit(params_):
        template(params_)
        measurement.queue()
        return measurement

    for _ in range(num_repeats):

        if interface == "autograd":
            params = pnp.array(params, requires_grad=True)
            opt = qml.GradientDescentOptimizer(stepsize=0.1)

            for i in range(n_steps):
                params = opt.step(circuit, params)

        elif interface == "tf":
            params = tf.Variable(params)
            opt = tf.keras.optimizers.SGD(learning_rate=0.1)

            for i in range(n_steps):
                with tf.GradientTape() as tape:
                    loss = circuit(params)
                gradients = tape.gradient(loss, [params])
                opt.apply_gradients(zip(gradients, [params]))

        elif interface == "torch":
            params = torch.tensor(params, requires_grad=True)
            opt = torch.optim.SGD([params], lr=0.1)

            def closure():
                opt.zero_grad()
                loss = circuit(params)
                loss.backward()
                return loss

            for i in range(n_steps):
                opt.step(closure)

    # TODO: jax

max_stars_repo_name: PennyLaneAI/benchmark

max_issues_repo_name: PennyLaneAI/benchmark

max_forks_repo_name: PennyLaneAI/benchmark

reason: import numpy,from numpy


text:
import os
import sys
import shutil
import pathlib
import collections
from typing import List, Union
import abc
import pprint
import tqdm

import pdb
import logging

import random
import numpy as np
import cv2
import imutils
from numpy.lib.arraysetops import isin
import skimage
import skimage.io
import skimage.transform
import albumentations as albu

import torch
import torchvision
import torch.utils
import torch.utils.tensorboard
import torchvision.transforms.functional

import segmentation_models_pytorch as smp
import pytorch_lightning as pl

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.cm

# Local Imports
sys.path.append("/home/students/edavalos/GitHub/FastPoseCNN/source_code/FastPoseCNN")
import setup_env

root = next(path for path in pathlib.Path(os.path.abspath(__file__)).parents if path.name == 'FastPoseCNN')
sys.path.append(str(pathlib.Path(__file__).parent))

import json_tools as jt
import data_manipulation as dm
import project as pj
import draw as dr
import visualize as vz
import transforms

#-------------------------------------------------------------------------------
# File Constants

ENCODER = 'resnext50_32x4d'
ENCODER_WEIGHTS = 'imagenet'
LOGGER = logging.getLogger('fastposecnn')

#-------------------------------------------------------------------------------
# Dataset-related functions

def seed_worker(worker_id):
    worker_seed = torch.initial_seed() % 2**32
    np.random.seed(worker_seed)
    random.seed(worker_seed)

#-------------------------------------------------------------------------------
# Custom Abstract Dataset

class Dataset(abc.ABC):

    @abc.abstractmethod
    def get_random_batched_sample(self):
        ...

    @property
    @abc.abstractmethod
    def CLASSES(self) -> List:
        ...

    @property
    @abc.abstractmethod
    def SYMMETRIC_CLASSES(self) -> List:
        ...

    @property
    @abc.abstractmethod
    def COLORMAP(self) -> np.ndarray:
        ...

    @property
    @abc.abstractmethod
    def INTRINSICS(self) -> np.ndarray:
        ...

#-------------------------------------------------------------------------------
# Pose Regression Datasets

class NOCSDataset(Dataset, torch.utils.data.Dataset):
    """NOCS Dataset. Read images, apply augmentation and preprocessing transformations.

    Args:
        dataset_dir (str): filepath to the dataset (train, valid, or test)
        max_size (int): maximum number of samples
        augmentation (albumentations.Compose): data transfromation pipeline
            (e.g. flip, scale, etc.)
        preprocessing (albumentations.Compose): data preprocessing
            (e.g. noralization, shape manipulation, etc.)
    """

    def __init__(
        self,
        dataset_dir: pathlib.Path,
        max_size: Union[int, None] = None,
        classes: Union[List, None] = None,
        augmentation: Union[albu.Compose, None] = None,
        preprocessing: Union[albu.Compose, None] = None
        ):

        # ! Debugging
        self.counter = 0

        # If None or just all the classes, no nead of class values map
        if classes is None:
            self.classes = self.CLASSES

        # then create class values map
        self.classes = classes
        self.class_values_map = {self.CLASSES.index(cls.lower()):self.classes.index(cls) for cls in self.classes}
        self.symmetric_classes = [self.classes.index(cls.lower()) for cls in self.SYMMETRIC_CLASSES if cls in self.classes]

        # Obtaining the filepaths for the images
        self.images_fps = self.get_image_paths_in_dir(dataset_dir, max_size=max_size)

        # Saving parameters
        self.augmentation = augmentation
        self.preprocessing = preprocessing

    def __getitem__(self, i):

        # DEBUGGING
        """
        if self.counter == 0:
            image_path = '/home/students/edavalos/GitHub/FastPoseCNN/datasets/NOCS/camera/train/27102/0001_color.png'
        elif self.counter == 1:
            image_path = '/home/students/edavalos/GitHub/FastPoseCNN/datasets/NOCS/camera/train/16523/0008_color.png'
        elif self.counter == 2:
            image_path = '/home/students/edavalos/GitHub/FastPoseCNN/datasets/NOCS/camera/train/10915/0000_color.png'

        self.images_fps[i] = pathlib.Path(image_path)
        self.counter += 1
        self.counter = (self.counter % 3)
        #"""

        #LOGGER.debug(f"LOADING IMAGE: {str(self.images_fps[i])}")

        # Reading data
        # Image
        image = skimage.io.imread(str(self.images_fps[i]))

        # Mask
        mask_fp = str(self.images_fps[i]).replace('_color.png', '_mask.png')

        if isinstance(self, CAMERADataset):
            mask = skimage.io.imread(mask_fp, 0)[:,:,0].astype('float')
        elif isinstance(self, REALDataset):
            mask = skimage.io.imread(mask_fp, 0).astype('float')
        else:
            raise NotImplementedError("Invalid dataset type")

        # Change the background from 255 to 0
        mask[mask == 255] = 0

        # Depth
        depth_fp = str(self.images_fps[i]).replace('_color.png', '_depth.png')
        depth = cv2.imread(depth_fp, -1) # has to be cv2, not skimage
        depth = dm.standardize_depth(depth)

        # Other data
        json_fp = str(self.images_fps[i]).replace('_color.png', '_meta+.json')
        json_data = jt.load_from_json(json_fp)

        # Removing distraction objects
        instances_mask = np.zeros_like(mask)
        for instance_id in json_data['instance_dict'].keys():
            instances_mask[mask == int(instance_id)] = int(instance_id)

        # Removing unwanted classes
        good_json_data = {'instance_dict': {}}
        good_instances_mask = np.zeros_like(instances_mask)

        # Iterating through all json data and keeping only data that is the selected class
        for enumerate_id, (id_value, class_id) in enumerate(json_data['instance_dict'].items()):

            # If the class is in the wanted class values, then keep it
            if class_id in self.class_values_map.keys():
                good_instances_mask[instances_mask == int(id_value)] = int(id_value)

                good_json_data['instance_dict'][int(id_value)] = self.class_values_map[class_id]

                for key in json_data.keys():
                    if key != 'instance_dict':
                        if key in good_json_data.keys():
                            good_json_data[key].append(json_data[key][enumerate_id])
                        else:
                            good_json_data[key] = [json_data[key][enumerate_id]]

        # Stacking data
        for key in good_json_data.keys():
            if key != 'instance_dict':
                good_json_data[key] = np.stack(good_json_data[key])

        # Generate the aggregated data (instance-lead data)
        agg_data = self.generate_agg_data(good_instances_mask, good_json_data)

        # Check if any data is invalid, if it is, simply return None for the sample
        if (agg_data['z'] <= 0).any():
            LOGGER.debug(f"{str(self.images_fps[i])} -> INVALID/CORRUPT SAMPLE: z <= 0, class_ids: {agg_data['class_ids']}")
            return None

        # Create dense representation of the data (class type is instances)
        #quaternions = dm.create_dense_quaternion(good_instances_mask, good_json_data)
        #scales = dm.create_dense_scales(good_instances_mask, good_json_data)
        #xy, z = dm.create_dense_3d_centers(good_instances_mask, good_json_data, self.INTRINSICS)

        # Generating class mask with the desired object class
        class_mask = np.zeros_like(good_instances_mask)
        for instance_id, class_id in good_json_data['instance_dict'].items():
            class_mask[good_instances_mask == int(instance_id)] = class_id

        # Storing mask and image into sample
        sample = {
            'clean_image': image,
            'image': image,
            'mask': class_mask,
            'depth': depth # For testing purposes only
            #'quaternion': quaternions,
            #'scales': scales,
            #'xy': xy,
            #'z': z
        }

        # apply augmentations
        """
        if self.augmentation:
            sample = self.augmentation(**sample)
        """

        # apply preprocessing
        if self.preprocessing:
            sample = self.preprocessing(**sample)

        # Converting numpyt to Torch dataformat convention
        sample = transforms.pose.numpy_to_torch()(**sample)

        # Normalize to maintain the -1 to +1 magnitude
        if sample['image'].dtype != np.uint8:
            sample['image'] /= np.max(np.abs(sample['image']))

        # Changing dtype
        sample.update({
            'path': self.images_fps[i],
            'image': skimage.img_as_float32(sample['image']),
            'mask': sample['mask'].astype('long'),
            'depth': sample['depth'].astype('float32'),
            #'quaternion': skimage.img_as_float32(sample['quaternion']),
            #'scales': skimage.img_as_float32(sample['scales']),
            #'xy': skimage.img_as_float32(sample['xy']),
            #'z': skimage.img_as_float32(sample['z']),
            'agg_data': agg_data
        })

        return sample

    def __len__(self):
        return len(self.images_fps)

    def get_image_paths_in_dir(self, dir_path, max_size=None):
        """
        Args:
            dir_path (pathlib object): The directory to be search for all possible
                color images
        Objective:
            Perform a search inside dir_path to collect all available color images
            and save them into a list
        Output:
            total_path_list (list): A list of all collected color images
        """

        total_path_list = [] # [[color, depth], ...]

        # handling additional directories
        eval_paths = [dir_path]
        i = 0

        while True:

            # Break condition: if no more directories to evaluate, end
            if len(eval_paths) <= i:
                break

            # Selecting the new path to evaluate
            eval_path = eval_paths[i]
            i += 1

            # for all the evaluate paths, do the following
            files = [x for x in eval_path.iterdir() if x.is_file()]

            # Adding all the color images into the total_path_list
            color_images = [x for x in files if x.name.find('color') != -1 and x.suffix == '.png']

            # Removing empty color images that do not have instances
            good_color_images = self.remove_empty_samples(color_images)
            total_path_list += good_color_images

            directories = [x for x in eval_path.iterdir() if x.is_dir()]
            eval_paths += directories

            # If max size is reached or overpassed, break from loop
            if max_size != None:
                if len(total_path_list) >= max_size:
                    break

        # removing any falsy elements
        total_path_list = [x for x in total_path_list if x]

        # Trimming excess if dataset_max_size is set
        if max_size != None:
            total_path_list = total_path_list[:max_size]

        return total_path_list

    def remove_empty_samples(self, file_paths):

        good_samples_fps = []

        for i in range(len(file_paths)):

            # Obtain the json data
            json_fp = str(file_paths[i]).replace('_color.png', '_meta+.json')
            json_data = jt.load_from_json(json_fp)

            # Get the instance data
            instance_dict = json_data['instance_dict']

            # Trim the classes that are not wanted
            good_instance_dict = {}
            for id_value, class_value in json_data['instance_dict'].items():

                # If the class is in the wanted class values, then keep it
                if class_value in self.class_values_map.keys():
                    good_instance_dict[int(id_value)] = self.class_values_map[class_value]

            # If instances, save them as good samples
            if good_instance_dict:
                good_samples_fps.append(file_paths[i])

        return good_samples_fps

    def get_random_batched_sample(self, batch_size=1, device=None):

        all_samples = []

        # Obtain all the sample needed
        for sample_id in np.random.choice(np.arange(self.__len__()), size=batch_size, replace=False):
            sample = self.__getitem__(sample_id)
            all_samples.append(sample)

        # Collate the samples
        batch = my_collate_fn(all_samples, device)

        return batch

    def generate_agg_data(self, instances_mask, json_data):

        # Obtaing the size of the images
        h, w = instances_mask.shape

        # Determining the number of instances
        num_of_instances = np.unique(instances_mask).shape[0] - 1

        # Creating a pure instances image
        agg_data = {
            'class_ids': np.zeros((num_of_instances,)),
            'symmetric_ids': np.zeros((num_of_instances,)),
            'instance_masks': np.zeros((num_of_instances, h, w)),
            'quaternion': np.zeros((num_of_instances, 4)),
            'scales': np.zeros((num_of_instances, 3)),
            'xy': np.zeros((num_of_instances, 2)),
            'z': np.zeros((num_of_instances, 1)),
            'T': np.zeros((num_of_instances, 3)),
            'R': np.zeros((num_of_instances, 3, 3)),
            'RT': np.zeros((num_of_instances, 4, 4))
        }

        # Modify the json data to work well (missing xy, z, R, and T, and
        # renaming quaternions to quaternion)
        json_data['quaternion'] = json_data['quaternions']
        json_data['RT'] = json_data['RTs']
        xyz_R_T = dm.extract_xyz_R_T_from_RTs(json_data['RTs'], self.INTRINSICS)
        json_data.update(xyz_R_T)

        # Filling in the data for each instance
        for enumerate_id, (instance_id, class_id) in enumerate(json_data['instance_dict'].items()):
            for data_name in agg_data.keys():

                # Different to account for datatype mismatch
                if data_name == 'class_ids':
                    agg_data[data_name][enumerate_id] = json_data['instance_dict'][instance_id]

                # Marking 1 for symmetric, 0 for non-symmetric classes
                elif data_name == 'symmetric_ids':
                    symmetric_ids = 1 * (json_data['instance_dict'][instance_id] in self.symmetric_classes)
                    agg_data[data_name][enumerate_id] = symmetric_ids

                # Use the mask instead of json_data
                elif data_name == 'instance_masks':
                    agg_data[data_name][enumerate_id] = np.where(instances_mask == instance_id, 1, 0)

                else:
                    # Grabbing the data to test data type
                    insert_data = json_data[data_name]
                    # Ensuring that the data is an numpy array
                    if isinstance(instance_id, np.ndarray) is False:
                        insert_data = np.array(insert_data)
                    # Storing data into larger numpy array (instances)
                    agg_data[data_name][enumerate_id] = insert_data[enumerate_id]

        # Reducing the size of the object
        agg_data['scales'] /= np.expand_dims(json_data['norm_factors'], axis=1)

        # Flip the xy to match the other style
        agg_data['xy'] = np.flip(agg_data['xy'], axis=1)

        return agg_data

class CAMERADataset(NOCSDataset):

    CLASSES = pj.constants.CAMERA_CLASSES
    SYMMETRIC_CLASSES = pj.constants.CAMERA_SYMMETRIC_CLASSES
    COLORMAP = pj.constants.COLORMAP['CAMERA']
    INTRINSICS = pj.constants.INTRINSICS['CAMERA']

class REALDataset(NOCSDataset):

    CLASSES = pj.constants.CAMERA_CLASSES
    SYMMETRIC_CLASSES = pj.constants.REAL_SYMMETRIC_CLASSES
    COLORMAP = pj.constants.COLORMAP['REAL']
    INTRINSICS = pj.constants.INTRINSICS['REAL']

#-------------------------------------------------------------------------------
# Dataloader

def my_collate_fn(batch, device=None):

    # Filtering any samples that were deemed corrupted or invalid
    batch = list(filter(lambda x : x is not None, batch))

    # If the batch is empty, simply return None
    if not batch:
        return None

    collate_batch = {}
    agg_data = {
        'sample_ids': []
    }

    # Iterating overall the data input data
    for sample_id, single_sample in enumerate(batch):
        for key in single_sample.keys():

            # Instance-based data
            if key == 'agg_data':
                continue

            # Placing all the same type of data into their containers
            if key in collate_batch.keys():
                collate_batch[key].append(single_sample[key])
            else:
                collate_batch[key] = [single_sample[key]]

        # If there is agg_data, the np.concat it
        if 'agg_data' in single_sample.keys():

            # Get subkey from the agg_data dictionary
            for subkey in single_sample['agg_data'].keys():

                # Placing all the same type of data into their containers
                if subkey in agg_data.keys():
                    agg_data[subkey].append(single_sample['agg_data'][subkey])
                else:
                    agg_data[subkey] = [single_sample['agg_data'][subkey]]

            # Appending sample_ids key to this to mark the sample to instances
            agg_data['sample_ids'].append(np.repeat(np.array(sample_id), single_sample['agg_data']['class_ids'].shape[0]))

    # Containers for stacked and concatinated data
    stacked_collate_batch = {}
    concated_agg_data = {}

    # If device is specified,
    if device:
        # Now stacking all the uniform data
        for key in collate_batch.keys():
            if isinstance(collate_batch[key][0], np.ndarray):
                stacked_collate_batch[key] = torch.from_numpy(np.stack(collate_batch[key])).to(device)
            else:
                stacked_collate_batch[key] = collate_batch[key]

        # Now concatinating all non-uniform data
        for subkey in agg_data.keys():
            concated_agg_data[subkey] = torch.from_numpy(np.concatenate(agg_data[subkey], axis=0)).to(device)

    # If no device is specified
    else:
        # Now stacking all the uniform data
        for key in collate_batch.keys():
            if isinstance(collate_batch[key][0], np.ndarray):
                stacked_collate_batch[key] = torch.from_numpy(np.stack(collate_batch[key]))
            else:
                stacked_collate_batch[key] = collate_batch[key]

        # Now concatinating all non-uniform data
        for subkey in agg_data.keys():
            concated_agg_data[subkey] = torch.from_numpy(np.concatenate(agg_data[subkey], axis=0))

    # Now storing agg data into the collate batch
    stacked_collate_batch['agg_data'] = concated_agg_data

    return stacked_collate_batch

#-------------------------------------------------------------------------------
# PyTorch-Lightning DataModule

class PoseRegressionDataModule(pl.LightningDataModule):

    def __init__(
        self,
        dataset_name='NOCS',
        batch_size=1,
        num_workers=0,
        selected_classes=None,
        encoder=None,
        encoder_weights=None,
        train_size=None,
        valid_size=None,
        is_deterministic=False
        ):

        super().__init__()

        # Saving parameters
        self.dataset_name = dataset_name
        self.selected_classes = selected_classes
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.encoder = encoder
        self.encoder_weights = encoder_weights
        self.train_size = train_size
        self.valid_size = valid_size
        self.is_deterministic = is_deterministic

    def setup(self, stage=None):

        # Obtaining the preprocessing_fn depending on the encoder and the encoder
        # weights
        if self.encoder and self.encoder_weights:
            preprocessing_fn = smp.encoders.get_preprocessing_fn(self.encoder, self.encoder_weights)
        else:
            preprocessing_fn = None

        # CAMERA / NOCS
        if self.dataset_name == 'CAMERA':

            train_dataset = CAMERADataset(
                dataset_dir=pathlib.Path(os.getenv("NOCS_CAMERA_TRAIN_DATASET")),
                max_size=self.train_size,
                classes=self.selected_classes,
                augmentation=transforms.pose.get_training_augmentation(),
                preprocessing=transforms.pose.get_preprocessing(preprocessing_fn)
            )

            valid_dataset = CAMERADataset(
                dataset_dir=pathlib.Path(os.getenv("NOCS_CAMERA_VALID_DATASET")),
                max_size=self.valid_size,
                classes=self.selected_classes,
                augmentation=transforms.pose.get_validation_augmentation(),
                preprocessing=transforms.pose.get_preprocessing(preprocessing_fn)
            )

            self.datasets = {
                'train': train_dataset,
                'valid': valid_dataset
            }

        # REAL / NOCS
        elif self.dataset_name == 'REAL':

            train_dataset = REALDataset(
                dataset_dir=pathlib.Path(os.getenv("NOCS_REAL_TRAIN_DATASET")),
                max_size=self.train_size,
                classes=self.selected_classes,
                augmentation=transforms.pose.get_training_augmentation(),
                preprocessing=transforms.pose.get_preprocessing(preprocessing_fn)
            )

            valid_dataset = REALDataset(
                dataset_dir=pathlib.Path(os.getenv("NOCS_REAL_TEST_DATASET")),
                max_size=self.valid_size,
                classes=self.selected_classes,
                augmentation=transforms.pose.get_validation_augmentation(),
                preprocessing=transforms.pose.get_preprocessing(preprocessing_fn)
            )

            self.datasets = {
                'train': train_dataset,
                'valid': valid_dataset
            }

        # INVALID DATASET
        else:
            raise RuntimeError('Dataset needs to be selected')

        print(f"Training datset size: {len(self.datasets['train'])}")
        print(f"Validation dataset size: {len(self.datasets['valid'])}")

    def get_loader(self, dataset_key):

        if dataset_key in self.datasets.keys():

            # Constructing general params
            params = {
                'dataset': self.datasets[dataset_key],
                'num_workers': self.num_workers,
                'batch_size': self.batch_size,
                'collate_fn': my_collate_fn,
                'pin_memory': True
            }

            # If deterministic, don't shuffle and provide seed_worker function
            if self.is_deterministic:
                params.update({
                    'worker_init_fn': seed_worker,
                    'shuffle': False
                })
            else: # else, enable shuffle
                params.update({
                    'shuffle': True
                })

            # Passing parameters
            dataloader = torch.utils.data.DataLoader(**params)

            return dataloader

        else:

            return None

    def train_dataloader(self):
        return self.get_loader('train')

    def val_dataloader(self):
        return self.get_loader('valid')

    def test_dataloader(self):
        return self.get_loader('test')

#-------------------------------------------------------------------------------
# Functions

def move_dict_to(a: dict, device):

    for key in a.keys():
        if isinstance(a[key], torch.Tensor):
            a[key] = a[key].to(device)

    return a

def move_batch_to(batch: dict, device):

    device_batch = {}

    for key in batch.keys():

        if key == 'agg_data':
            agg_data_dict = move_dict_to(batch[key], device)
            device_batch[key] = agg_data_dict

        else:

            if isinstance(batch[key], torch.Tensor):
                device_batch[key] = batch[key].to(device)
            else:
                device_batch[key] = batch[key]

    return device_batch

def test_pose_camera_dataset():

    preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)

    dataset = CAMERADataset(
        dataset_dir=pathlib.Path(os.getenv("NOCS_CAMERA_TRAIN_DATASET")),
        max_size=300,
        classes=pj.constants.CAMERA_CLASSES,
        augmentation=transforms.pose.get_training_augmentation(),
        preprocessing=transforms.pose.get_preprocessing(preprocessing_fn)
    )

    # Testing dataloader
    dataloader = torch.utils.data.DataLoader(
        dataset,
        batch_size=3,
        collate_fn=my_collate_fn,
        shuffle=False
    )

    # Iterative through data batches
    for i, batch in tqdm.tqdm(enumerate(dataloader)):
        fig = vz.visualize_gt_pose(batch, dataset.COLORMAP, dataset.INTRINSICS)
        fig_path = pathlib.Path(os.getenv("TEST_OUTPUT")) / f'quat_pose{i}.png'
        fig.savefig(str(fig_path), dpi=400)

#-------------------------------------------------------------------------------
# Main Code

if __name__ == '__main__':

    # For testing the datasets

    # camera dataset
    test_pose_camera_dataset()

max_stars_repo_name: edavalosanaya/MastersProject

max_issues_repo_name: edavalosanaya/FastPoseCNN

max_forks_repo_name: edavalosanaya/FastPoseCNN